build-logic: add crowdin and psl convention plugins

build-logic/automation-plugins/build.gradle.kts

@@ -1 +1,27 @@
-plugins { `kotlin-dsl` }
+/*
+ * Copyright © 2014-2021 The Android Password Store Authors. All Rights Reserved.
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+plugins {
+  `kotlin-dsl`
+  `kotlin-dsl-precompiled-script-plugins`
+}
+
+gradlePlugin {
+  plugins {
+    register("crowdin") {
+      id = "com.github.android-password-store.crowdin-plugin"
+      implementationClass = "crowdin.CrowdinDownloadPlugin"
+    }
+    register("psl") {
+      id = "com.github.android-password-store.psl-plugin"
+      implementationClass = "psl.PublicSuffixListPlugin"
+    }
+  }
+}
+
+dependencies {
+  implementation(libs.build.download)
+  implementation(libs.build.okhttp)
+}

build-logic/automation-plugins/src/main/kotlin/crowdin/CrowdinExtension.kt

@@ -0,0 +1,20 @@
+/*
+ * Copyright © 2014-2021 The Android Password Store Authors. All Rights Reserved.
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+package crowdin
+
+/** Extension for configuring [CrowdinPlugin] */
+interface CrowdinExtension {
+
+  /** Configure the project name on Crowdin */
+  var projectName: String
+
+  /**
+   * Don't delete downloaded and extracted translation archives from build directory.
+   *
+   * Useful for debugging.
+   */
+  var skipCleanup: Boolean
+}

build-logic/automation-plugins/src/main/kotlin/crowdin/CrowdinPlugin.kt

@@ -0,0 +1,127 @@
+/*
+ * Copyright © 2014-2021 The Android Password Store Authors. All Rights Reserved.
+ * SPDX-License-Identifier: GPL-3.0-only
+ */
+
+package crowdin
+
+import de.undercouch.gradle.tasks.download.Download
+import java.io.File
+import javax.xml.parsers.DocumentBuilderFactory
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import org.gradle.api.GradleException
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.tasks.Copy
+import org.gradle.kotlin.dsl.create
+import org.gradle.kotlin.dsl.register
+import org.w3c.dom.Document
+
+private const val EXCEPTION_MESSAGE =
+  """Applying `crowdin-plugin` requires a projectName to be configured via the "crowdin" extension."""
+private const val CROWDIN_BUILD_API_URL =
+  "https://api.crowdin.com/api/project/%s/export?login=%s&account-key=%s"
+
+class CrowdinDownloadPlugin : Plugin<Project> {
+
+  override fun apply(project: Project) {
+    with(project) {
+      val buildDirectory = layout.buildDirectory.asFile.forUseAtConfigurationTime().get()
+      val extension = extensions.create<CrowdinExtension>("crowdin")
+      afterEvaluate {
+        val projectName = extension.projectName
+        if (projectName.isEmpty()) {
+          throw GradleException(EXCEPTION_MESSAGE)
+        }
+        tasks.register("buildOnApi") {
+          doLast {
+            val login = providers.environmentVariable("CROWDIN_LOGIN").forUseAtConfigurationTime()
+            val key =
+              providers.environmentVariable("CROWDIN_PROJECT_KEY").forUseAtConfigurationTime()
+            if (!login.isPresent) {
+              throw GradleException("CROWDIN_LOGIN environment variable must be set")
+            }
+            if (!key.isPresent) {
+              throw GradleException("CROWDIN_PROJECT_KEY environment variable must be set")
+            }
+            val client = OkHttpClient()
+            val url = CROWDIN_BUILD_API_URL.format(projectName, login.get(), key.get())
+            val request = Request.Builder().url(url).get().build()
+            client.newCall(request).execute()
+          }
+        }
+        val downloadCrowdin =
+          tasks.register<Download>("downloadCrowdin") {
+            dependsOn("buildOnApi")
+            src("https://crowdin.com/backend/download/project/$projectName.zip")
+            dest("$buildDirectory/translations.zip")
+            overwrite(true)
+          }
+        val extractCrowdin =
+          tasks.register<Copy>("extractCrowdin") {
+            dependsOn(downloadCrowdin)
+            doFirst { File(buildDir, "translations").deleteRecursively() }
+            from(zipTree("$buildDirectory/translations.zip"))
+            into("$buildDirectory/translations")
+          }
+        val extractStrings =
+          tasks.register<Copy>("extractStrings") {
+            dependsOn(extractCrowdin)
+            from("$buildDirectory/translations/")
+            into("${projectDir}/src/")
+            setFinalizedBy(setOf("removeIncompleteStrings"))
+          }
+        tasks.register("removeIncompleteStrings") {
+          doLast {
+            val sourceSets = arrayOf("main", "nonFree")
+            for (sourceSet in sourceSets) {
+              val stringFiles =
+                File("${projectDir}/src/$sourceSet").walkTopDown().filter {
+                  it.name == "strings.xml"
+                }
+              val sourceFile =
+                stringFiles.firstOrNull { it.path.endsWith("values/strings.xml") }
+                  ?: throw GradleException("No root strings.xml found in '$sourceSet' sourceSet")
+              val sourceDoc = parseDocument(sourceFile)
+              val baselineStringCount = countStrings(sourceDoc)
+              val threshold = 0.80 * baselineStringCount
+              stringFiles.forEach { file ->
+                if (file != sourceFile) {
+                  val doc = parseDocument(file)
+                  val stringCount = countStrings(doc)
+                  if (stringCount < threshold) {
+                    file.delete()
+                  }
+                }
+              }
+            }
+          }
+        }
+        tasks.register("crowdin") {
+          dependsOn(extractStrings)
+          if (!extension.skipCleanup) {
+            doLast {
+              File("$buildDirectory/translations").deleteRecursively()
+              File("$buildDirectory/nonFree-translations").deleteRecursively()
+              File("$buildDirectory/translations.zip").delete()
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private fun parseDocument(file: File): Document {
+    val dbFactory = DocumentBuilderFactory.newInstance()
+    val documentBuilder = dbFactory.newDocumentBuilder()
+    return documentBuilder.parse(file)
+  }
+
+  private fun countStrings(document: Document): Int {
+    // Normalization is beneficial for us
+    // https://stackoverflow.com/questions/13786607/normalization-in-dom-parsing-with-java-how-does-it-work
+    document.documentElement.normalize()
+    return document.getElementsByTagName("string").length
+  }
+}

build-logic/automation-plugins/src/main/kotlin/psl/PublicSuffixListPlugin.kt

@@ -0,0 +1,122 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package psl
+
+import java.io.File
+import java.util.TreeSet
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okio.ByteString
+import okio.ByteString.Companion.encodeUtf8
+import okio.buffer
+import okio.sink
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+
+/**
+ * Gradle plugin to update the public suffix list used by the `lib-publicsuffixlist` component.
+ *
+ * Base on PublicSuffixListGenerator from OkHttp:
+ * https://github.com/square/okhttp/blob/master/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.java
+ */
+class PublicSuffixListPlugin : Plugin<Project> {
+  override fun apply(project: Project) {
+    project.tasks.register("updatePSL") {
+      doLast {
+        val filename = project.projectDir.absolutePath + "/src/main/assets/publicsuffixes"
+        updatePublicSuffixList(filename)
+      }
+    }
+  }
+
+  private fun updatePublicSuffixList(destination: String) {
+    val list = fetchPublicSuffixList()
+    writeListToDisk(destination, list)
+  }
+
+  private fun writeListToDisk(destination: String, data: PublicSuffixListData) {
+    val fileSink = File(destination).sink()
+
+    fileSink.buffer().use { sink ->
+      sink.writeInt(data.totalRuleBytes)
+
+      for (domain in data.sortedRules) {
+        sink.write(domain).writeByte('\n'.toInt())
+      }
+
+      sink.writeInt(data.totalExceptionRuleBytes)
+
+      for (domain in data.sortedExceptionRules) {
+        sink.write(domain).writeByte('\n'.toInt())
+      }
+    }
+  }
+
+  private fun fetchPublicSuffixList(): PublicSuffixListData {
+    val client = OkHttpClient.Builder().build()
+
+    val request =
+      Request.Builder().url("https://publicsuffix.org/list/public_suffix_list.dat").build()
+
+    client.newCall(request).execute().use { response ->
+      val source = requireNotNull(response.body).source()
+
+      val data = PublicSuffixListData()
+
+      while (!source.exhausted()) {
+        val line = source.readUtf8LineStrict()
+
+        if (line.trim { it <= ' ' }.isEmpty() || line.startsWith("//")) {
+          continue
+        }
+
+        if (line.contains(WILDCARD_CHAR)) {
+          assertWildcardRule(line)
+        }
+
+        var rule = line.encodeUtf8()
+
+        if (rule.startsWith(EXCEPTION_RULE_MARKER)) {
+          rule = rule.substring(1)
+          // We use '\n' for end of value.
+          data.totalExceptionRuleBytes += rule.size + 1
+          data.sortedExceptionRules.add(rule)
+        } else {
+          data.totalRuleBytes += rule.size + 1 // We use '\n' for end of value.
+          data.sortedRules.add(rule)
+        }
+      }
+
+      return data
+    }
+  }
+
+  @Suppress("TooGenericExceptionThrown", "ThrowsCount")
+  private fun assertWildcardRule(rule: String) {
+    if (rule.indexOf(WILDCARD_CHAR) != 0) {
+      throw RuntimeException("Wildcard is not not in leftmost position")
+    }
+
+    if (rule.indexOf(WILDCARD_CHAR, 1) != -1) {
+      throw RuntimeException("Rule contains multiple wildcards")
+    }
+
+    if (rule.length == 1) {
+      throw RuntimeException("Rule wildcards the first level")
+    }
+  }
+
+  companion object {
+    private const val WILDCARD_CHAR = "*"
+    private val EXCEPTION_RULE_MARKER = "!".encodeUtf8()
+  }
+}
+
+data class PublicSuffixListData(
+  var totalRuleBytes: Int = 0,
+  var totalExceptionRuleBytes: Int = 0,
+  val sortedRules: TreeSet<ByteString> = TreeSet(),
+  val sortedExceptionRules: TreeSet<ByteString> = TreeSet()
+)