build-logic: add crowdin and psl convention plugins
This commit is contained in:
Harsh Shandilya
parent
dfee170bd8
commit
70cdd61797
4 changed files with 296 additions and 1 deletions
|
|
@ -1 +1,27 @@
|
||||||
plugins { `kotlin-dsl` }
|
/*
|
||||||
|
* Copyright © 2014-2021 The Android Password Store Authors. All Rights Reserved.
|
||||||
|
* SPDX-License-Identifier: GPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
plugins {
|
||||||
|
`kotlin-dsl`
|
||||||
|
`kotlin-dsl-precompiled-script-plugins`
|
||||||
|
}
|
||||||
|
|
||||||
|
gradlePlugin {
|
||||||
|
plugins {
|
||||||
|
register("crowdin") {
|
||||||
|
id = "com.github.android-password-store.crowdin-plugin"
|
||||||
|
implementationClass = "crowdin.CrowdinDownloadPlugin"
|
||||||
|
}
|
||||||
|
register("psl") {
|
||||||
|
id = "com.github.android-password-store.psl-plugin"
|
||||||
|
implementationClass = "psl.PublicSuffixListPlugin"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
implementation(libs.build.download)
|
||||||
|
implementation(libs.build.okhttp)
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
/*
|
||||||
|
* Copyright © 2014-2021 The Android Password Store Authors. All Rights Reserved.
|
||||||
|
* SPDX-License-Identifier: GPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
package crowdin
|
||||||
|
|
||||||
|
/** Extension for configuring [CrowdinPlugin] */
|
||||||
|
interface CrowdinExtension {
|
||||||
|
|
||||||
|
/** Configure the project name on Crowdin */
|
||||||
|
var projectName: String
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Don't delete downloaded and extracted translation archives from build directory.
|
||||||
|
*
|
||||||
|
* Useful for debugging.
|
||||||
|
*/
|
||||||
|
var skipCleanup: Boolean
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,127 @@
|
||||||
|
/*
|
||||||
|
* Copyright © 2014-2021 The Android Password Store Authors. All Rights Reserved.
|
||||||
|
* SPDX-License-Identifier: GPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
package crowdin
|
||||||
|
|
||||||
|
import de.undercouch.gradle.tasks.download.Download
|
||||||
|
import java.io.File
|
||||||
|
import javax.xml.parsers.DocumentBuilderFactory
|
||||||
|
import okhttp3.OkHttpClient
|
||||||
|
import okhttp3.Request
|
||||||
|
import org.gradle.api.GradleException
|
||||||
|
import org.gradle.api.Plugin
|
||||||
|
import org.gradle.api.Project
|
||||||
|
import org.gradle.api.tasks.Copy
|
||||||
|
import org.gradle.kotlin.dsl.create
|
||||||
|
import org.gradle.kotlin.dsl.register
|
||||||
|
import org.w3c.dom.Document
|
||||||
|
|
||||||
|
private const val EXCEPTION_MESSAGE =
|
||||||
|
"""Applying `crowdin-plugin` requires a projectName to be configured via the "crowdin" extension."""
|
||||||
|
private const val CROWDIN_BUILD_API_URL =
|
||||||
|
"https://api.crowdin.com/api/project/%s/export?login=%s&account-key=%s"
|
||||||
|
|
||||||
|
class CrowdinDownloadPlugin : Plugin<Project> {
|
||||||
|
|
||||||
|
override fun apply(project: Project) {
|
||||||
|
with(project) {
|
||||||
|
val buildDirectory = layout.buildDirectory.asFile.forUseAtConfigurationTime().get()
|
||||||
|
val extension = extensions.create<CrowdinExtension>("crowdin")
|
||||||
|
afterEvaluate {
|
||||||
|
val projectName = extension.projectName
|
||||||
|
if (projectName.isEmpty()) {
|
||||||
|
throw GradleException(EXCEPTION_MESSAGE)
|
||||||
|
}
|
||||||
|
tasks.register("buildOnApi") {
|
||||||
|
doLast {
|
||||||
|
val login = providers.environmentVariable("CROWDIN_LOGIN").forUseAtConfigurationTime()
|
||||||
|
val key =
|
||||||
|
providers.environmentVariable("CROWDIN_PROJECT_KEY").forUseAtConfigurationTime()
|
||||||
|
if (!login.isPresent) {
|
||||||
|
throw GradleException("CROWDIN_LOGIN environment variable must be set")
|
||||||
|
}
|
||||||
|
if (!key.isPresent) {
|
||||||
|
throw GradleException("CROWDIN_PROJECT_KEY environment variable must be set")
|
||||||
|
}
|
||||||
|
val client = OkHttpClient()
|
||||||
|
val url = CROWDIN_BUILD_API_URL.format(projectName, login.get(), key.get())
|
||||||
|
val request = Request.Builder().url(url).get().build()
|
||||||
|
client.newCall(request).execute()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val downloadCrowdin =
|
||||||
|
tasks.register<Download>("downloadCrowdin") {
|
||||||
|
dependsOn("buildOnApi")
|
||||||
|
src("https://crowdin.com/backend/download/project/$projectName.zip")
|
||||||
|
dest("$buildDirectory/translations.zip")
|
||||||
|
overwrite(true)
|
||||||
|
}
|
||||||
|
val extractCrowdin =
|
||||||
|
tasks.register<Copy>("extractCrowdin") {
|
||||||
|
dependsOn(downloadCrowdin)
|
||||||
|
doFirst { File(buildDir, "translations").deleteRecursively() }
|
||||||
|
from(zipTree("$buildDirectory/translations.zip"))
|
||||||
|
into("$buildDirectory/translations")
|
||||||
|
}
|
||||||
|
val extractStrings =
|
||||||
|
tasks.register<Copy>("extractStrings") {
|
||||||
|
dependsOn(extractCrowdin)
|
||||||
|
from("$buildDirectory/translations/")
|
||||||
|
into("${projectDir}/src/")
|
||||||
|
setFinalizedBy(setOf("removeIncompleteStrings"))
|
||||||
|
}
|
||||||
|
tasks.register("removeIncompleteStrings") {
|
||||||
|
doLast {
|
||||||
|
val sourceSets = arrayOf("main", "nonFree")
|
||||||
|
for (sourceSet in sourceSets) {
|
||||||
|
val stringFiles =
|
||||||
|
File("${projectDir}/src/$sourceSet").walkTopDown().filter {
|
||||||
|
it.name == "strings.xml"
|
||||||
|
}
|
||||||
|
val sourceFile =
|
||||||
|
stringFiles.firstOrNull { it.path.endsWith("values/strings.xml") }
|
||||||
|
?: throw GradleException("No root strings.xml found in '$sourceSet' sourceSet")
|
||||||
|
val sourceDoc = parseDocument(sourceFile)
|
||||||
|
val baselineStringCount = countStrings(sourceDoc)
|
||||||
|
val threshold = 0.80 * baselineStringCount
|
||||||
|
stringFiles.forEach { file ->
|
||||||
|
if (file != sourceFile) {
|
||||||
|
val doc = parseDocument(file)
|
||||||
|
val stringCount = countStrings(doc)
|
||||||
|
if (stringCount < threshold) {
|
||||||
|
file.delete()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tasks.register("crowdin") {
|
||||||
|
dependsOn(extractStrings)
|
||||||
|
if (!extension.skipCleanup) {
|
||||||
|
doLast {
|
||||||
|
File("$buildDirectory/translations").deleteRecursively()
|
||||||
|
File("$buildDirectory/nonFree-translations").deleteRecursively()
|
||||||
|
File("$buildDirectory/translations.zip").delete()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseDocument(file: File): Document {
|
||||||
|
val dbFactory = DocumentBuilderFactory.newInstance()
|
||||||
|
val documentBuilder = dbFactory.newDocumentBuilder()
|
||||||
|
return documentBuilder.parse(file)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun countStrings(document: Document): Int {
|
||||||
|
// Normalization is beneficial for us
|
||||||
|
// https://stackoverflow.com/questions/13786607/normalization-in-dom-parsing-with-java-how-does-it-work
|
||||||
|
document.documentElement.normalize()
|
||||||
|
return document.getElementsByTagName("string").length
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,122 @@
|
||||||
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||||
|
|
||||||
|
package psl
|
||||||
|
|
||||||
|
import java.io.File
|
||||||
|
import java.util.TreeSet
|
||||||
|
import okhttp3.OkHttpClient
|
||||||
|
import okhttp3.Request
|
||||||
|
import okio.ByteString
|
||||||
|
import okio.ByteString.Companion.encodeUtf8
|
||||||
|
import okio.buffer
|
||||||
|
import okio.sink
|
||||||
|
import org.gradle.api.Plugin
|
||||||
|
import org.gradle.api.Project
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gradle plugin to update the public suffix list used by the `lib-publicsuffixlist` component.
|
||||||
|
*
|
||||||
|
* Base on PublicSuffixListGenerator from OkHttp:
|
||||||
|
* https://github.com/square/okhttp/blob/master/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.java
|
||||||
|
*/
|
||||||
|
class PublicSuffixListPlugin : Plugin<Project> {
|
||||||
|
override fun apply(project: Project) {
|
||||||
|
project.tasks.register("updatePSL") {
|
||||||
|
doLast {
|
||||||
|
val filename = project.projectDir.absolutePath + "/src/main/assets/publicsuffixes"
|
||||||
|
updatePublicSuffixList(filename)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun updatePublicSuffixList(destination: String) {
|
||||||
|
val list = fetchPublicSuffixList()
|
||||||
|
writeListToDisk(destination, list)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun writeListToDisk(destination: String, data: PublicSuffixListData) {
|
||||||
|
val fileSink = File(destination).sink()
|
||||||
|
|
||||||
|
fileSink.buffer().use { sink ->
|
||||||
|
sink.writeInt(data.totalRuleBytes)
|
||||||
|
|
||||||
|
for (domain in data.sortedRules) {
|
||||||
|
sink.write(domain).writeByte('\n'.toInt())
|
||||||
|
}
|
||||||
|
|
||||||
|
sink.writeInt(data.totalExceptionRuleBytes)
|
||||||
|
|
||||||
|
for (domain in data.sortedExceptionRules) {
|
||||||
|
sink.write(domain).writeByte('\n'.toInt())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun fetchPublicSuffixList(): PublicSuffixListData {
|
||||||
|
val client = OkHttpClient.Builder().build()
|
||||||
|
|
||||||
|
val request =
|
||||||
|
Request.Builder().url("https://publicsuffix.org/list/public_suffix_list.dat").build()
|
||||||
|
|
||||||
|
client.newCall(request).execute().use { response ->
|
||||||
|
val source = requireNotNull(response.body).source()
|
||||||
|
|
||||||
|
val data = PublicSuffixListData()
|
||||||
|
|
||||||
|
while (!source.exhausted()) {
|
||||||
|
val line = source.readUtf8LineStrict()
|
||||||
|
|
||||||
|
if (line.trim { it <= ' ' }.isEmpty() || line.startsWith("//")) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if (line.contains(WILDCARD_CHAR)) {
|
||||||
|
assertWildcardRule(line)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rule = line.encodeUtf8()
|
||||||
|
|
||||||
|
if (rule.startsWith(EXCEPTION_RULE_MARKER)) {
|
||||||
|
rule = rule.substring(1)
|
||||||
|
// We use '\n' for end of value.
|
||||||
|
data.totalExceptionRuleBytes += rule.size + 1
|
||||||
|
data.sortedExceptionRules.add(rule)
|
||||||
|
} else {
|
||||||
|
data.totalRuleBytes += rule.size + 1 // We use '\n' for end of value.
|
||||||
|
data.sortedRules.add(rule)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return data
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Suppress("TooGenericExceptionThrown", "ThrowsCount")
|
||||||
|
private fun assertWildcardRule(rule: String) {
|
||||||
|
if (rule.indexOf(WILDCARD_CHAR) != 0) {
|
||||||
|
throw RuntimeException("Wildcard is not not in leftmost position")
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rule.indexOf(WILDCARD_CHAR, 1) != -1) {
|
||||||
|
throw RuntimeException("Rule contains multiple wildcards")
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rule.length == 1) {
|
||||||
|
throw RuntimeException("Rule wildcards the first level")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private const val WILDCARD_CHAR = "*"
|
||||||
|
private val EXCEPTION_RULE_MARKER = "!".encodeUtf8()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data class PublicSuffixListData(
|
||||||
|
var totalRuleBytes: Int = 0,
|
||||||
|
var totalExceptionRuleBytes: Int = 0,
|
||||||
|
val sortedRules: TreeSet<ByteString> = TreeSet(),
|
||||||
|
val sortedExceptionRules: TreeSet<ByteString> = TreeSet()
|
||||||
|
)
|
||||||
Loading…
Add table
Add a link
Reference in a new issue