Fetch translations from Smartling

Smartling exposes a simple REST API that’s fairly easy to adopt.

Some differences from the old approach:

- We get UTF-8 back instead of UTF-16, so there’s no need to use iconv.

- We don’t support “nb_NO”, so we don’t need to remove it each time we
  fetch translations.

- We get back English fallbacks for .stringsdict files, so there’s no
  need to merge them manually ourselves.

- We no longer support country-specific locales *and* the root language,
  so we don’t need to merge, for example, “es” into “es-MX”.

- We handle language mapping & duplication inside the Swift script,
  which will hopefully be more reliable than cp’ing directories.

Scripts/precommit.py

@@ -200,6 +200,9 @@ def process(filepath):
     if lines[0].startswith("#!"):
         shebang = lines[0] + "\n"
         lines = lines[1:]
+    elif lines[0].startswith("// swift-tools-version:"):
+        shebang = lines[0] + "\n"
+        lines = lines[1:]
 
     while lines and lines[0].startswith("//"):
         lines = lines[1:]

Scripts/translation-tool/.gitignore

@@ -0,0 +1,2 @@
+.build/
+.swiftpm/

Scripts/translation-tool/Package.swift

@@ -0,0 +1,19 @@
+// swift-tools-version: 5.6
+//
+//  Copyright (c) 2022 Open Whisper Systems. All rights reserved.
+//
+
+import PackageDescription
+
+let package = Package(
+    name: "translation-tool",
+    platforms: [.macOS(.v12)],
+    dependencies: [],
+    targets: [
+        .executableTarget(
+            name: "translation-tool",
+            dependencies: [],
+            path: "src"
+        )
+    ]
+)

Scripts/translation-tool/src/CLI.swift

@@ -0,0 +1,108 @@
+//
+//  Copyright (c) 2022 Open Whisper Systems. All rights reserved.
+//
+
+import Foundation
+
+enum Constant {
+    static let concurrentRequestLimit = 12
+    static let projectIdentifier = "4b899d72e"
+}
+
+@main
+struct CLI {
+
+    // MARK: - Entrypoint
+
+    static func main() async throws {
+        guard let (userIdentifier, userSecret) = try loadUserParameters() else {
+            showIntructionsForUserParameters()
+        }
+        let repositoryURL = URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
+        let client = Smartling(
+            projectIdentifier: Constant.projectIdentifier,
+            userIdentifier: userIdentifier,
+            userSecret: userSecret
+        )
+        let cli = CLI(repositoryURL: repositoryURL, client: client)
+        for arg in CommandLine.arguments.dropFirst() {
+            switch arg {
+            case "upload-metadata":
+                try await cli.uploadFiles(metadataFiles)
+            case "upload-resources":
+                try await cli.uploadFiles(resourceFiles)
+            case "download-metadata":
+                try MetadataFile.checkForUnusedLocalizations(in: repositoryURL)
+                try await cli.downloadFiles(metadataFiles)
+            case "download-resources":
+                try ResourceFile.checkForUnusedLocalizations(in: repositoryURL)
+                try await cli.downloadFiles(resourceFiles)
+            default:
+                print("Unknown action: \(arg)")
+            }
+        }
+    }
+
+    // MARK: - Upload & Download
+
+    private static let metadataFiles: [MetadataFile] = [
+        MetadataFile(filename: "release_notes.txt"),
+        MetadataFile(filename: "description.txt")
+    ]
+
+    private static let resourceFiles: [ResourceFile] = [
+        ResourceFile(filename: "InfoPlist.strings"),
+        ResourceFile(filename: "Localizable.strings"),
+        ResourceFile(filename: "PluralAware.stringsdict")
+    ]
+
+    var repositoryURL: URL
+    var client: Smartling
+
+    private func uploadFiles(_ files: [TranslatableFile]) async throws {
+        try await withLimitedThrowingTaskGroup(limit: Constant.concurrentRequestLimit) { taskGroup in
+            for translatableFile in files {
+                try await taskGroup.addTask {
+                    try await client.uploadSourceFile(
+                        at: repositoryURL.appendingPathComponent(translatableFile.relativeSourcePath)
+                    )
+                    print("Uploaded \(translatableFile.filename)")
+                }
+            }
+        }
+    }
+
+    private func downloadFiles(_ files: [TranslatableFile]) async throws {
+        // Each of these kicks off a bunch of downloads in parallel, so it's fine to do these serially.
+        for translatableFile in files {
+            try await translatableFile.downloadAllTranslations(to: repositoryURL, using: client)
+        }
+    }
+
+    // MARK: - Config
+
+    private static func loadUserParameters() throws -> (String, String)? {
+        let fileURL = FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".smartling/auth")
+        let fileContent: String
+        do {
+            fileContent = try String(contentsOf: fileURL).trimmingCharacters(in: .whitespacesAndNewlines)
+        } catch CocoaError.fileReadNoSuchFile {
+            return nil
+        }
+        let components = fileContent.split(separator: "\n")
+        guard components.count == 2 else {
+            return nil
+        }
+        return (String(components[0]), String(components[1]))
+    }
+
+    private static func showIntructionsForUserParameters() -> Never {
+        print("")
+        print("Couldn't load user identifier/user secret from ~/.smartling/auth.")
+        print("")
+        print("That file should contain two lines: (1) user identifier & (2) user secret.")
+        print("You can create a token using Smartling's web interface.")
+        print("")
+        exit(1)
+    }
+}

Scripts/translation-tool/src/FileManager.swift

@@ -0,0 +1,18 @@
+//
+//  Copyright (c) 2022 Open Whisper Systems. All rights reserved.
+//
+
+import Foundation
+
+extension FileManager {
+    /// Copies an item, replacing the destination if it already exists.
+    func copyItem(at srcURL: URL, replacingItemAt dstURL: URL) throws {
+        do {
+            try removeItem(at: dstURL)
+        } catch CocoaError.fileNoSuchFile {
+            // not an error if the file doesn't exist
+        }
+        try createDirectory(at: dstURL.deletingLastPathComponent(), withIntermediateDirectories: true)
+        try copyItem(at: srcURL, to: dstURL)
+    }
+}

Scripts/translation-tool/src/LimitedThrowingTaskGroup.swift

@@ -0,0 +1,29 @@
+//
+//  Copyright (c) 2022 Open Whisper Systems. All rights reserved.
+//
+
+import Foundation
+
+struct LimitedThrowingTaskGroup {
+    var taskGroup: ThrowingTaskGroup<Void, Error>
+    var remainingCapacity: Int
+
+    mutating func addTask(operation: @escaping @Sendable () async throws -> Void) async throws {
+        if remainingCapacity > 0 {
+            remainingCapacity -= 1
+        } else {
+            // Once we've kicked off the maximum number of concurrent tasks, we always
+            // wait for one to finish before starting the next one.
+            try await taskGroup.next()
+        }
+        taskGroup.addTask(operation: operation)
+    }
+}
+
+func withLimitedThrowingTaskGroup(limit: Int, body: (inout LimitedThrowingTaskGroup) async throws -> Void) async rethrows {
+    try await withThrowingTaskGroup(of: Void.self) { taskGroup in
+        var limitedTaskGroup = LimitedThrowingTaskGroup(taskGroup: taskGroup, remainingCapacity: limit)
+        try await body(&limitedTaskGroup)
+        try await taskGroup.waitForAll()
+    }
+}

Scripts/translation-tool/src/MetadataFile.swift

@@ -0,0 +1,104 @@
+//
+//  Copyright (c) 2022 Open Whisper Systems. All rights reserved.
+//
+
+import Foundation
+
+// Maps Smartling language codes to App Store Connect language codes.
+private let languageMap: [String: [String]] = [
+    // These languages are returned from Smartling and need to be moved to their correct final destination.
+    "ar": ["ar-SA"],
+    "ca": ["ca"],
+    "cs": ["cs"],
+    "da": ["da"],
+    "de": ["de-DE"],
+    "el": ["el"],
+    "es": ["es-ES", "es-MX"],
+    "fi": ["fi"],
+    "fr": ["fr-CA", "fr-FR"],
+    "he": ["he"],
+    "hi-IN": ["hi"],
+    "hr-HR": ["hr"],
+    "hu": ["hu"],
+    "id": ["id"],
+    "it": ["it"],
+    "ja": ["ja"],
+    "ko": ["ko"],
+    "ms": ["ms"],
+    "nb": ["no"],
+    "nl": ["nl-NL"],
+    "pl": ["pl"],
+    "pt-BR": ["pt-BR"],
+    "pt-PT": ["pt-PT"],
+    "ro-RO": ["ro"],
+    "ru": ["ru"],
+    "sk-SK": ["sk"],
+    "sv": ["sv"],
+    "th": ["th"],
+    "tr": ["tr"],
+    "uk-UA": ["uk"],
+    "vi": ["vi"],
+    "zh-CN": ["zh-Hans"],
+    "zh-HK": ["zh-Hant"]
+
+    // These don't exist in App Store Connect, so there's no need to fetch them from Smartling.
+    // "bn-BD": [],
+    // "fa-IR": [],
+    // "ga-IE": [],
+    // "gu-IN": [],
+    // "mr-IN": [],
+    // "sr-RS": [],
+    // "sr-YR": [],
+    // "ur": [],
+    // "zh-TW": [],
+]
+
+private let extraEnglishLanguages: [String] = ["en-AU", "en-CA", "en-GB"]
+
+struct MetadataFile: TranslatableFile {
+    var filename: String
+
+    var relativeSourcePath: String { relativePath(for: "en-US") }
+
+    private static let relativeDirectoryPath = "fastlane/metadata"
+
+    private func relativePath(for languageCode: String) -> String {
+        return "\(Self.relativeDirectoryPath)/\(languageCode)/\(filename)"
+    }
+
+    func downloadAllTranslations(to repositoryURL: URL, using client: Smartling) async throws {
+        try await withLimitedThrowingTaskGroup(limit: Constant.concurrentRequestLimit) { taskGroup in
+            try await taskGroup.addTask {
+                // English is special. Instead of downloading a file, we copy the file we
+                // uploaded to other English languages.
+                let fileURL = repositoryURL.appendingPathComponent(relativeSourcePath)
+                try processDownloadedFile(at: fileURL, repositoryURL: repositoryURL, localIdentifiers: extraEnglishLanguages)
+            }
+            for (remoteIdentifier, localIdentifiers) in languageMap {
+                try await taskGroup.addTask {
+                    let fileURL = try await client.downloadTranslatedFile(for: filename, in: remoteIdentifier)
+                    try processDownloadedFile(at: fileURL, repositoryURL: repositoryURL, localIdentifiers: localIdentifiers)
+                }
+            }
+        }
+    }
+
+    private func processDownloadedFile(at fileURL: URL, repositoryURL: URL, localIdentifiers: [String]) throws {
+        for localIdentifier in localIdentifiers {
+            let localRelativePath = relativePath(for: localIdentifier)
+            try FileManager.default.copyItem(
+                at: fileURL,
+                replacingItemAt: repositoryURL.appendingPathComponent(localRelativePath)
+            )
+            print("Saved \(localRelativePath)")
+        }
+    }
+
+    static func checkForUnusedLocalizations(in repositoryURL: URL) throws {
+        try checkForUnusedLocalizations(
+            in: repositoryURL.appendingPathComponent(Self.relativeDirectoryPath),
+            suffix: "",
+            expectedLocalizations: languageMap.flatMap { $1 } + extraEnglishLanguages + ["en-US"]
+        )
+    }
+}

Scripts/translation-tool/src/ResourceFile.swift

@@ -0,0 +1,86 @@
+//
+//  Copyright (c) 2022 Open Whisper Systems. All rights reserved.
+//
+
+import Foundation
+
+// Maps Smartling language codes to .strings/.stringsdict language codes.
+private let languageMap: [String: String] = [
+    "ar": "ar",
+    "bn-BD": "bn",
+    "ca": "ca",
+    "cs": "cs",
+    "da": "da",
+    "de": "de",
+    "el": "el",
+    "es": "es",
+    "fa-IR": "fa",
+    "fi": "fi",
+    "fr": "fr",
+    "ga-IE": "ga",
+    "gu-IN": "gu",
+    "he": "he",
+    "hi-IN": "hi",
+    "hr-HR": "hr",
+    "hu": "hu",
+    "id": "id",
+    "it": "it",
+    "ja": "ja",
+    "ko": "ko",
+    "mr-IN": "mr",
+    "ms": "ms",
+    "nb": "nb",
+    "nl": "nl",
+    "pl": "pl",
+    "pt-BR": "pt_BR",
+    "pt-PT": "pt_PT",
+    "ro-RO": "ro",
+    "ru": "ru",
+    "sk-SK": "sk",
+    "sr-YR": "sr",
+    "sv": "sv",
+    "th": "th",
+    "tr": "tr",
+    "uk-UA": "uk",
+    "ur": "ur",
+    "vi": "vi",
+    "zh-CN": "zh_CN",
+    "zh-HK": "zh_HK",
+    "zh-TW": "zh_TW"
+]
+
+struct ResourceFile: TranslatableFile {
+    var filename: String
+
+    var relativeSourcePath: String {
+        "\(Self.relativeDirectoryPath)/en.lproj/\(filename)"
+    }
+
+    private static let relativeDirectoryPath = "Signal/translations"
+
+    private func relativePath(for languageCode: String) -> String {
+        return "\(Self.relativeDirectoryPath)/\(languageCode).lproj/\(filename)"
+    }
+
+    func downloadAllTranslations(to repositoryURL: URL, using client: Smartling) async throws {
+        try await withLimitedThrowingTaskGroup(limit: Constant.concurrentRequestLimit) { taskGroup in
+            for (remoteIdentifier, localIdentifier) in languageMap {
+                let fileURL = try await client.downloadTranslatedFile(for: filename, in: remoteIdentifier)
+                let localRelativePath = relativePath(for: localIdentifier)
+                try FileManager.default.copyItem(
+                    at: fileURL,
+                    replacingItemAt: repositoryURL.appendingPathComponent(localRelativePath)
+                )
+                print("Saved \(localRelativePath)")
+            }
+        }
+    }
+
+    static func checkForUnusedLocalizations(in repositoryURL: URL) throws {
+        try checkForUnusedLocalizations(
+            in: repositoryURL.appendingPathComponent(relativeDirectoryPath),
+            suffix: ".lproj",
+            expectedLocalizations: languageMap.map { $1 } + ["en"]
+        )
+    }
+}

Scripts/translation-tool/src/Smartling.swift

@@ -0,0 +1,199 @@
+//
+//  Copyright (c) 2022 Open Whisper Systems. All rights reserved.
+//
+
+import Foundation
+
+class Smartling {
+    let projectIdentifier: String
+    let userIdentifier: String
+    let userSecret: String
+
+    init(projectIdentifier: String, userIdentifier: String, userSecret: String) {
+        self.projectIdentifier = projectIdentifier
+        self.userIdentifier = userIdentifier
+        self.userSecret = userSecret
+    }
+
+    fileprivate struct Token {
+        var accessToken: String
+        var expirationDate: Date
+    }
+
+    private var latestTokenTask: Task<Token, Error>?
+
+    @MainActor
+    private func fetchToken() async throws -> Token {
+        assert(Thread.isMainThread)
+        if let latestToken = try await latestTokenTask?.value, latestToken.expirationDate.timeIntervalSinceNow > 5 {
+            return latestToken
+        }
+        let task = Task {
+            let rawToken = try await fetchNewToken()
+            let newToken = Token(
+                accessToken: rawToken.accessToken,
+                expirationDate: Date(timeIntervalSinceNow: TimeInterval(rawToken.expiresIn))
+            )
+            print("Got new token that expires at \(newToken.expirationDate)")
+            return newToken
+        }
+        latestTokenTask = task
+        return try await task.value
+    }
+
+    struct FetchedToken: Decodable {
+        var accessToken: String
+        var expiresIn: Int
+    }
+
+    private func fetchNewToken() async throws -> FetchedToken {
+        struct AuthenticationRequest: Encodable {
+            var userIdentifier: String
+            var userSecret: String
+        }
+
+        let request = AuthenticationRequest(userIdentifier: userIdentifier, userSecret: userSecret)
+        return try await postRequest(urlPath: "/auth-api/v2/authenticate", request: request)
+    }
+
+    func uploadSourceFile(at fileURL: URL) async throws {
+        let urlPath = "/files-api/v2/projects/\(projectIdentifier)/file"
+        var urlRequest = buildRequest(url: buildURL(path: urlPath), token: try await fetchToken())
+        urlRequest.httpMethod = "POST"
+        try urlRequest.addFile(at: fileURL)
+        _ = try await URLSession.shared.data(for: urlRequest, expecting: 200)
+    }
+
+    func downloadTranslatedFile(for filename: String, in localeIdentifier: String) async throws -> URL {
+        let urlPath = "/files-api/v2/projects/\(projectIdentifier)/locales/\(localeIdentifier)/file"
+        let url = buildURL(path: urlPath, queryItems: [
+            "fileUri": filename,
+            "retrievalType": "published",
+            "includeOriginalStrings": "true"
+        ])
+        var urlRequest = buildRequest(url: url, token: try await fetchToken())
+        urlRequest.httpMethod = "GET"
+        return try await URLSession.shared.download(for: urlRequest, expecting: 200)
+    }
+}
+
+private extension Smartling {
+    func buildURL(path: String, queryItems: [String: String]? = nil) -> URL {
+        var urlComponents = URLComponents()
+        urlComponents.scheme = "https"
+        urlComponents.host = "api.smartling.com"
+        urlComponents.path = path
+        urlComponents.queryItems = queryItems?.map { URLQueryItem(name: $0.key, value: $0.value) }
+        return urlComponents.url!
+    }
+
+    func buildRequest(url: URL, token: Token? = nil) -> URLRequest {
+        var request = URLRequest(url: url)
+        if let token = token {
+            request.addValue("Bearer \(token.accessToken)", forHTTPHeaderField: "Authorization")
+        }
+        return request
+    }
+
+    // Wraps a JSON response from Smartling.
+    struct Response<T: Decodable>: Decodable {
+        var response: WrappedResponse
+        struct WrappedResponse: Decodable {
+            var data: T
+        }
+    }
+
+    func postRequest<Req: Encodable, Resp: Decodable>(urlPath: String, request: Req) async throws -> Resp {
+        var urlRequest = buildRequest(url: buildURL(path: urlPath))
+        urlRequest.httpMethod = "POST"
+        urlRequest.httpBody = try JSONEncoder().encode(request)
+        urlRequest.setValue("application/json", forHTTPHeaderField: "Content-Type")
+        let responseData = try await URLSession.shared.data(for: urlRequest, expecting: 200)
+        let wrappedResponse = try JSONDecoder().decode(Response<Resp>.self, from: responseData)
+        return wrappedResponse.response.data
+    }
+}
+
+extension URLRequest {
+    mutating func addFile(at fileURL: URL) throws {
+        let boundary = UUID().uuidString
+        setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type")
+        let filename = fileURL.lastPathComponent
+        httpBody = Self.multipartFormData(boundary: boundary, items: [
+            ("file", .file(name: filename, value: try Data(contentsOf: fileURL))),
+            ("fileUri", .text(value: filename)),
+            ("fileType", .text(value: Self.fileType(for: fileURL)))
+        ])
+    }
+
+    private enum MultipartFormDataItem {
+        case text(value: String)
+        case file(name: String, value: Data)
+    }
+
+    private static func multipartFormData(boundary: String, items: [(String, MultipartFormDataItem)]) -> Data {
+        var result = Data()
+        func addLine(_ dataValue: Data) {
+            result.append(dataValue)
+            result.append("\r\n".data(using: .utf8)!)
+        }
+        func addLine(_ stringValue: String) {
+            addLine(stringValue.data(using: .utf8)!)
+        }
+        for (fieldName, item) in items {
+            addLine("--\(boundary)")
+            switch item {
+            case let .text(value):
+                addLine("Content-Disposition: form-data; name=\"\(fieldName)\"")
+                addLine("")
+                addLine(value)
+            case let .file(name, value):
+                addLine("Content-Disposition: form-data; name=\"\(fieldName)\"; filename=\"\(name)\"")
+                addLine("Content-Type: application/octet-stream")
+                addLine("")
+                addLine(value)
+            }
+        }
+        addLine("--\(boundary)--")
+        return result
+    }
+
+    private static func fileType(for url: URL) -> String {
+        let pathExtension = url.pathExtension
+        switch pathExtension {
+        case "txt":
+            return "plain_text"
+        case "strings":
+            return "ios"
+        case "stringsdict":
+            return "stringsdict"
+        default:
+            fatalError("Can't upload file with .\(pathExtension) extension")
+        }
+    }
+}
+
+private extension URLSession {
+    enum HTTPError: Error {
+        case statusCode(Int?)
+    }
+
+    func data(for urlRequest: URLRequest, expecting expectedStatusCode: Int) async throws -> Data {
+        let (data, urlResponse) = try await data(for: urlRequest)
+        try handleResponse(urlResponse: urlResponse, expecting: expectedStatusCode)
+        return data
+    }
+
+    func download(for urlRequest: URLRequest, expecting expectedStatusCode: Int) async throws -> URL {
+        let (data, urlResponse) = try await download(for: urlRequest)
+        try handleResponse(urlResponse: urlResponse, expecting: expectedStatusCode)
+        return data
+    }
+
+    private func handleResponse(urlResponse: URLResponse, expecting expectedStatusCode: Int) throws {
+        let statusCode = (urlResponse as? HTTPURLResponse)?.statusCode
+        guard statusCode == expectedStatusCode else {
+            throw HTTPError.statusCode(statusCode)
+        }
+    }
+}

Scripts/translation-tool/src/TranslatableFile.swift

@@ -0,0 +1,49 @@
+//
+//  Copyright (c) 2022 Open Whisper Systems. All rights reserved.
+//
+
+import Foundation
+
+protocol TranslatableFile {
+    var filename: String { get }
+    var relativeSourcePath: String { get }
+
+    /// Checks if there are localizations without a remote counterpart.
+    ///
+    /// This is most useful when adding or removing languages.
+    static func checkForUnusedLocalizations(in repositoryURL: URL) throws
+
+    /// Downloads translations for all languages.
+    func downloadAllTranslations(to repositoryURL: URL, using client: Smartling) async throws
+}
+
+extension TranslatableFile {
+
+    /// Compares localization directories on disk against what's expected.
+    ///
+    /// If there are localization directories on disk that aren't being updated
+    /// by this script, that likely means we're shipping stale translations. We
+    /// check this each time we download translations.
+    static func checkForUnusedLocalizations(in directoryURL: URL, suffix: String, expectedLocalizations: [String]) throws {
+        let localLocalizationCodes = try fetchLocalLocalizationCodes(in: directoryURL, suffix: suffix)
+        let unusedLocalizationCodes = localLocalizationCodes.subtracting(expectedLocalizations)
+
+        guard unusedLocalizationCodes.isEmpty else {
+            let sortedLocalizationCodes = unusedLocalizationCodes.sorted(by: <)
+            print("We're shipping languages that aren't pulling translations: \(sortedLocalizationCodes)")
+            print("(stored in \(directoryURL.path))")
+            exit(1)
+        }
+    }
+
+    private static func fetchLocalLocalizationCodes(in directoryURL: URL, suffix: String) throws -> Set<String> {
+        var result = Set<String>()
+        for filename in try FileManager.default.contentsOfDirectory(atPath: directoryURL.path) {
+            guard filename.hasSuffix(suffix) else {
+                continue
+            }
+            result.insert(String(filename.dropLast(suffix.count)))
+        }
+        return result
+    }
+}

Scripts/translation/MergeStringsDictFiles.swift

@@ -1,116 +0,0 @@
-//
-//  Copyright (c) 2022 Open Whisper Systems. All rights reserved.
-//
-
-import Foundation
-
-func readPlistFile(path: String) -> [String: Any]? {
-    do {
-        let data = try Data(contentsOf: URL(fileURLWithPath: path))
-        return try PropertyListSerialization.propertyList(from: data, options: [], format: nil) as? [String: Any]
-    } catch {
-    }
-    return nil
-}
-
-func writePlistFile(contents: [String: Any], path: String) -> Bool {
-    do {
-        let data = try PropertyListSerialization.data(fromPropertyList: contents, format: .xml, options: 0)
-        try? data.write(to: URL(fileURLWithPath: path))
-        return true
-    } catch {
-    }
-    return false
-}
-
-func hasInvalidPlaceholder(string template: String, placeholder: String, fallback: String?) -> Bool {
-    var template = template[...]
-    var placeholderAlreadyReferenced = false
-    while let index = template.firstIndex(of: "%") {
-        template = template[index...].dropFirst()
-        // check whether the placeholder is referenced more than once (which isn't allowed)
-        if template.starts(with: placeholder) {
-            if placeholderAlreadyReferenced {
-                return true
-            }
-            placeholderAlreadyReferenced = true
-        } else if let item = template.components(separatedBy: "$").first {
-            // we never have more than three arguments passed to NSLocalizedString
-            if !["2", "3"].contains(item) {
-                return true
-            }
-            // check whether this placeholder is contained in the fallback (if provided)
-            else if let fallback = fallback, !fallback.contains("%\(item)$") {
-                return true
-            }
-        }
-    }
-    return false
-}
-
-func hasInvalidPlaceholder(contents: [String: Any]?, fallback: [String: Any]?) -> Bool {
-    guard let contents = contents else {
-        return true
-    }
-    var hasError = false
-    for key in contents.keys {
-        let fallback = fallback?[key] as? [String: Any]
-        if let entry = contents[key] as? [String: Any], let placeholder = entry["NSStringFormatValueTypeKey"] as? String,
-          "NSStringPluralRuleType" == entry["NSStringFormatSpecTypeKey"] as? String {
-            for variant in ["zero", "one", "two", "few", "many", "other"] {
-                if let template = entry[variant] as? String {
-                    if hasInvalidPlaceholder(string: template, placeholder: placeholder, fallback: fallback?[variant] as? String) {
-                        print("*** template for variant \(variant) with placeholder \(placeholder) is invalid: \(template)")
-                        hasError = true
-                    }
-                }
-            }
-        }
-    }
-    return hasError
-}
-
-if CommandLine.arguments.count != 3 {
-    print("usage MergeStringsDictFiles <defaultLanguageFile> <targetLanguageFile>")
-} else {
-    let sourcePath = CommandLine.arguments[1]
-    let destinationPath = CommandLine.arguments[2]
-    if let destinationDict = readPlistFile(path: destinationPath) {
-        var destinationDict = destinationDict
-        if let sourceDict = readPlistFile(path: sourcePath) {
-            var changed = false
-            for key in sourceDict.keys {
-                // if the entry is completely missing take it from the source
-                if !destinationDict.keys.contains(key) {
-                    destinationDict[key] = sourceDict[key]
-                    changed = true
-                    print("added \(key)")
-                }
-                // if the entry only contains the format key replace it, too
-                else if let entries = destinationDict[key] as? [String: Any], entries.keys.count < 2 {
-                    destinationDict[key] = sourceDict[key]
-                    changed = true
-                    print("updated \(key)")
-                }
-                // if the entry contains invalid usage of placeholders replace it, too 
-                else if hasInvalidPlaceholder(contents: destinationDict[key] as? [String: Any],
-                  fallback: sourceDict[key] as? [String: Any]) {
-                    destinationDict[key] = sourceDict[key]
-                    changed = true
-                    print("replacing \(key) due to invalid format usage")
-                }
-            }
-            if changed {
-                if writePlistFile(contents: destinationDict, path: destinationPath) {
-                    print("\(destinationPath) updated")
-                } else {
-                    print("error updating \(destinationPath)")
-                }
-            }
-        } else {
-            print("skipped \(destinationPath), because source file \(sourcePath) does not exist or doesn't contain valid entries")
-        }
-    } else {
-        print("skipped \(destinationPath), does not exist or doesn't contain valid entries")
-    }
-}

Scripts/translation/pull-metadata-translations

@@ -3,19 +3,10 @@
 set -x
 set -e
 
-# https://docs.fastlane.tools/actions/deliver/#available-language-codes
-APP_STORE_LANGUAGES='ar,ca,cs,da,de,el,en_AU,en_CA,en_GB,en_US,es,fi,fr,he,hi,hr,hu,id,it,ja,ko,ms,nl,nb,pl,pt_BR,pt_PT,ro,ru,sk,sv,th,tr,uk,vi,zh_CN,zh_TW'
+BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+REPO_ROOT=$BIN_DIR/../..
+cd $REPO_ROOT
 
 echo "Pulling metadata translations..."
-
-BASE_DIR=$(git rev-parse --show-toplevel)
-METADATA_ROOT="$BASE_DIR/fastlane/metadata"
-
-cd $METADATA_ROOT
-
-# Legacy hack: pull *any existing* app store descriptios regardless of their completion.
-# Once supported, we don't want to drop any translations.
-tx pull --force -l $APP_STORE_LANGUAGES
-
-cp -R fr-FR/ fr-CA
-cp -R es-ES/ es-MX
+swift run --package-path Scripts/translation-tool translation-tool download-metadata

Scripts/translation/pull-translations

@@ -6,66 +6,12 @@ set -e
 BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 REPO_ROOT=$BIN_DIR/../..
+cd $REPO_ROOT
+swift run --package-path Scripts/translation-tool translation-tool download-resources
+
 LOCALIZATION_ROOT=$REPO_ROOT/Signal/translations
-
 cd $LOCALIZATION_ROOT
 
-# Pull all translations which are at least 80% complete
-tx pull -a --minimum-perc=80
-
-# Legacy hack: pull *any existing* translations regardless of their completion.
-# Once supported, we don't want to drop any translations.
-tx pull --force
-
-# There's some problem with nb_NO, iconv fails to parse it.
-# I'm guessing it some kind of unicode encoding issue.
-rm -rf nb_NO.lproj
-
-for dir in *.lproj
-do
-
-  # en.lproj is already utf-8
-  if [[ "$dir" = "en.lproj" ]]; then
-    continue
-  fi
-
-  pushd $dir
-
-  translations=$(ls *.strings)
-  for translation in $translations; do
-    # Transifex pulls utf-16, but our string linting script needs utf-8.
-    # Plus we can see the string diffs in GH this way.
-    echo "Converting $dir/$translation to UTF-8"
-    iconv -f UTF-16 -t UTF-8 $translation > $translation.utf8
-    mv $translation.utf8 $translation
-  done
-
-  # missing translations in stringsdict files are not filled with the source language automatically
-  if [[ "$dir" == ??"_"* ]]; then
-    echo "skipping $dir/PluralAware.stringsdict right now"
-  else
-    swift $BIN_DIR/MergeStringsDictFiles.swift ../en.lproj/PluralAware.stringsdict PluralAware.stringsdict
-  fi
-  popd
-
-done
-
-for dir in ??_*.lproj
-do
-  pushd $dir
-  lang=${dir:0:2}
-
-  # merge with "main" (country independent) version of the language
-  if [ -e ../$lang.lproj/PluralAware.stringsdict ]; then
-    swift $BIN_DIR/MergeStringsDictFiles.swift ../$lang.lproj/PluralAware.stringsdict PluralAware.stringsdict
-  else
-    swift $BIN_DIR/MergeStringsDictFiles.swift ../en.lproj/PluralAware.stringsdict PluralAware.stringsdict
-  fi
-
-  popd
-
-done
-
 # Parse the PluralAware.stringsdict files to ensure they're not malformed.
 lang_errors=()
 for dir in *.lproj

Scripts/translation/push-metadata-source

@@ -3,9 +3,9 @@
 set -x
 set -e
 
-BASE_DIR=$(git rev-parse --show-toplevel)
-METADATA_ROOT="$BASE_DIR/fastlane/metadata"
+BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-cd $METADATA_ROOT
+REPO_ROOT=$BIN_DIR/../..
+cd $REPO_ROOT
 
-tx push --source
+swift run --package-path Scripts/translation-tool translation-tool upload-metadata

Scripts/translation/push-translation-source

@@ -6,8 +6,7 @@ set -e
 BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 REPO_ROOT=$BIN_DIR/../..
-LOCALIZATION_ROOT=$REPO_ROOT/Signal/translations
-cd $LOCALIZATION_ROOT
+cd $REPO_ROOT
 
-tx push --source
+swift run --package-path Scripts/translation-tool translation-tool upload-resources
 

Scripts/translation/sync-translations

@@ -5,8 +5,7 @@ set -e
 BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 REPO_ROOT=$BIN_DIR/../..
-LOCALIZATION_ROOT=$REPO_ROOT/Signal/translations
-cd $LOCALIZATION_ROOT
+cd $REPO_ROOT
 
 cat <<EOS
 

Signal/translations/.tx/config

@@ -1,17 +0,0 @@
-[main]
-host = https://www.transifex.com
-
-[o:signalapp:p:signal-ios:r:localizablestrings-30]
-file_filter = <lang>.lproj/Localizable.strings
-source_file = en.lproj/Localizable.strings
-source_lang = en
-
-[o:signalapp:p:signal-ios:r:infopliststrings]
-file_filter = <lang>.lproj/InfoPlist.strings
-source_file = en.lproj/InfoPlist.strings
-source_lang = en
-
-[o:signalapp:p:signal-ios:r:pluralawarestringsdict]
-file_filter = <lang>.lproj/PluralAware.stringsdict
-source_file = en.lproj/PluralAware.stringsdict
-source_lang = en

fastlane/metadata/.tx/config

@@ -1,14 +0,0 @@
-[main]
-host = https://www.transifex.com
-lang_map = ar: ar-SA, de: de-DE, en_AU: en-AU, en_CA: en-CA, en_GB: en-GB, en_US: en-US, es: es-ES, fr: fr-FR, nb: no, nl: nl-NL, pt_BR: pt-BR, pt_PT: pt-PT, zh_CN: zh-Hans, zh_TW: zh-Hant
-
-
-[o:signalapp:p:signal-ios:r:app-store-release-notes]
-file_filter = <lang>/release_notes.txt
-source_file = en-US/release_notes.txt
-source_lang = en
-
-[o:signalapp:p:signal-ios:r:app-store-description]
-file_filter = <lang>/description.txt
-source_file = en-US/description.txt
-source_lang = en