diff options
| -rw-r--r-- | README.mkdn | 122 | ||||
| -rw-r--r-- | config/q10.0.yaml | 286 | ||||
| -rw-r--r-- | config/q10.0_extra_packages.xml | 11 | ||||
| -rwxr-xr-x | crowdin_sync.py | 559 |
4 files changed, 978 insertions, 0 deletions
diff --git a/README.mkdn b/README.mkdn new file mode 100644 index 0000000..607f3d1 --- /dev/null +++ b/README.mkdn @@ -0,0 +1,122 @@ +crowdin_sync.py +============== + +Introduction +------------ +This script is forked and modified from Cyanogenmod/LineageOS and CarbonROM. It is used to synchronize AICP translations with Crowdin. +It can handle automatic commiting to Gerrit, submitting open commits on Gerrit as well as pushing/downloading to/from Crowdin. + + +Prerequisites +------------- +1. Python version 2.7.x is needed to execute this script (as it is necessary for building the sources anyway). + +2. The package "python-lxml" is used for removing empty or faulty translations after download. So it needs to be installed on the system. (also see: https://lxml.de/) + +3. The package "python-yaml" must be installed as it is used for parsing the YAML configuration files for determining which files to push. + +4. The package "python-git" is used for Git integration and must be installed (also see: https://gitpython.readthedocs.io/en/stable/). + +5. The prebuilt java version of crowdin-cli >= 2.0.x (see: https://support.crowdin.com/cli-tool/) is required for this to work. + It can be download from here: https://downloads.crowdin.com/cli/v2/crowdin-cli.zip + + Follow the installation instructions on the web page and set up the cli-tool correctly. + + *Note: The current limitation is that the JAR-file must be accessible via the path "/usr/local/bin/". You can of course change that in the python script file.* + +6. Currently the crowdin-cli tool requires either Linux or macOS and an appropiate Java version >= 1.8.xx to work. + +7. <code>/config/AICP-version_extra_packages.xml</code> must be copied to <code>.repo/local_manifests</code> of each "AICP-version" tree. + This makes sure you will sync all the extra packages not included in the main manifest. + Please remember that you should comment out the packages you already have in your local aicp_manifest.xml file + as these are device dependent. + + +Executing +--------- +Export the needed environment variables to set the API key and the base path to your **.bashrc** or **.bash-profile**. +The base path should contain all AICP trees in subfolders, named after AICP branches. + +Needed directory structure: + +* /home +* /your_username + * /aicp + * /config/.yaml #all YAML config files go in here! + * /p9.0 + * /.repo + * /local_manifests/p9.0_extra_packages.xml + * crowdin_sync.py + +Enviroment variables to export: + + export AICP_CROWDIN_API_KEY=aicp_api_key + export AICP_CROWDIN_BASE_PATH_p9_0=your_base_path_and_branch + +Example: + + export AICP_CROWDIN_API_KEY=54e01e81--your-api-key--f6a2724a #Can be found in your project settings page! + export AICP_CROWDIN_BASE_PATH_p9_0=/home/your_username/aicp/p9.0 #This is dependent on the real path to your source tree + +Execute: +The python script "crowdin_sync.py" and the "config" directory should be copied into the base folder structure, e.g. /home/your_username/aicp, like shown above. + +<code>./crowdin_sync.py --username your_gerrit_username --branch AICP_version [--upload-sources] [--upload-translations] [--download] [--local-download] [--submit]</code> + +The script incorporates also a little help that can be invoked by executing: + +<code>./crowdin_sync.py --help</code> + +It will display the following: + +<pre><code>usage: crowdin_sync.py [--help] --username USERNAME --branch BRANCH [--config CONFIG] [--upload-sources] [--upload-translations] [--download] [--local-download] [--submit]<br /> +<br /> +Synchronising AICP translations with Crowdin<br /> +<br /> +optional arguments:<br /> +--help show this help message and exit<br /> +--username USERNAME Gerrit username<br /> +--branch BRANCH AICP branch<br /> +-c CONFIG, --config CONFIG Custom yaml config file to use<br /> +--upload-sources Upload sources to AICP Crowdin<br /> +--upload-translations Upload AICP translations to Crowdin<br /> +--download Download AICP translations from Crowdin<br /> +--local-download Local download AICP translations from Crowdin to PC<br /> +--submit Merge open AICP translations on GerritName<br /></code></pre> + +Examples: + +<code>./crowdin_sync.py --username GerritName --branch p9.0 --upload-sources</code> + +Will upload specified local files from the YAML-config to Crowdin. Translations already there will be preserved. + +<code>./crowdin_sync.py --username GerritName --branch p9.0 --upload-translations</code> + +Will upload local translations to Crowdin, based on YAML-config and from your local sources. + +<code>./crowdin_sync.py --username GerritName --branch p9.0 --download</code> + +Will download translations from Crowdin of the specified branch (p9.0), based on YAML-config, to your local sources, +delete empty translations and upload updated or new translations to AICP Gerrit for review. + +<code>./crowdin_sync.py -username GerritName --branch p9.0 --local-download</code> + +Will download translations from Crowdin of the specified branch (p9.0), based on YAML-config, to your local sources +and delete empty translations. This is useful to perform local builds and test the imported translations. + +<code>./crowdin_sync.py -username GerritName --branch p9.0 --submit</code> + +Will search for open translations commits on AICP's Gerrit on the specified branch (p9.0) and +automatically review, verify and submit them into the repositories. This is useful after successful builds and requires +Gerrit Admin rights to preform this action. + + +Notes: +------ + - The script and the crowdin-cli JAR file provide some output that make the actions performed show up + in the terminal, so you can follow the execution of the commands. + - The crowdin JAR file will display a message in the terminal, if it is outdated and found a + newer version available for download. This is not an error, just a reminder for you! + - When committing a translation fails, the reason of it cannot be determined everytime. The script will + simply display the message "Nothing to commit" or an error message mentioning the specific file/commit. + The script will continue when this happens and display "Finished! Nothing to do or commit anymore.". diff --git a/config/q10.0.yaml b/config/q10.0.yaml new file mode 100644 index 0000000..9829bc5 --- /dev/null +++ b/config/q10.0.yaml @@ -0,0 +1,286 @@ +# p9.0.yaml +# +# Crowdin configuration file for AICP +# +# Copyright (C) 2014-2016 The CyanogenMod Project +# Copyright (C) 2017-2019 The LineageOS Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Your crowdin's credentials +api_key_env: AICP_CROWDIN_API_KEY +base_path_env: AICP_CROWDIN_BASE_PATH_q10_0 +project_identifier: aicp +preserve_hierarchy: true + +files: +# Frameworks + + # frameworks-res + - + source: '/frameworks/base/core/res/res/values/aicp_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: &anchor + android_code: + # we need this mapping since Crowdin expects directories + # to be named like "values-uk-rUA" + # acording to specification instead of just "uk" + af: af + am: am + ar: ar + as: as-rIN + ast: ast-rES + az: az-rAZ + be: be-rBY + bg: bg + bn: bn-rBD + bs: bs-rBA + ca: ca + cs: cs + cy: cy + da: da + de: de + el: el + en-AU: en-rAU + en-GB: en-rGB + en-IN: en-rIN + en-PT: en-rPT + eo: eo + es-ES: es + es-MX: es-rMX + es-US: es-rUS + et: et-rEE + eu: eu-rES + fa: fa + fi: fi + fr: fr + fr-CA: fr-rCA + fy-NL: fy-rNL + gl: gl-rES + gu-IN: gu-rIN + he: iw + hi: hi + hr: hr + hu: hu + hy-AM: hy-rAM + id: in + is: is-rIS + it: it + ja: ja + ka: ka-rGE + kk: kk-rKZ + km: km-rKH + kn: kn-rIN + ko: ko + ku: ku + ky: ky-rKG + lb: lb + lo: lo-rLA + lt: lt + lv: lv + mk: mk-rMK + ml-IN: ml-rIN + mn: mn-rMN + mr: mr-rIN + ms: ms-rMY + my: my-rMM + nb: nb + ne-NP: ne-rNP + nl: nl + or: or-rIN + pa-IN: pa-rIN + pl: pl + pt-PT: pt-rPT + pt-BR: pt-rBR + rm-CH: rm + ro: ro + ru: ru + si-LK: si-rLK + sk: sk + sl: sl + sq: sq-rAL + sr: sr + sv-SE: sv + sw: sw + ta: ta-rIN + te: te-rIN + th: th + tl: tl + tr: tr + ug: ug + uk: uk + ur-PK: ur-rPK + uz: uz-rUZ + vi: vi + zh-CN: zh-rCN + zh-HK: zh-rHK + zh-TW: zh-rTW + zu: zu + + # frameworks-opt-slimrecent + - + source: '/frameworks/opt/slimrecent/res/values/strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + + # frameworks-opt-aicpgear + - + source: '/frameworks/opt/aicpgear/preference/res/values/color_picker_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/frameworks/opt/aicpgear/preference/res/values/seekbar_preference_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + + # frameworks-base-packages-SettingsLib + - + source: '/frameworks/base/packages/SettingsLib/res/values/aicp_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + + # SystemUI + - + source: '/frameworks/base/packages/SystemUI/res/values/aicp_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/frameworks/base/packages/SystemUI/res-keyguard/values/aicp_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + +# Packages + + # AicpExtras + - + source: '/packages/apps/AicpExtras/res/values/strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/packages/apps/AicpExtras/res/values/changelog_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/packages/apps/AicpExtras/res/values/dslv_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/packages/apps/AicpExtras/res/values/master_switch_preference_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/packages/apps/AicpExtras/res/values/stats_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/packages/apps/AicpExtras/res/values/switch_bar_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + + # Updater + - + source: '/packages/apps/Updater/res/values/strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + + # Dialer + - + source: '/packages/apps/Dialer/java/com/android/dialer/app/res/values/aicp_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/packages/apps/Dialer/java/com/android/dialer/lookup/res/values/aicp_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + + # JamesDSPManager + - + source: '/packages/apps/JamesDSPManager/app/src/main/res/values/strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/packages/apps/JamesDSPManager/app/src/main/res/values/arrays.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + +# # FMRadio +# - +# source: '/packages/apps/FMRadio/res/values/cm_strings.xml' +# translation: '/%original_path%-%android_code%/%original_file_name%' +# languages_mapping: *anchor +# - +# source: '/packages/apps/FMRadio/res/values/strings.xml' +# translation: '/%original_path%-%android_code%/%original_file_name%' +# languages_mapping: *anchor + + # Launcher3 + - + source: '/packages/apps/Launcher3/res/values/aicp_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + + # Settings + - + source: '/packages/apps/Settings/res/values/aicp_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + +# # SmartNavigation +# - +# source: '/packages/apps/SmartNav/res/values/strings.xml' +# translation: '/%original_path%-%android_code%/%original_file_name%' +# languages_mapping: *anchor + +# Providers + +# # MediaProvider +# - +# source: '/packages/providers/MediaProvider/res/values/aicp_strings.xml' +# translation: '/%original_path%-%android_code%/%original_file_name%' +# languages_mapping: *anchor + +# Services + + # OmniJaws + - + source: '/packages/services/OmniJaws/res/values/strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/packages/services/OmniJaws/res/values/aicp_strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + +# Device specific + + # device_oppo_common DeviceHandler for OnePlus + - + source: '/device/oppo/common/DeviceHandler/res/values/strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + - + source: '/device/oppo/common/overlay/packages/apps/Settings/res/values/devicehandlerstrings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + + # DeviceSettings + - + source: '/packages/resources/devicesettings/res/values/strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor + + # packages_apps_FlipFlap + - + source: '/packages/apps/FlipFlap/res/values/strings.xml' + translation: '/%original_path%-%android_code%/%original_file_name%' + languages_mapping: *anchor diff --git a/config/q10.0_extra_packages.xml b/config/q10.0_extra_packages.xml new file mode 100644 index 0000000..562ac5f --- /dev/null +++ b/config/q10.0_extra_packages.xml @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="UTF-8"?> +<manifest> + <!-- Extra packages not included in the main manifest --> + + <!-- Devicespecific repositories that can be translated --> + <project path="device/oppo/common" name="AICP/device_oppo_common" /> + <project path="packages/apps/FlipFlap" name="AICP/packages_apps_FlipFlap" /> + <project path="packages/apps/MusicFX" name="AICP/packages_apps_MusicFX" /> + <project path="packages/apps/FMRadio" name="AICP/packages_apps_FMRadio" /> + <project path="packages/apps/JamesDSPManager" name="AICP/packages_apps_JamesDSPManager" /> +</manifest> diff --git a/crowdin_sync.py b/crowdin_sync.py new file mode 100755 index 0000000..ac822c6 --- /dev/null +++ b/crowdin_sync.py @@ -0,0 +1,559 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# crowdin-cli_sync.py +# +# Updates Crowdin source translations and pushes translations +# directly to AICP Gerrit. +# +# Copyright (C) 2014-2016 The CyanogenMod Project +# Copyright (C) 2017-2019 The LineageOS Project +# This code has been modified. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ################################# IMPORTS ################################## # + +from __future__ import print_function + +import argparse +import json +import git +import os +import re +import subprocess +import sys +import yaml + +from lxml import etree +from xml.dom import minidom + +# ################################# GLOBALS ################################## # + +_DIR = os.path.dirname(os.path.realpath(__file__)) +_COMMITS_CREATED = False + +# ################################ FUNCTIONS ################################# # + + +def run_subprocess(cmd, silent=False): + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + comm = p.communicate() + exit_code = p.returncode + if exit_code != 0 and not silent: + print("There was an error running the subprocess.\n" + "cmd: %s\n" + "exit code: %d\n" + "stdout: %s\n" + "stderr: %s" % (cmd, exit_code, comm[0], comm[1]), + file=sys.stderr) + return comm, exit_code + + +def add_target_paths(config_files, repo, base_path, project_path): + # Add or remove the files given in the config files to the commit + count = 0 + file_paths = [] + for f in config_files: + fh = open(f, "r") + try: + config = yaml.load(fh, Loader=yaml.FullLoader) + for tf in config['files']: + if project_path in tf['source']: + target_path = tf['translation'] + lang_codes = tf['languages_mapping']['android_code'] + for l in lang_codes: + lpath = get_target_path(tf['translation'], tf['source'], + lang_codes[l], project_path) + file_paths.append(lpath) + except yaml.YAMLError as e: + print(e, '\n Could not parse YAML.') + exit() + fh.close() + + # Strip all comments + for f in file_paths: + clean_file(base_path, project_path, f) + + # Modified and untracked files + modified = repo.git.ls_files(m=True, o=True) + for m in modified.split('\n'): + if m in file_paths: + repo.git.add(m) + count += 1 + + deleted = repo.git.ls_files(d=True) + for d in deleted.split('\n'): + if d in file_paths: + repo.git.rm(d) + count += 1 + + return count + + +def split_path(path): + # Split the given string to path and filename + if '/' in path: + original_file_name = path[1:][path.rfind("/"):] + original_path = path[:path.rfind("/")] + else: + original_file_name = path + original_path = '' + + return original_path, original_file_name + + +def get_target_path(pattern, source, lang, project_path): + # Make strings like '/%original_path%-%android_code%/%original_file_name%' valid file paths + # based on the source string's path + original_path, original_file_name = split_path(source) + + target_path = pattern #.lstrip('/') + target_path = target_path.replace('%original_path%', original_path) + target_path = target_path.replace('%android_code%', lang) + target_path = target_path.replace('%original_file_name%', original_file_name) + target_path = target_path.replace(project_path, '') + target_path = target_path.lstrip('/') + return target_path + + +def clean_file(base_path, project_path, filename): + path = base_path + '/' + project_path + '/' + filename + + # We don't want to create every file, just work with those already existing + if not os.path.isfile(path): + return + + try: + fh = open(path, 'r+') + except: + print('Something went wrong while opening file %s' % (path)) + return + + XML = fh.read() + tree = etree.fromstring(XML) + + header = '' + comments = tree.xpath('//comment()') + for c in comments: + p = c.getparent() + if p is None: + # Keep all comments in header + header += str(c).replace('\\n', '\n').replace('\\t', '\t') + '\n' + continue + p.remove(c) + + content = '' + + # Take the original xml declaration and prepend it + declaration = XML.split('\n')[0] + if '<?' in declaration: + content = declaration + '\n' + + content += etree.tostring(tree, pretty_print=True, encoding="utf-8", xml_declaration=False) + + if header != '': + content = content.replace('?>\n', '?>\n' + header) + + # Sometimes spaces are added, we don't want them + content = re.sub("[ ]*<\/resources>", "</resources>", content) + + # Overwrite file with content stripped by all comments + fh.seek(0) + fh.write(content) + fh.truncate() + fh.close() + + # Remove files which don't have any translated strings + empty_contents = { + '<resources/>', + '<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>', + ('<resources xmlns:android=' + '"http://schemas.android.com/apk/res/android"/>'), + ('<resources xmlns:android="http://schemas.android.com/apk/res/android"' + ' xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>'), + ('<resources xmlns:tools="http://schemas.android.com/tools"' + ' xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>'), + ('<resources xmlns:android="http://schemas.android.com/apk/res/android">\n</resources>'), + ('<resources xmlns:android="http://schemas.android.com/apk/res/android"' + ' xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">\n</resources>'), + ('<resources xmlns:tools="http://schemas.android.com/tools"' + ' xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">\n</resources>'), + ('<resources>\n</resources>') + } + for line in empty_contents: + if line in content: + print('Removing ' + path) + os.remove(path) + break + +def push_as_commit(config_files, base_path, path, name, branch, username): + print('Committing %s on branch %s' % (name, branch)) + + # Get path + project_path = path + path = os.path.join(base_path, path) + if not path.endswith('.git'): + path = os.path.join(path, '.git') + + # Create repo object + repo = git.Repo(path) + + # Add all files to commit + count = add_target_paths(config_files, repo, base_path, project_path) + + if count == 0: + print('Nothing to commit') + return + + # Create commit; if it fails, probably empty so skipping + try: + repo.git.commit(m='Automatic AICP translation import') + except: + print('Failed to create commit for %s, probably empty: skipping' + % name, file=sys.stderr) + return + + # Push commit + try: + repo.git.push('ssh://%s@gerrit.aicp-rom.com:29418/%s' % (username, name), + 'HEAD:refs/for/%s%%topic=Translations-%s' % (branch, branch)) + print('Successfully pushed commit for %s' % name) + except: + print('Failed to push commit for %s' % name, file=sys.stderr) + + _COMMITS_CREATED = True + + +def submit_gerrit(branch, username): + # Find all open translation changes + cmd = ['ssh', '-p', '29418', + '{}@gerrit.aicp-rom.com'.format(username), + 'gerrit', 'query', + 'status:open', + 'branch:{}'.format(branch), + 'message:"Automatic AICP translation import"', + 'topic:Translations-{}'.format(branch), + '--current-patch-set', + '--format=JSON'] + commits = 0 + msg, code = run_subprocess(cmd) + if code != 0: + print('Failed: {0}'.format(msg[1])) + return + + # Each line is one valid JSON object, except the last one, which is empty + for line in msg[0].strip('\n').split('\n'): + js = json.loads(line) + # We get valid JSON, but not every result line is a line we want + if not 'currentPatchSet' in js or not 'revision' in js['currentPatchSet']: + continue + # Add Code-Review +2 and Verified +1 labels and submit + cmd = ['ssh', '-p', '29418', + '{}@gerrit.aicp-rom.com'.format(username), + 'gerrit', 'review', + '--verified +1', + '--code-review +2', + '--submit', js['currentPatchSet']['revision']] + msg, code = run_subprocess(cmd, True) + if code != 0: + errorText = msg[1].replace('\n\n', '; ').replace('\n', '') + print('Submitting commit {0} failed: {1}'.format(js['url'], errorText)) + else: + print('Success when submitting commit {0}'.format(js['url'])) + + commits += 1 + + if commits == 0: + print("Nothing to submit!") + return + + +def check_run(cmd): + p = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) + ret = p.wait() + if ret != 0: + print('Failed to run cmd: %s' % ' '.join(cmd), file=sys.stderr) + sys.exit(ret) + + +def find_xml(base_path): + for dp, dn, file_names in os.walk(base_path): + for f in file_names: + if os.path.splitext(f)[1] == '.xml': + yield os.path.join(dp, f) + +# ############################################################################ # + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Synchronising AICP translations with Crowdin") + parser.add_argument('--username', help='Gerrit username', + required=True) + parser.add_argument('--branch', help='AICP branch', + required=True) + parser.add_argument('-c', '--config', help='Custom yaml config') + parser.add_argument('--upload-sources', action='store_true', + help='Upload sources to AICP Crowdin') + parser.add_argument('--upload-translations', action='store_true', + help='Upload AICP translations to Crowdin') + parser.add_argument('--download', action='store_true', + help='Download AICP translations from Crowdin') + parser.add_argument('--local-download', action='store_true', + help='local Download AICP translations from Crowdin') + parser.add_argument('--submit', action='store_true', + help='Auto-Merge open AICP translations on Gerrit') + return parser.parse_args() + +# ################################# PREPARE ################################## # + + +def check_dependencies(): + # Check for Java version of crowdin-cli + cmd = ['find', '/usr/local/bin/crowdin-cli.jar'] + if run_subprocess(cmd, silent=True)[1] != 0: + print('You have not installed crowdin-cli.jar in its default location.', file=sys.stderr) + return False + return True + + +def load_xml(x): + try: + return minidom.parse(x) + except IOError: + print('You have no %s.' % x, file=sys.stderr) + return None + except Exception: + print('Malformed %s.' % x, file=sys.stderr) + return None + + +def check_files(files): + for f in files: + if not os.path.isfile(f): + print('You have no %s.' % f, file=sys.stderr) + return False + return True + +# ################################### MAIN ################################### # + + +def upload_sources_crowdin(branch, config): + if config: + print('\nUploading sources to Crowdin (custom config)') + check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s/config/%s' % (_DIR, config), + 'upload', 'sources', '--branch=%s' % branch]) + else: + print('\nUploading sources to Crowdin (AOSP supported languages)') + check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s/config/%s.yaml' % (_DIR, branch), + 'upload', 'sources', '--branch=%s' % branch]) + + print('\nUploading GZOSP sources to Crowdin (AOSP supported languages)') + check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s/config/%s_gzosp.yaml' % (_DIR, branch), + 'upload', 'sources', '--branch=%s' % branch]) + + +def upload_translations_crowdin(branch, config): + if config: + print('\nUploading translations to Crowdin (custom config)') + check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s/config/%s' % (_DIR, config), + 'upload', 'translations', '--branch=%s' % branch, + '--no-import-duplicates', '--import-eq-suggestions', + '--auto-approve-imported']) + else: + print('\nUploading translations to Crowdin ' + '(AOSP supported languages)') + check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s/config/%s.yaml' % (_DIR, branch), + 'upload', 'translations', '--branch=%s' % branch, + '--no-import-duplicates', '--import-eq-suggestions', + '--auto-approve-imported']) + + print('\nUploading GZOSP translations to Crowdin ' + '(AOSP supported languages)') + check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s/config/%s_gzosp.yaml' % (_DIR, branch), + 'upload', 'translations', '--branch=%s' % branch, + '--no-import-duplicates', '--import-eq-suggestions', + '--auto-approve-imported']) + + +def local_download(base_path, branch, xml, config): + if config: + print('\nDownloading translations from Crowdin (custom config)') + check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s/config/%s' % (_DIR, config), + 'download', '--branch=%s' % branch]) + else: + print('\nDownloading translations from Crowdin ' + '(AOSP supported languages)') + check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s/config/%s.yaml' % (_DIR, branch), + 'download', '--branch=%s' % branch]) + + print('\nDownloading GZOSP translations from Crowdin ' + '(AOSP supported languages)') + check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s/config/%s_gzosp.yaml' % (_DIR, branch), + 'download', '--branch=%s' % branch]) + + +def download_crowdin(base_path, branch, xml, username, config): + local_download(base_path, branch, xml, config) + + print('\nCreating a list of pushable translations') + # Get all files that Crowdin pushed + paths = [] + if config: + files = [('%s/config/%s' % (_DIR, config))] + else: + files = [('%s/config/%s.yaml' % (_DIR, branch)), + ('%s/config/%s_gzosp.yaml' % (_DIR, branch))] + for c in files: + cmd = ['java', '-jar', '/usr/local/bin/crowdin-cli.jar', + '--config=%s' % c, 'list', 'project', '--branch=%s' % branch] + comm, ret = run_subprocess(cmd) + if ret != 0: + sys.exit(ret) + for p in str(comm[0]).split("\n"): + paths.append(p.replace('/%s' % branch, '')) + + print('\nUploading translations to AICP Gerrit') + items = [x for sub in xml for x in sub.getElementsByTagName('project')] + all_projects = [] + + for path in paths: + path = path.strip() + if not path: + continue + + if "/res" not in path: + print('WARNING: Cannot determine project root dir of ' + '[%s], skipping.' % path) + continue + result = path.split('/res')[0].strip('/') + if result == path.strip('/'): + print('WARNING: Cannot determine project root dir of ' + '[%s], skipping.' % path) + continue + + if result in all_projects: + continue + + # When a project has multiple translatable files, Crowdin will + # give duplicates. + # We don't want that (useless empty commits), so we save each + # project in all_projects and check if it's already in there. + all_projects.append(result) + + # Search %(branch)/platform_manifest/*_default.xml or + # config/%(branch)_extra_packages.xml for the project's name + for project in items: + path = project.attributes['path'].value + if not (result + '/').startswith(path +'/'): + continue + if result != path: + if path in all_projects: + break + result = path + all_projects.append(result) + + br = project.getAttribute('revision') or branch + + push_as_commit(files, base_path, result, + project.getAttribute('name'), br, username) + break + + +def main(): + args = parse_args() + default_branch = args.branch + + if args.submit: + if args.username is None: + print('Argument -u/--username is required for submitting!') + sys.exit(1) + submit_gerrit(default_branch, args.username) + sys.exit(0) + + base_path_branch_suffix = default_branch.replace('.', '_') + base_path_env = 'AICP_CROWDIN_BASE_PATH_%s' % base_path_branch_suffix + base_path = os.getenv(base_path_env) + if base_path is None: + cwd = os.getcwd() + print('You have not set %s. Defaulting to %s' % (base_path_env, cwd)) + base_path = cwd + if not os.path.isdir(base_path): + print('%s is not a real directory: %s' % (base_path_env, base_path)) + sys.exit(1) + + if not check_dependencies(): + sys.exit(1) + + xml_default = load_xml(x='%s/platform_manifest/default.xml' % base_path) + if xml_default is None: + sys.exit(1) + + xml_extra = load_xml(x='%s/config/%s_extra_packages.xml' % (_DIR, default_branch)) + if xml_extra is None: + sys.exit(1) + + xml_gzosp = load_xml(x='%s/platform_manifest/gzosp_default.xml' % base_path) + if xml_gzosp is None: + sys.exit(1) + + xml_aicp = load_xml(x='%s/platform_manifest/aicp_default.xml' % base_path) + if xml_aicp is not None: + xml_files = (xml_default, xml_aicp, xml_extra) + else: + xml_files = (xml_default, xml_extra, xml_gzosp) + + if args.config: + files = [('%s/config/%s' % (_DIR, args.config))] + else: + files = [('%s/config/%s.yaml' % (_DIR, default_branch)), + ('%s/config/%s_gzosp.yaml' % (_DIR, default_branch))] + if not check_files(files): + sys.exit(1) + + if args.download and args.username is None: + print('Argument --username is required to perform this action') + sys.exit(1) + + if args.upload_sources: + upload_sources_crowdin(default_branch, args.config) + + if args.upload_translations: + upload_translations_crowdin(default_branch, args.config) + + if args.download: + download_crowdin(base_path, default_branch, xml_files, + args.username, args.config) + + if args.local_download: + local_download(base_path, default_branch, xml_files, args.config) + + if _COMMITS_CREATED: + print('\nDone!') + sys.exit(0) + else: + print('\nFinished! Nothing to do or commit anymore.') + sys.exit(-1) + +if __name__ == '__main__': + main() |