crowdin: update the script

Update including squashed commits from LineageOS:

crowdin: Improve console output

* Reduce the amount of output lines - this makes it more readable
* Add a signal handler to handle aborts - we don't need a stacktrace here

crowdin: PyYAML yaml.load(input) deprecation

* YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated,
  as the default Loader is unsafe.
  Please read https://msg.pyyaml.org/load for full details
* Use yaml.safe_load(input) instead - we only use a small subset of YAML  so this works as expected
* Test: Compared the results of yaml.load() and yaml.safe_load()

crowdin: Replace minidom with lxml.etree

* We use lxml already in other places, so thats just commonisation
* It was still marked as "TODO", so get it done

crowdin: Fix removing invalid product strings

* Removing items from a list while iterating it is not a good idea
  and results in unexpected results
* Instead of removing them, add them to an ignore list and check the current item against it

crowdin: Fix console-output on download

* When we have errors or messages during cleanup,
  the string is currently in the wrong line

crowdin: Fix submitting

Original author: "Michael W <baddaemon87@gmail.com>"

Change-Id: I3520cae14b4d9c1924c27e72f8982bac4547fc57

1 files changed, 18 insertions, 15 deletions

diff --git a/crowdin_sync.py b/crowdin_sync.py
index 7781371..a6a26a9 100755
--- a/crowdin_sync.py
+++ b/crowdin_sync.py
@@ -37,7 +37,6 @@ import yaml
 
 from lxml import etree
 from signal import signal, SIGINT
-from xml.dom import minidom
 
 # ################################# GLOBALS ################################## #
 
@@ -155,7 +154,11 @@ def clean_xml_file(base_path, project_path, filename):
     # Remove strings with 'product=*' attribute but no 'product=default'
     # This will ensure aapt2 will not throw an error when building these
     productStrings = tree.xpath("//string[@product]")
+    alreadyRemoved = []
     for ps in productStrings:
+        # if we already removed the items, don't process them
+        if ps in alreadyRemoved:
+            continue
         stringName = ps.get('name')
         stringsWithSameName = tree.xpath("//string[@name='{0}']".format(stringName))
 
@@ -170,11 +173,11 @@ def clean_xml_file(base_path, project_path, filename):
         # Every occurance of the string has to be removed when no string with the same name and
         # 'product=default' (or no product attribute) was found
         if not hasProductDefault:
-            print("{0}: Found string '{1}' with missing 'product=default' attribute"
-                  .format(path, stringName))
+            print("\n{0}: Found string '{1}' with missing 'product=default' attribute"
+                  .format(path, stringName), end='')
             for string in stringsWithSameName:
                 tree.remove(string)
-                productStrings.remove(string)
+                alreadyRemoved.append(string)
 
     header = ''
     comments = tree.xpath('//comment()')
@@ -210,7 +213,7 @@ def clean_xml_file(base_path, project_path, filename):
     # Remove files which don't have any translated strings
     contentList = list(tree)
     if len(contentList) == 0:
-        print('Removing ' + path)
+        print('\nRemoving ' + path)
         os.remove(path)
 
 
@@ -243,7 +246,7 @@ def reset_file(filepath, repo):
 
 
 def push_as_commit(config_files, base_path, path, name, branch, username):
-    print('Committing %s on branch %s: ' % (name, branch), end='')
+    print('\nCommitting %s on branch %s: ' % (name, branch), end='')
 
     # Get path
     project_path = path
@@ -374,12 +377,12 @@ def check_dependencies():
 
 def load_xml(x):
     try:
-        return minidom.parse(x)
-    except IOError:
-        print('You have no %s.' % x, file=sys.stderr)
+        return etree.parse(x)
+    except etree.XMLSyntaxError:
+        print('Malformed %s.' % x, file=sys.stderr)
         return None
     except Exception:
-        print('Malformed %s.' % x, file=sys.stderr)
+        print('You have no %s.' % x, file=sys.stderr)
         return None
 
 
@@ -433,7 +436,7 @@ def local_download(base_path, branch, xml, config):
     else:
         print('\nDownloading translations from Crowdin '
               '(AOSP supported languages)')
-        check_run(['crowdin-cli',
+        check_run(['java', '-jar', '/usr/local/bin/crowdin-cli.jar',
                    '--config=%s/config/%s.yaml' % (_DIR, branch),
                    'download', '--branch=%s' % branch])
 
@@ -480,7 +483,7 @@ def download_crowdin(base_path, branch, xml, username, config):
             paths.append(p.replace('/%s' % branch, ''))
 
     print('\nUploading translations to AICP Gerrit')
-    items = [x for sub in xml for x in sub.getElementsByTagName('project')]
+    items = [x for xmlfile in xml for x in xmlfile.findall("//project")]
     all_projects = []
 
     for path in paths:
@@ -524,7 +527,7 @@ def download_crowdin(base_path, branch, xml, username, config):
         resultPath = None
         resultProject = None
         for project in items:
-            path = project.attributes['path'].value
+            path = project.get('path')
             if not (result + '/').startswith(path +'/'):
                 continue
             # We want the longest match, so projects in subfolders of other projects are also
@@ -543,10 +546,10 @@ def download_crowdin(base_path, branch, xml, username, config):
             result = resultPath
             all_projects.append(result)
 
-        br = resultProject.getAttribute('revision') or branch
+        br = resultProject.get('revision') or branch
 
         push_as_commit(files, base_path, result,
-                       resultProject.getAttribute('name'), br, username)
+                       resultProject.get('name'), br, username)
 
 
 def sig_handler(signal_received, frame):