From 2b6d692f063b34338931e5a79d62fa3c30edc77e Mon Sep 17 00:00:00 2001
From: Hans-Christoph Steiner <hans@eds.org>
Date: Tue, 7 Jun 2016 20:13:54 +0200
Subject: [PATCH] use UTF8 as default instead of ASCII for .java .gradle
 pom.xml

.java .gradle and XML files all can use any encoding.  Most code is ASCII,
but authors' names, etc. can easily be non-ASCII.  UTF-8 is by far the most
common file encoding.  While UTF-8 is the default encoding inside the code
in Python 3, it still has to deal with the real world, so the encoding
needs to be explicitly set when reading and writing files. So this switches
fdroidserver to expect UTF-8 instead of ASCII when parsing these files. For
now, this commit means that we only support UTF-8 encoded *.java, pom.xml
or *.gradle files.  Ideally, the code would detect the encoding and use the
actual one, but that's a lot more work, and its something that will not
happen often. We can cross that bridge when we come to it.

One approach, which is taken in the commit when possible, is to keep the
data as `bytes`, in which case the encoding doesn't matter.

This also fixes this crash when parsing gradle and maven files with
non-ASCII chars:

ERROR: test_adapt_gradle (__main__.BuildTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/var/lib/jenkins/workspace/fdroidserver-eighthave/tests/build.TestCase", line 59, in test_adapt_gradle
    fdroidserver.build.adapt_gradle(testsdir)
  File "/var/lib/jenkins/workspace/fdroidserver-eighthave/fdroidserver/build.py", line 445, in adapt_gradle
    path)
  File "/var/lib/jenkins/workspace/fdroidserver-eighthave/fdroidserver/common.py", line 188, in regsub_file
    text = f.read()
  File "/usr/lib/python3.4/encodings/ascii.py", line 26, in decode
    return codecs.ascii_decode(input, self.errors)[0]
UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 9460: ordinal not in range(128)
---
 fdroidserver/common.py   | 10 +++++-----
 fdroidserver/metadata.py |  2 +-
 fdroidserver/scanner.py  |  4 ++--
 tests/build.TestCase     |  4 ++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 0a3c8e75..03a8743d 100644
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -184,10 +184,10 @@ def fill_config_defaults(thisconfig):
 
 
 def regsub_file(pattern, repl, path):
-    with open(path, 'r') as f:
+    with open(path, 'rb') as f:
         text = f.read()
-    text = re.sub(pattern, repl, text)
-    with open(path, 'w') as f:
+    text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
+    with open(path, 'wb') as f:
         f.write(text)
 
 
@@ -1724,14 +1724,14 @@ def remove_signing_keys(build_dir):
         if 'build.gradle' in files:
             path = os.path.join(root, 'build.gradle')
 
-            with open(path, "r") as o:
+            with open(path, "r", encoding='utf8') as o:
                 lines = o.readlines()
 
             changed = False
 
             opened = 0
             i = 0
-            with open(path, "w") as o:
+            with open(path, "w", encoding='utf8') as o:
                 while i < len(lines):
                     line = lines[i]
                     i += 1
diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py
index a9be8ef0..ded6cd1a 100644
--- a/fdroidserver/metadata.py
+++ b/fdroidserver/metadata.py
@@ -836,7 +836,7 @@ def get_default_app_info(metadatapath=None):
             for root, dirs, files in os.walk(os.getcwd()):
                 if 'build.gradle' in files:
                     p = os.path.join(root, 'build.gradle')
-                    with open(p) as f:
+                    with open(p, 'rb') as f:
                         data = f.read()
                     m = pattern.search(data)
                     if m:
diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py
index 53c938fb..41ecbb50 100644
--- a/fdroidserver/scanner.py
+++ b/fdroidserver/scanner.py
@@ -199,7 +199,7 @@ def scan_source(build_dir, root_dir, build):
             elif ext == 'java':
                 if not os.path.isfile(fp):
                     continue
-                with open(fp, 'r') as f:
+                with open(fp, 'r', encoding='utf8') as f:
                     for line in f:
                         if 'DexClassLoader' in line:
                             count += handleproblem('DexClassLoader', fd, fp)
@@ -208,7 +208,7 @@ def scan_source(build_dir, root_dir, build):
             elif ext == 'gradle':
                 if not os.path.isfile(fp):
                     continue
-                with open(fp, 'r') as f:
+                with open(fp, 'r', encoding='utf8') as f:
                     lines = f.readlines()
                 for i, line in enumerate(lines):
                     if is_used_by_gradle(line):
diff --git a/tests/build.TestCase b/tests/build.TestCase
index b3b90fc3..369c3836 100755
--- a/tests/build.TestCase
+++ b/tests/build.TestCase
@@ -57,13 +57,13 @@ class BuildTest(unittest.TestCase):
         fdroidserver.build.config = {}
         fdroidserver.build.config['build_tools'] = teststring
         fdroidserver.build.adapt_gradle(testsdir)
-        pattern = re.compile("buildToolsVersion[\s=]+'%s'\s+" % teststring)
+        pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8'))
         for p in ('source-files/fdroid/fdroidclient/build.gradle',
                   'source-files/Zillode/syncthing-silk/build.gradle',
                   'source-files/open-keychain/open-keychain/build.gradle',
                   'source-files/osmandapp/osmand/build.gradle',
                   'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle'):
-            with open(os.path.join(testsdir, p), 'r') as f:
+            with open(os.path.join(testsdir, p), 'rb') as f:
                 filedata = f.read()
             self.assertIsNotNone(pattern.search(filedata))