Merge branch 'utf8-fixes' into 'master'

UTF-8 fixes

See merge request fdroid/fdroidserver!587

fdroidserver/common.py

@@ -1934,7 +1934,7 @@ class KnownApks:
         self.path = os.path.join('stats', 'known_apks.txt')
         self.apks = {}
         if os.path.isfile(self.path):
-            with open(self.path, 'r', encoding='utf8') as f:
+            with open(self.path, 'r') as f:
                 for line in f:
                     t = line.rstrip().split(' ')
                     if len(t) == 2:
@@ -1962,7 +1962,7 @@ class KnownApks:
                 line += ' ' + added.strftime('%Y-%m-%d')
             lst.append(line)
 
-        with open(self.path, 'w', encoding='utf8') as f:
+        with open(self.path, 'w') as f:
             for line in sorted(lst, key=natural_key):
                 f.write(line + '\n')
 
@@ -2349,14 +2349,14 @@ def remove_signing_keys(build_dir):
         if 'build.gradle' in files:
             path = os.path.join(root, 'build.gradle')
 
-            with open(path, "r", encoding='utf8') as o:
+            with open(path, "r") as o:
                 lines = o.readlines()
 
             changed = False
 
             opened = 0
             i = 0
-            with open(path, "w", encoding='utf8') as o:
+            with open(path, "w") as o:
                 while i < len(lines):
                     line = lines[i]
                     i += 1
@@ -3138,7 +3138,7 @@ def write_to_config(thisconfig, key, value=None, config_file=None):
     if not os.path.exists(cfg):
         open(cfg, 'a').close()
         logging.info("Creating empty " + cfg)
-    with open(cfg, 'r', encoding="utf-8") as f:
+    with open(cfg, 'r') as f:
         lines = f.readlines()
 
     # make sure the file ends with a carraige return
@@ -3157,7 +3157,7 @@ def write_to_config(thisconfig, key, value=None, config_file=None):
     # second instance of this line for this key in the document.
     didRepl = False
     # edit config file
-    with open(cfg, 'w', encoding="utf-8") as f:
+    with open(cfg, 'w') as f:
         for line in lines:
             if pattern.match(line) or pattern2.match(line):
                 if not didRepl:

fdroidserver/init.py

@@ -37,12 +37,12 @@ options = None
 
 def disable_in_config(key, value):
     '''write a key/value to the local config.py, then comment it out'''
-    with open('config.py', 'r', encoding='utf8') as f:
+    with open('config.py', 'r') as f:
         data = f.read()
     pattern = r'\n[\s#]*' + key + r'\s*=\s*"[^"]*"'
     repl = '\n#' + key + ' = "' + value + '"'
     data = re.sub(pattern, repl, data)
-    with open('config.py', 'w', encoding='utf8') as f:
+    with open('config.py', 'w') as f:
         f.writelines(data)
 
 

fdroidserver/metadata.py

@@ -706,7 +706,7 @@ def parse_srclib(metadatapath):
     if not os.path.exists(metadatapath):
         return thisinfo
 
-    metafile = open(metadatapath, "r", encoding='utf-8')
+    metafile = open(metadatapath, "r")
 
     n = 0
     for line in metafile:
@@ -1014,7 +1014,7 @@ def parse_metadata(metadatapath, check_vcs=False, refresh=True):
     else:
         app.id = name
 
-    with open(metadatapath, 'r', encoding='utf-8') as mf:
+    with open(metadatapath, 'r') as mf:
         if ext == 'txt':
             parse_txt_metadata(mf, app)
         elif ext == 'json':
@@ -1565,7 +1565,7 @@ def write_metadata(metadatapath, app):
                           .format(path=metadatapath, formats=', '.join(accepted)))
 
     try:
-        with open(metadatapath, 'w', encoding='utf8') as mf:
+        with open(metadatapath, 'w') as mf:
             if ext == 'txt':
                 return write_txt(mf, app)
             elif ext == 'yml':

fdroidserver/rewritemeta.py

@@ -34,7 +34,7 @@ def proper_format(app):
     s = io.StringIO()
     # TODO: currently reading entire file again, should reuse first
     # read in metadata.py
-    with open(app.metadatapath, 'r', encoding='utf8') as f:
+    with open(app.metadatapath, 'r') as f:
         cur_content = f.read()
     _ignored, extension = common.get_extension(app.metadatapath)
     if extension == 'yml':

fdroidserver/scanner.py

@@ -230,7 +230,7 @@ def scan_source(build_dir, build=metadata.Build()):
             elif ext == 'java':
                 if not os.path.isfile(filepath):
                     continue
-                with open(filepath, 'r', encoding='utf8', errors='replace') as f:
+                with open(filepath, 'r', errors='replace') as f:
                     for line in f:
                         if 'DexClassLoader' in line:
                             count += handleproblem('DexClassLoader', path_in_build_dir, filepath)
@@ -239,7 +239,7 @@ def scan_source(build_dir, build=metadata.Build()):
             elif ext == 'gradle':
                 if not os.path.isfile(filepath):
                     continue
-                with open(filepath, 'r', encoding='utf8', errors='replace') as f:
+                with open(filepath, 'r', errors='replace') as f:
                     lines = f.readlines()
                 for i, line in enumerate(lines):
                     if is_used_by_gradle(line):

fdroidserver/tail.py

@@ -67,7 +67,7 @@ class Tail(object):
         Arguments:
             s - Number of seconds to wait between each iteration; Defaults to 1. '''
 
-        with open(self.tailed_file, encoding='utf8') as file_:
+        with open(self.tailed_file) as file_:
             # Go to the end of file
             file_.seek(0, 2)
             while not self.t_stop.is_set():

fdroidserver/update.py

@@ -695,7 +695,7 @@ def _get_localized_dict(app, locale):
 def _set_localized_text_entry(app, locale, key, f):
     limit = config['char_limits'][key]
     localized = _get_localized_dict(app, locale)
-    with open(f) as fp:
+    with open(f, errors='replace') as fp:
         text = fp.read()[:limit]
         if len(text) > 0:
             localized[key] = text
@@ -703,7 +703,7 @@ def _set_localized_text_entry(app, locale, key, f):
 
 def _set_author_entry(app, key, f):
     limit = config['char_limits']['author']
-    with open(f) as fp:
+    with open(f, errors='replace') as fp:
         text = fp.read()[:limit]
         if len(text) > 0:
             app[key] = text
@@ -1722,7 +1722,7 @@ def make_categories_txt(repodir, categories):
     catdata = ''
     for cat in sorted(categories):
         catdata += cat + '\n'
-    with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
+    with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
         f.write(catdata)
 
 
@@ -2065,7 +2065,7 @@ def main():
         # Generate latest apps data for widget
         if os.path.exists(os.path.join('stats', 'latestapps.txt')):
             data = ''
-            with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
+            with open(os.path.join('stats', 'latestapps.txt'), 'r') as f:
                 for line in f:
                     appid = line.rstrip()
                     data += appid + "\t"
@@ -2074,7 +2074,7 @@ def main():
                     if app.icon is not None:
                         data += app.icon + "\t"
                     data += app.License + "\n"
-            with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
+            with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
                 f.write(data)
 
     if cachechanged:

tests/common.TestCase

@@ -427,7 +427,7 @@ class CommonTest(unittest.TestCase):
     def test_write_to_config(self):
         with tempfile.TemporaryDirectory() as tmpPath:
             cfgPath = os.path.join(tmpPath, 'config.py')
-            with open(cfgPath, 'w', encoding='utf-8') as f:
+            with open(cfgPath, 'w') as f:
                 f.write(textwrap.dedent("""\
                     # abc
                     # test = 'example value'
@@ -445,7 +445,7 @@ class CommonTest(unittest.TestCase):
             fdroidserver.common.write_to_config(cfg, 'test', value='test value', config_file=cfgPath)
             fdroidserver.common.write_to_config(cfg, 'new_key', value='new', config_file=cfgPath)
 
-            with open(cfgPath, 'r', encoding='utf-8') as f:
+            with open(cfgPath, 'r') as f:
                 self.assertEqual(f.read(), textwrap.dedent("""\
                     # abc
                     test = 'test value'
@@ -465,7 +465,7 @@ class CommonTest(unittest.TestCase):
             with open(cfgPath, 'w') as f:
                 pass
             fdroidserver.common.write_to_config({}, 'key', 'val', cfgPath)
-            with open(cfgPath, 'r', encoding='utf-8') as f:
+            with open(cfgPath, 'r') as f:
                 self.assertEqual(f.read(), textwrap.dedent("""\
 
                 key = "val"

tests/metadata.TestCase

@@ -81,8 +81,8 @@ class MetadataTest(unittest.TestCase):
                 fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)
 
         # assert rewrite result
-        with open(os.path.join(testdir, 'fake.ota.update.yml'), 'r', encoding='utf-8') as result:
+        with open(os.path.join(testdir, 'fake.ota.update.yml'), 'r') as result:
-            with open('metadata-rewrite-yml/fake.ota.update.yml', 'r', encoding='utf-8') as orig:
+            with open('metadata-rewrite-yml/fake.ota.update.yml', 'r') as orig:
                 self.maxDiff = None
                 self.assertEqual(result.read(), orig.read())
 
@@ -97,8 +97,8 @@ class MetadataTest(unittest.TestCase):
                 fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)
 
         # assert rewrite result
-        with open(os.path.join(testdir, 'org.fdroid.fdroid.yml'), 'r', encoding='utf-8') as result:
+        with open(os.path.join(testdir, 'org.fdroid.fdroid.yml'), 'r') as result:
-            with open('metadata-rewrite-yml/org.fdroid.fdroid.yml', 'r', encoding='utf-8') as orig:
+            with open('metadata-rewrite-yml/org.fdroid.fdroid.yml', 'r') as orig:
                 self.maxDiff = None
                 self.assertEqual(result.read(), orig.read())
 
@@ -113,8 +113,8 @@ class MetadataTest(unittest.TestCase):
                 fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)
 
         # assert rewrite result
-        with open(os.path.join(testdir, 'app.with.special.build.params.yml'), 'r', encoding='utf-8') as result:
+        with open(os.path.join(testdir, 'app.with.special.build.params.yml'), 'r') as result:
-            with open('metadata-rewrite-yml/app.with.special.build.params.yml', 'r', encoding='utf-8') as orig:
+            with open('metadata-rewrite-yml/app.with.special.build.params.yml', 'r') as orig:
                 self.maxDiff = None
                 self.assertEqual(result.read(), orig.read())