diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 467f1cb1..1128b640 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,8 @@ image: mvdan/fdroid-ci:server-20160429 test: script: + - ls -l /usr/lib/python3* + - mkdir -p /usr/lib/python3.4/site-packages/ - pip3 install -e . - cd tests - ./complete-ci-tests diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 86937640..03a8743d 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -184,10 +184,10 @@ def fill_config_defaults(thisconfig): def regsub_file(pattern, repl, path): - with open(path, 'r') as f: + with open(path, 'rb') as f: text = f.read() - text = re.sub(pattern, repl, text) - with open(path, 'w') as f: + text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text) + with open(path, 'wb') as f: f.write(text) @@ -1017,7 +1017,7 @@ def get_library_references(root_dir): proppath = os.path.join(root_dir, 'project.properties') if not os.path.isfile(proppath): return libraries - with open(proppath, 'r') as f: + with open(proppath, 'r', encoding='iso-8859-1') as f: for line in f: if not line.startswith('android.library.reference.'): continue @@ -1356,7 +1356,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= props = "" if os.path.isfile(path): logging.info("Updating local.properties file at %s" % path) - with open(path, 'r') as f: + with open(path, 'r', encoding='iso-8859-1') as f: props += f.read() props += '\n' else: @@ -1378,7 +1378,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= # Add java.encoding if necessary if build.encoding: props += "java.encoding=%s\n" % build.encoding - with open(path, 'w') as f: + with open(path, 'w', encoding='iso-8859-1') as f: f.write(props) flavours = [] @@ -1539,7 +1539,7 @@ class KnownApks: self.path = os.path.join('stats', 'known_apks.txt') self.apks = {} if os.path.isfile(self.path): - with open(self.path, 'r') as f: + with open(self.path, 'r', encoding='utf8') as f: for line in f: t = line.rstrip().split(' ') if len(t) == 2: @@ -1563,7 +1563,7 @@ class KnownApks: line += ' ' + time.strftime('%Y-%m-%d', added) lst.append(line) - with open(self.path, 'w') as f: + with open(self.path, 'w', encoding='utf8') as f: for line in sorted(lst, key=natural_key): f.write(line + '\n') @@ -1724,14 +1724,14 @@ def remove_signing_keys(build_dir): if 'build.gradle' in files: path = os.path.join(root, 'build.gradle') - with open(path, "r") as o: + with open(path, "r", encoding='utf8') as o: lines = o.readlines() changed = False opened = 0 i = 0 - with open(path, "w") as o: + with open(path, "w", encoding='utf8') as o: while i < len(lines): line = lines[i] i += 1 @@ -1771,12 +1771,12 @@ def remove_signing_keys(build_dir): if propfile in files: path = os.path.join(root, propfile) - with open(path, "r") as o: + with open(path, "r", encoding='iso-8859-1') as o: lines = o.readlines() changed = False - with open(path, "w") as o: + with open(path, "w", encoding='iso-8859-1') as o: for line in lines: if any(line.startswith(s) for s in ('key.store', 'key.alias')): changed = True @@ -1838,10 +1838,10 @@ def place_srclib(root_dir, number, libpath): lines = [] if os.path.isfile(proppath): - with open(proppath, "r") as o: + with open(proppath, "r", encoding='iso-8859-1') as o: lines = o.readlines() - with open(proppath, "w") as o: + with open(proppath, "w", encoding='iso-8859-1') as o: placed = False for line in lines: if line.startswith('android.library.reference.%d=' % number): @@ -2009,7 +2009,7 @@ def write_to_config(thisconfig, key, value=None): if value is None: origkey = key + '_orig' value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key] - with open('config.py', 'r') as f: + with open('config.py', 'r', encoding='utf8') as f: data = f.read() pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"' repl = '\n' + key + ' = "' + value + '"' @@ -2020,7 +2020,7 @@ def write_to_config(thisconfig, key, value=None): # make sure the file ends with a carraige return if not re.match('\n$', data): data += '\n' - with open('config.py', 'w') as f: + with open('config.py', 'w', encoding='utf8') as f: f.writelines(data) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 4b301de9..caa77734 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -35,12 +35,12 @@ options = None def disable_in_config(key, value): '''write a key/value to the local config.py, then comment it out''' - with open('config.py', 'r') as f: + with open('config.py', 'r', encoding='utf8') as f: data = f.read() pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"' repl = '\n#' + key + ' = "' + value + '"' data = re.sub(pattern, repl, data) - with open('config.py', 'w') as f: + with open('config.py', 'w', encoding='utf8') as f: f.writelines(data) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 71da8453..ded6cd1a 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -836,7 +836,7 @@ def get_default_app_info(metadatapath=None): for root, dirs, files in os.walk(os.getcwd()): if 'build.gradle' in files: p = os.path.join(root, 'build.gradle') - with open(p) as f: + with open(p, 'rb') as f: data = f.read() m = pattern.search(data) if m: @@ -1384,7 +1384,7 @@ def write_metadata(metadatapath, app): raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % ( metadatapath, ', '.join(accepted))) - with open(metadatapath, 'w') as mf: + with open(metadatapath, 'w', encoding='utf8') as mf: if ext == 'txt': return write_txt(mf, app) elif ext == 'yml': diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 052deb4e..65a50841 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -33,7 +33,7 @@ def proper_format(app): s = io.StringIO() # TODO: currently reading entire file again, should reuse first # read in metadata.py - with open(app.metadatapath, 'r') as f: + with open(app.metadatapath, 'r', encoding='utf8') as f: cur_content = f.read() metadata.write_txt(s, app) content = s.getvalue() diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 53c938fb..41ecbb50 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -199,7 +199,7 @@ def scan_source(build_dir, root_dir, build): elif ext == 'java': if not os.path.isfile(fp): continue - with open(fp, 'r') as f: + with open(fp, 'r', encoding='utf8') as f: for line in f: if 'DexClassLoader' in line: count += handleproblem('DexClassLoader', fd, fp) @@ -208,7 +208,7 @@ def scan_source(build_dir, root_dir, build): elif ext == 'gradle': if not os.path.isfile(fp): continue - with open(fp, 'r') as f: + with open(fp, 'r', encoding='utf8') as f: lines = f.readlines() for i, line in enumerate(lines): if is_used_by_gradle(line): diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 80d8d8ab..ba7a3998 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1017,7 +1017,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories): catdata = '' for cat in categories: catdata += cat + '\n' - with open(os.path.join(repodir, 'categories.txt'), 'w') as f: + with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f: f.write(catdata) @@ -1229,7 +1229,7 @@ def main(): if 'name' not in apk: logging.error(apk['id'] + ' does not have a name! Skipping...') continue - f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w') + f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8') f.write("License:Unknown\n") f.write("Web Site:\n") f.write("Source Code:\n") @@ -1339,7 +1339,7 @@ def main(): # Generate latest apps data for widget if os.path.exists(os.path.join('stats', 'latestapps.txt')): data = '' - with open(os.path.join('stats', 'latestapps.txt'), 'r') as f: + with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f: for line in f: appid = line.rstrip() data += appid + "\t" @@ -1348,7 +1348,7 @@ def main(): if app.icon is not None: data += app.icon + "\t" data += app.License + "\n" - with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f: + with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f: f.write(data) if cachechanged: diff --git a/tests/build.TestCase b/tests/build.TestCase index b3b90fc3..369c3836 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -57,13 +57,13 @@ class BuildTest(unittest.TestCase): fdroidserver.build.config = {} fdroidserver.build.config['build_tools'] = teststring fdroidserver.build.adapt_gradle(testsdir) - pattern = re.compile("buildToolsVersion[\s=]+'%s'\s+" % teststring) + pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8')) for p in ('source-files/fdroid/fdroidclient/build.gradle', 'source-files/Zillode/syncthing-silk/build.gradle', 'source-files/open-keychain/open-keychain/build.gradle', 'source-files/osmandapp/osmand/build.gradle', 'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle'): - with open(os.path.join(testsdir, p), 'r') as f: + with open(os.path.join(testsdir, p), 'rb') as f: filedata = f.read() self.assertIsNotNone(pattern.search(filedata))