From 48c43546294d4a88f23b19946cc6ee9e54c663f5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 14 Jun 2021 15:38:37 +0200 Subject: [PATCH] always open Android source files as UTF-8 Android Studio recommends "you use UTF-8 encoding whenever possible", so this code assumes the files use UTF-8. UTF-8 is also the default encoding on GNU/Linux and macOS. https://sites.google.com/a/android.com/tools/knownissues/encoding Windows will probably default to UTF16, since that's the native encoding for files. So forcing things to use UTF-8 should help compatibility. --- fdroidserver/common.py | 8 ++++++-- fdroidserver/update.py | 2 +- tests/build.TestCase | 4 ++-- tests/common.TestCase | 24 +++++++++++++----------- tests/scanner.TestCase | 8 ++++---- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 236855b3..16a0f413 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1633,6 +1633,10 @@ def parse_androidmanifests(paths, app): Extract some information from the AndroidManifest.xml at the given path. Returns (version, vercode, package), any or all of which might be None. All values returned are strings. + + Android Studio recommends "you use UTF-8 encoding whenever possible", so + this code assumes the files use UTF-8. + https://sites.google.com/a/android.com/tools/knownissues/encoding """ ignoreversions = app.UpdateCheckIgnore @@ -1663,7 +1667,7 @@ def parse_androidmanifests(paths, app): flavour = app['Builds'][-1].gradle[-1] if path.endswith('.gradle') or path.endswith('.gradle.kts'): - with open(path, 'r') as f: + with open(path, 'r', encoding='utf-8') as f: android_plugin_file = False inside_flavour_group = 0 inside_required_flavour = 0 @@ -1864,7 +1868,7 @@ def get_gradle_subdir(build_dir, paths): if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)): for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')): for f in (path.parent / m.group(1)).glob('build.gradle*'): - with f.open() as fp: + with f.open(encoding='utf-8') as fp: for line in fp.readlines(): if ANDROID_PLUGIN_REGEX.match(line): return f.parent.relative_to(build_dir) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 1c3a1ed7..9e3df12a 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1021,7 +1021,7 @@ def copy_triple_t_store_metadata(apps): sg_list = glob.glob(os.path.join('build', packageName, 'settings.gradle*')) if sg_list: settings_gradle = sg_list[0] - with open(settings_gradle) as fp: + with open(settings_gradle, encoding='utf-8') as fp: data = fp.read() for matches in setting_gradle_pattern.findall(data): for m in matches: diff --git a/tests/build.TestCase b/tests/build.TestCase index c1b16e6b..7558237d 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -339,12 +339,12 @@ class BuildTest(unittest.TestCase): os.mkdir('build') os.mkdir('build/reports') - with open('build.gradle', 'w') as fp: + with open('build.gradle', 'w', encoding='utf-8') as fp: fp.write('// placeholder') os.mkdir('bin') os.mkdir('gen') - with open('build.xml', 'w') as fp: + with open('build.xml', 'w', encoding='utf-8') as fp: fp.write( textwrap.dedent( """ diff --git a/tests/common.TestCase b/tests/common.TestCase index dbded144..a9d215f4 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -315,14 +315,15 @@ class CommonTest(unittest.TestCase): fdroidserver.common.prepare_source(FakeVcs(), app, build, fdroidclient_testdir, fdroidclient_testdir, fdroidclient_testdir) - with open(os.path.join(fdroidclient_testdir, 'build.gradle'), 'r') as f: - filedata = f.read() + fdroidclient_testdir = Path(fdroidclient_testdir) + build_gradle = fdroidclient_testdir / 'build.gradle' + filedata = build_gradle.read_text(encoding='utf-8') self.assertIsNotNone( re.search(r"\s+compileSdkVersion %s\s+" % testint, filedata) ) - with open(os.path.join(fdroidclient_testdir, 'AndroidManifest.xml')) as f: - filedata = f.read() + androidmanifest_xml = fdroidclient_testdir / 'AndroidManifest.xml' + filedata = androidmanifest_xml.read_text(encoding='utf-8') self.assertIsNone(re.search('android:debuggable', filedata)) self.assertIsNotNone( re.search('android:versionName="%s"' % build.versionName, filedata) @@ -342,10 +343,10 @@ class CommonTest(unittest.TestCase): ) subdir = 'baz/bar' - subdir_path = os.path.join(app_build_dir, subdir) - os.makedirs(subdir_path) - with open(os.path.join(subdir_path, 'build.gradle'), 'w') as fp: - fp.write('// just a test placeholder') + subdir_path = Path(app_build_dir) / subdir + subdir_path.mkdir(parents=True, exist_ok=True) + build_gradle = subdir_path / 'build.gradle' + build_gradle.write_text('// just a test placeholder', encoding='utf-8') config = dict() fdroidserver.common.fill_config_defaults(config) @@ -921,7 +922,7 @@ class CommonTest(unittest.TestCase): self.assertNotEqual(0, len(files)) for f in files: appid, versionCode = os.path.splitext(os.path.basename(f))[0][12:].split('_') - with open(f) as fp: + with open(f, encoding='utf-8') as fp: m = fdroidserver.common.APK_ID_TRIPLET_REGEX.match(fp.read()) if m: self.assertEqual(appid, m.group(1)) @@ -962,6 +963,7 @@ class CommonTest(unittest.TestCase): def test_get_sdkversions_androguard(self): """This is a sanity test that androguard isn't broken""" + def get_minSdkVersion(apkfile): apk = fdroidserver.common._get_androguard_APK(apkfile) return fdroidserver.common.get_min_sdk_version(apk) @@ -1852,9 +1854,9 @@ class CommonTest(unittest.TestCase): self.assertEqual([], data['fdroiddata']['untrackedFiles']) dirtyfile = 'dirtyfile' - with open(dirtyfile, 'w') as fp: + with open(dirtyfile, 'w', encoding='utf-8') as fp: fp.write('this is just a test') - with open(file_in_git, 'a') as fp: + with open(file_in_git, 'a', encoding='utf-8') as fp: fp.write('\nappend some stuff') self.assertEqual([], data['fdroiddata']['modifiedFiles']) diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 428cca9c..3fd52694 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -76,7 +76,7 @@ class ScannerTest(unittest.TestCase): build = fdroidserver.metadata.Build() build.gradle = [flavor] regexs = fdroidserver.scanner.get_gradle_compile_commands(build) - with open(f) as fp: + with open(f, encoding='utf-8') as fp: for line in fp.readlines(): for regex in regexs: m = regex.match(line) @@ -93,7 +93,7 @@ class ScannerTest(unittest.TestCase): fdroidserver.scanner.config = None fdroidserver.scanner.options = mock.Mock() fdroidserver.scanner.options.json = True - with open('build.gradle', 'w') as fp: + with open('build.gradle', 'w', encoding='utf-8') as fp: fp.write( textwrap.dedent( """ @@ -233,7 +233,7 @@ class ScannerTest(unittest.TestCase): fp.write('placeholder') self.assertTrue(os.path.exists(f)) - with open('build.xml', 'w') as fp: + with open('build.xml', 'w', encoding='utf-8') as fp: fp.write( textwrap.dedent( """ @@ -288,7 +288,7 @@ class ScannerTest(unittest.TestCase): fdroidserver.scanner.options = mock.Mock() build = fdroidserver.metadata.Build() build.scandelete = ['build.gradle'] - with open('build.gradle', 'w') as fp: + with open('build.gradle', 'w', encoding='utf-8') as fp: fp.write( textwrap.dedent( """