diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8aaef2ed..a88f8867 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -434,6 +434,48 @@ Build documentation: - docs/build/html/ +Windows: + tags: + - windows + script: + - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" + - choco install --no-progress -y git --force --params "/GitAndUnixToolsOnPath" + - choco install --no-progress -y python3 + - choco install --no-progress -y jdk8 + - choco install --no-progress -y rsync + - refreshenv + - python -m pip install --upgrade babel pip setuptools + - python -m pip install -e . + + - $files = @(Get-ChildItem tests\*.TestCase) + - foreach ($f in $files) { + write-output $f; + python $f; + if( $LASTEXITCODE -eq 0 ) { + write-output "SUCCESS $f"; + } else { + write-output "ERROR $f failed"; + } + } + + # these are the tests that must pass + - python tests\checkupdates.TestCase + - python tests\exception.TestCase + - python tests\import.TestCase + - python tests\init.TestCase + - python tests\lint.TestCase + - python tests\main.TestCase + - python tests\metadata.TestCase + - python tests\rewritemeta.TestCase + - python tests\vcs.TestCase + after_script: + - Copy-Item C:\ProgramData\chocolatey\logs\chocolatey.log + artifacts: + when: always + paths: + - "*.log" + + pages: image: alpine:latest stage: deploy diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 7c4d900b..89097166 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -23,7 +23,6 @@ import glob import subprocess import posixpath import re -import resource import sys import tarfile import threading @@ -1026,17 +1025,22 @@ def main(): raise FDroidException("No apps to process.") # make sure enough open files are allowed to process everything - soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) - if len(apps) > soft: - try: - soft = len(apps) * 2 - if soft > hard: - soft = hard - resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) - logging.debug(_('Set open file limit to {integer}') - .format(integer=soft)) - except (OSError, ValueError) as e: - logging.warning(_('Setting open file limit failed: ') + str(e)) + try: + import resource # not available on Windows + + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if len(apps) > soft: + try: + soft = len(apps) * 2 + if soft > hard: + soft = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + logging.debug(_('Set open file limit to {integer}') + .format(integer=soft)) + except (OSError, ValueError) as e: + logging.warning(_('Setting open file limit failed: ') + str(e)) + except ImportError: + pass if options.latest: for app in apps.values(): diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 09b66c44..423acc49 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -336,6 +336,12 @@ def read_config(opts=None): reading it. config.py is deprecated and supported for backwards compatibility. + config.yml requires ASCII or UTF-8 encoding because this code does + not auto-detect the file's encoding. That is left up to the YAML + library. YAML allows ASCII, UTF-8, UTF-16, and UTF-32 encodings. + Since it is a good idea to manage config.yml (WITHOUT PASSWORDS!) + in git, it makes sense to use a globally standard encoding. + """ global config, options @@ -354,7 +360,7 @@ def read_config(opts=None): if os.path.exists(config_file): logging.debug(_("Reading '{config_file}'").format(config_file=config_file)) - with open(config_file) as fp: + with open(config_file, encoding='utf-8') as fp: config = yaml.safe_load(fp) elif os.path.exists(old_config_file): logging.warning(_("""{oldfile} is deprecated, use {newfile}""") @@ -557,9 +563,12 @@ def find_sdk_tools_cmd(cmd): sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools') if os.path.exists(sdk_platform_tools): tooldirs.append(sdk_platform_tools) - tooldirs.append('/usr/bin') + if os.path.exists('/usr/bin'): + tooldirs.append('/usr/bin') for d in tooldirs: path = os.path.join(d, cmd) + if not os.path.isfile(path): + path += '.exe' if os.path.isfile(path): if cmd == 'aapt': test_aapt_version(path) @@ -1630,6 +1639,10 @@ def parse_androidmanifests(paths, app): Extract some information from the AndroidManifest.xml at the given path. Returns (version, vercode, package), any or all of which might be None. All values returned are strings. + + Android Studio recommends "you use UTF-8 encoding whenever possible", so + this code assumes the files use UTF-8. + https://sites.google.com/a/android.com/tools/knownissues/encoding """ ignoreversions = app.UpdateCheckIgnore @@ -1660,7 +1673,7 @@ def parse_androidmanifests(paths, app): flavour = app['Builds'][-1].gradle[-1] if path.endswith('.gradle') or path.endswith('.gradle.kts'): - with open(path, 'r') as f: + with open(path, 'r', encoding='utf-8') as f: android_plugin_file = False inside_flavour_group = 0 inside_required_flavour = 0 @@ -1859,9 +1872,9 @@ def get_gradle_subdir(build_dir, paths): if not first_gradle_dir: first_gradle_dir = path.parent.relative_to(build_dir) if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)): - for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text()): + for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')): for f in (path.parent / m.group(1)).glob('build.gradle*'): - with f.open() as fp: + with f.open(encoding='utf-8') as fp: for line in fp.readlines(): if ANDROID_PLUGIN_REGEX.match(line): return f.parent.relative_to(build_dir) @@ -2403,7 +2416,7 @@ class KnownApks: self.path = os.path.join('stats', 'known_apks.txt') self.apks = {} if os.path.isfile(self.path): - with open(self.path, 'r') as f: + with open(self.path, 'r', encoding='utf-8') as f: for line in f: t = line.rstrip().split(' ') if len(t) == 2: diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index b3dc1fb2..59b15472 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -482,7 +482,7 @@ def parse_yaml_srclib(metadatapath): if type(data) is not dict: if platform.system() == 'Windows': # Handle symlink on Windows - symlink = metadatapath.parent / metadatapath.read_text() + symlink = metadatapath.parent / metadatapath.read_text(encoding='utf-8') if symlink.is_file(): with symlink.open("r", encoding="utf-8") as s: data = yaml.load(s, Loader=SafeLoader) @@ -740,7 +740,7 @@ def parse_metadata(metadatapath): app.id = name if metadatapath.suffix == '.yml': - with metadatapath.open('r') as mf: + with metadatapath.open('r', encoding='utf-8') as mf: parse_yaml_metadata(mf, app) else: _warn_or_exception(_('Unknown metadata format: {path} (use: *.yml)') diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 56b6016c..deb954ce 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -88,10 +88,10 @@ def read_fingerprints_from_keystore(): if p.returncode != 0: raise FDroidException('could not read keystore {}'.format(config['keystore'])) - realias = re.compile('Alias name: (?P.+)\n') - resha256 = re.compile(r'\s+SHA256: (?P[:0-9A-F]{95})\n') + realias = re.compile('Alias name: (?P.+)' + os.linesep) + resha256 = re.compile(r'\s+SHA256: (?P[:0-9A-F]{95})' + os.linesep) fps = {} - for block in p.output.split(('*' * 43) + '\n' + '*' * 43): + for block in p.output.split(('*' * 43) + os.linesep + '*' * 43): s_alias = realias.search(block) s_sha256 = resha256.search(block) if s_alias and s_sha256: diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 172359b8..a6406cb9 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -36,7 +36,7 @@ def proper_format(app): s = io.StringIO() # TODO: currently reading entire file again, should reuse first # read in metadata.py - cur_content = Path(app.metadatapath).read_text() + cur_content = Path(app.metadatapath).read_text(encoding='utf-8') if Path(app.metadatapath).suffix == '.yml': metadata.write_yaml(s, app) content = s.getvalue() diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 1c3a1ed7..9e3df12a 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1021,7 +1021,7 @@ def copy_triple_t_store_metadata(apps): sg_list = glob.glob(os.path.join('build', packageName, 'settings.gradle*')) if sg_list: settings_gradle = sg_list[0] - with open(settings_gradle) as fp: + with open(settings_gradle, encoding='utf-8') as fp: data = fp.read() for matches in setting_gradle_pattern.findall(data): for m in matches: diff --git a/tests/build.TestCase b/tests/build.TestCase index 33ad0f90..7558237d 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -284,7 +284,7 @@ class BuildTest(unittest.TestCase): onserver=False, refresh=False, ) - # now run `fdroid buid --onserver` + # now run `fdroid build --onserver` self.assertTrue('r21e' not in config['ndk_paths']) fdroidserver.build.build_local( app, @@ -339,12 +339,12 @@ class BuildTest(unittest.TestCase): os.mkdir('build') os.mkdir('build/reports') - with open('build.gradle', 'w') as fp: + with open('build.gradle', 'w', encoding='utf-8') as fp: fp.write('// placeholder') os.mkdir('bin') os.mkdir('gen') - with open('build.xml', 'w') as fp: + with open('build.xml', 'w', encoding='utf-8') as fp: fp.write( textwrap.dedent( """ diff --git a/tests/common.TestCase b/tests/common.TestCase index dbded144..2619d4b7 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -315,14 +315,15 @@ class CommonTest(unittest.TestCase): fdroidserver.common.prepare_source(FakeVcs(), app, build, fdroidclient_testdir, fdroidclient_testdir, fdroidclient_testdir) - with open(os.path.join(fdroidclient_testdir, 'build.gradle'), 'r') as f: - filedata = f.read() + fdroidclient_testdir = Path(fdroidclient_testdir) + build_gradle = fdroidclient_testdir / 'build.gradle' + filedata = build_gradle.read_text(encoding='utf-8') self.assertIsNotNone( re.search(r"\s+compileSdkVersion %s\s+" % testint, filedata) ) - with open(os.path.join(fdroidclient_testdir, 'AndroidManifest.xml')) as f: - filedata = f.read() + androidmanifest_xml = fdroidclient_testdir / 'AndroidManifest.xml' + filedata = androidmanifest_xml.read_text(encoding='utf-8') self.assertIsNone(re.search('android:debuggable', filedata)) self.assertIsNotNone( re.search('android:versionName="%s"' % build.versionName, filedata) @@ -331,6 +332,7 @@ class CommonTest(unittest.TestCase): re.search('android:versionCode="%s"' % build.versionCode, filedata) ) + @unittest.skipIf(os.name == 'nt', "`fdroid build` assumes POSIX scripting") def test_prepare_sources_with_prebuild_subdir(self): testdir = tempfile.mkdtemp( prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir @@ -342,10 +344,10 @@ class CommonTest(unittest.TestCase): ) subdir = 'baz/bar' - subdir_path = os.path.join(app_build_dir, subdir) - os.makedirs(subdir_path) - with open(os.path.join(subdir_path, 'build.gradle'), 'w') as fp: - fp.write('// just a test placeholder') + subdir_path = Path(app_build_dir) / subdir + subdir_path.mkdir(parents=True, exist_ok=True) + build_gradle = subdir_path / 'build.gradle' + build_gradle.write_text('// just a test placeholder', encoding='utf-8') config = dict() fdroidserver.common.fill_config_defaults(config) @@ -921,7 +923,7 @@ class CommonTest(unittest.TestCase): self.assertNotEqual(0, len(files)) for f in files: appid, versionCode = os.path.splitext(os.path.basename(f))[0][12:].split('_') - with open(f) as fp: + with open(f, encoding='utf-8') as fp: m = fdroidserver.common.APK_ID_TRIPLET_REGEX.match(fp.read()) if m: self.assertEqual(appid, m.group(1)) @@ -962,6 +964,7 @@ class CommonTest(unittest.TestCase): def test_get_sdkversions_androguard(self): """This is a sanity test that androguard isn't broken""" + def get_minSdkVersion(apkfile): apk = fdroidserver.common._get_androguard_APK(apkfile) return fdroidserver.common.get_min_sdk_version(apk) @@ -1685,6 +1688,34 @@ class CommonTest(unittest.TestCase): config = fdroidserver.common.read_config(fdroidserver.common.options) self.assertEqual('yml', config.get('apksigner')) + def test_with_config_yml_utf8(self): + """Make sure it is possible to use config.yml in UTF-8 encoding.""" + testdir = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) + os.chdir(testdir) + teststr = '/πÇÇ现代通用字-български-عربي1/ö/yml' + with open('config.yml', 'w', encoding='utf-8') as fp: + fp.write('apksigner: ' + teststr) + self.assertTrue(os.path.exists('config.yml')) + self.assertFalse(os.path.exists('config.py')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual(teststr, config.get('apksigner')) + + def test_with_config_yml_utf8_as_ascii(self): + """Make sure it is possible to use config.yml Unicode encoded as ASCII.""" + testdir = tempfile.mkdtemp( + prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir + ) + os.chdir(testdir) + teststr = '/πÇÇ现代通用字-български-عربي1/ö/yml' + with open('config.yml', 'w') as fp: + yaml.dump({'apksigner': teststr}, fp) + self.assertTrue(os.path.exists('config.yml')) + self.assertFalse(os.path.exists('config.py')) + config = fdroidserver.common.read_config(fdroidserver.common.options) + self.assertEqual(teststr, config.get('apksigner')) + def test_with_config_yml_with_env_var(self): """Make sure it is possible to use config.yml alone.""" testdir = tempfile.mkdtemp( @@ -1852,9 +1883,9 @@ class CommonTest(unittest.TestCase): self.assertEqual([], data['fdroiddata']['untrackedFiles']) dirtyfile = 'dirtyfile' - with open(dirtyfile, 'w') as fp: + with open(dirtyfile, 'w', encoding='utf-8') as fp: fp.write('this is just a test') - with open(file_in_git, 'a') as fp: + with open(file_in_git, 'a', encoding='utf-8') as fp: fp.write('\nappend some stuff') self.assertEqual([], data['fdroiddata']['modifiedFiles']) diff --git a/tests/init.TestCase b/tests/init.TestCase index da59e1a0..f19c59e5 100755 --- a/tests/init.TestCase +++ b/tests/init.TestCase @@ -54,6 +54,7 @@ class InitTest(unittest.TestCase): config = fdroidserver.common.read_config(fdroidserver.common.options) self.assertIsNone(config.get('keypass')) + @unittest.skipIf(os.name == 'nt', "calling main() like this hangs on Windows") def test_main_in_empty_dir(self): """Test that `fdroid init` will find apksigner and add it to the config""" testdir = tempfile.mkdtemp( diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 297271ab..07d8edcf 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -231,8 +231,8 @@ class MetadataTest(unittest.TestCase): self.maxDiff = None file_name = 'fake.ota.update.yml' self.assertEqual( - (testdir / file_name).read_text(), - (Path('metadata-rewrite-yml') / file_name).read_text(), + (testdir / file_name).read_text(encoding='utf-8'), + (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) def test_rewrite_yaml_fdroidclient(self): @@ -253,8 +253,8 @@ class MetadataTest(unittest.TestCase): self.maxDiff = None file_name = 'org.fdroid.fdroid.yml' self.assertEqual( - (testdir / file_name).read_text(), - (Path('metadata-rewrite-yml') / file_name).read_text(), + (testdir / file_name).read_text(encoding='utf-8'), + (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) def test_rewrite_yaml_special_build_params(self): @@ -274,8 +274,8 @@ class MetadataTest(unittest.TestCase): self.maxDiff = None file_name = 'app.with.special.build.params.yml' self.assertEqual( - (testdir / file_name).read_text(), - (Path('metadata-rewrite-yml') / file_name).read_text(), + (testdir / file_name).read_text(encoding='utf-8'), + (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'), ) def test_post_parse_yaml_metadata(self): @@ -1084,6 +1084,32 @@ class MetadataTest(unittest.TestCase): }, ) + def test_build_ndk_path(self): + """""" + config = {'ndk_paths': {}, 'sdk_path': tempfile.mkdtemp(prefix='android-sdk-')} + fdroidserver.common.config = config + + build = fdroidserver.metadata.Build() + build.ndk = 'r10e' + self.assertEqual('', build.ndk_path()) + + correct = '/fake/path/ndk/r21b' + config['ndk_paths'] = {'r21b': correct} + self.assertEqual('', build.ndk_path()) + config['ndk_paths'] = {'r10e': correct} + self.assertEqual(correct, build.ndk_path()) + + r10e = '/fake/path/ndk/r10e' + r22b = '/fake/path/ndk/r22e' + config['ndk_paths'] = {'r10e': r10e, 'r22b': r22b} + self.assertEqual(r10e, build.ndk_path()) + + build.ndk = ['r10e', 'r22b'] + self.assertEqual(r10e, build.ndk_path()) + + build.ndk = ['r22b', 'r10e'] + self.assertEqual(r22b, build.ndk_path()) + if __name__ == "__main__": parser = optparse.OptionParser() diff --git a/tests/rewritemeta.TestCase b/tests/rewritemeta.TestCase index 1271f9d8..309e2f45 100755 --- a/tests/rewritemeta.TestCase +++ b/tests/rewritemeta.TestCase @@ -48,7 +48,7 @@ class RewriteMetaTest(unittest.TestCase): rewritemeta.main() self.assertEqual( - Path('metadata/a.yml').read_text(), + Path('metadata/a.yml').read_text(encoding='utf-8'), textwrap.dedent( '''\ License: Unknown @@ -62,7 +62,7 @@ class RewriteMetaTest(unittest.TestCase): ) self.assertEqual( - Path('metadata/b.yml').read_text(), + Path('metadata/b.yml').read_text(encoding='utf-8'), textwrap.dedent( '''\ License: Unknown @@ -90,7 +90,7 @@ class RewriteMetaTest(unittest.TestCase): rewritemeta.main() self.assertEqual( - Path('metadata/a.yml').read_text(), 'AutoName: a' + Path('metadata/a.yml').read_text(encoding='utf-8'), 'AutoName: a' ) diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 428cca9c..3fd52694 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -76,7 +76,7 @@ class ScannerTest(unittest.TestCase): build = fdroidserver.metadata.Build() build.gradle = [flavor] regexs = fdroidserver.scanner.get_gradle_compile_commands(build) - with open(f) as fp: + with open(f, encoding='utf-8') as fp: for line in fp.readlines(): for regex in regexs: m = regex.match(line) @@ -93,7 +93,7 @@ class ScannerTest(unittest.TestCase): fdroidserver.scanner.config = None fdroidserver.scanner.options = mock.Mock() fdroidserver.scanner.options.json = True - with open('build.gradle', 'w') as fp: + with open('build.gradle', 'w', encoding='utf-8') as fp: fp.write( textwrap.dedent( """ @@ -233,7 +233,7 @@ class ScannerTest(unittest.TestCase): fp.write('placeholder') self.assertTrue(os.path.exists(f)) - with open('build.xml', 'w') as fp: + with open('build.xml', 'w', encoding='utf-8') as fp: fp.write( textwrap.dedent( """ @@ -288,7 +288,7 @@ class ScannerTest(unittest.TestCase): fdroidserver.scanner.options = mock.Mock() build = fdroidserver.metadata.Build() build.scandelete = ['build.gradle'] - with open('build.gradle', 'w') as fp: + with open('build.gradle', 'w', encoding='utf-8') as fp: fp.write( textwrap.dedent( """ diff --git a/tests/update.TestCase b/tests/update.TestCase index 2d4fd6ca..04bf42ca 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -411,7 +411,7 @@ class UpdateTest(unittest.TestCase): def javagetsig(self, apkfile): getsig_dir = 'getsig' - if not os.path.exists(getsig_dir + "/getsig.class"): + if not os.path.exists(os.path.join(getsig_dir, "getsig.class")): logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir) sys.exit(1) # FDroidPopen needs some config to work