Merge branch 'port-to-windows' into 'master'

some fixes to help port to Windows

See merge request fdroid/fdroidserver!678

.gitlab-ci.yml

@@ -434,6 +434,48 @@ Build documentation:
       - docs/build/html/
 
 
+Windows:
+  tags:
+    - windows
+  script:
+    - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+    - choco install --no-progress -y git --force --params "/GitAndUnixToolsOnPath"
+    - choco install --no-progress -y python3
+    - choco install --no-progress -y jdk8
+    - choco install --no-progress -y rsync
+    - refreshenv
+    - python -m pip install --upgrade babel pip setuptools
+    - python -m pip install -e .
+
+    - $files = @(Get-ChildItem tests\*.TestCase)
+    - foreach ($f in $files) {
+          write-output $f;
+          python $f;
+          if( $LASTEXITCODE -eq 0 ) {
+              write-output "SUCCESS $f";
+          } else {
+              write-output "ERROR $f failed";
+          }
+      }
+
+    # these are the tests that must pass
+    - python tests\checkupdates.TestCase
+    - python tests\exception.TestCase
+    - python tests\import.TestCase
+    - python tests\init.TestCase
+    - python tests\lint.TestCase
+    - python tests\main.TestCase
+    - python tests\metadata.TestCase
+    - python tests\rewritemeta.TestCase
+    - python tests\vcs.TestCase
+  after_script:
+    - Copy-Item C:\ProgramData\chocolatey\logs\chocolatey.log
+  artifacts:
+    when: always
+    paths:
+      - "*.log"
+
+
 pages:
   image: alpine:latest
   stage: deploy

fdroidserver/build.py

@@ -23,7 +23,6 @@ import glob
 import subprocess
 import posixpath
 import re
-import resource
 import sys
 import tarfile
 import threading
@@ -1026,17 +1025,22 @@ def main():
         raise FDroidException("No apps to process.")
 
     # make sure enough open files are allowed to process everything
-    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
-    if len(apps) > soft:
-        try:
-            soft = len(apps) * 2
-            if soft > hard:
-                soft = hard
-            resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
-            logging.debug(_('Set open file limit to {integer}')
-                          .format(integer=soft))
-        except (OSError, ValueError) as e:
-            logging.warning(_('Setting open file limit failed: ') + str(e))
+    try:
+        import resource  # not available on Windows
+
+        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if len(apps) > soft:
+            try:
+                soft = len(apps) * 2
+                if soft > hard:
+                    soft = hard
+                resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+                logging.debug(_('Set open file limit to {integer}')
+                              .format(integer=soft))
+            except (OSError, ValueError) as e:
+                logging.warning(_('Setting open file limit failed: ') + str(e))
+    except ImportError:
+        pass
 
     if options.latest:
         for app in apps.values():

fdroidserver/common.py

@@ -336,6 +336,12 @@ def read_config(opts=None):
     reading it.  config.py is deprecated and supported for backwards
     compatibility.
 
+    config.yml requires ASCII or UTF-8 encoding because this code does
+    not auto-detect the file's encoding.  That is left up to the YAML
+    library.  YAML allows ASCII, UTF-8, UTF-16, and UTF-32 encodings.
+    Since it is a good idea to manage config.yml (WITHOUT PASSWORDS!)
+    in git, it makes sense to use a globally standard encoding.
+
     """
     global config, options
 
@@ -354,7 +360,7 @@ def read_config(opts=None):
 
     if os.path.exists(config_file):
         logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
-        with open(config_file) as fp:
+        with open(config_file, encoding='utf-8') as fp:
             config = yaml.safe_load(fp)
     elif os.path.exists(old_config_file):
         logging.warning(_("""{oldfile} is deprecated, use {newfile}""")
@@ -557,9 +563,12 @@ def find_sdk_tools_cmd(cmd):
         sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
         if os.path.exists(sdk_platform_tools):
             tooldirs.append(sdk_platform_tools)
-    tooldirs.append('/usr/bin')
+    if os.path.exists('/usr/bin'):
+        tooldirs.append('/usr/bin')
     for d in tooldirs:
         path = os.path.join(d, cmd)
+        if not os.path.isfile(path):
+            path += '.exe'
         if os.path.isfile(path):
             if cmd == 'aapt':
                 test_aapt_version(path)
@@ -1630,6 +1639,10 @@ def parse_androidmanifests(paths, app):
     Extract some information from the AndroidManifest.xml at the given path.
     Returns (version, vercode, package), any or all of which might be None.
     All values returned are strings.
+
+    Android Studio recommends "you use UTF-8 encoding whenever possible", so
+    this code assumes the files use UTF-8.
+    https://sites.google.com/a/android.com/tools/knownissues/encoding
     """
 
     ignoreversions = app.UpdateCheckIgnore
@@ -1660,7 +1673,7 @@ def parse_androidmanifests(paths, app):
             flavour = app['Builds'][-1].gradle[-1]
 
         if path.endswith('.gradle') or path.endswith('.gradle.kts'):
-            with open(path, 'r') as f:
+            with open(path, 'r', encoding='utf-8') as f:
                 android_plugin_file = False
                 inside_flavour_group = 0
                 inside_required_flavour = 0
@@ -1859,9 +1872,9 @@ def get_gradle_subdir(build_dir, paths):
         if not first_gradle_dir:
             first_gradle_dir = path.parent.relative_to(build_dir)
         if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)):
-            for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text()):
+            for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')):
                 for f in (path.parent / m.group(1)).glob('build.gradle*'):
-                    with f.open() as fp:
+                    with f.open(encoding='utf-8') as fp:
                         for line in fp.readlines():
                             if ANDROID_PLUGIN_REGEX.match(line):
                                 return f.parent.relative_to(build_dir)
@@ -2403,7 +2416,7 @@ class KnownApks:
         self.path = os.path.join('stats', 'known_apks.txt')
         self.apks = {}
         if os.path.isfile(self.path):
-            with open(self.path, 'r') as f:
+            with open(self.path, 'r', encoding='utf-8') as f:
                 for line in f:
                     t = line.rstrip().split(' ')
                     if len(t) == 2:

fdroidserver/metadata.py

@@ -482,7 +482,7 @@ def parse_yaml_srclib(metadatapath):
             if type(data) is not dict:
                 if platform.system() == 'Windows':
                     # Handle symlink on Windows
-                    symlink = metadatapath.parent / metadatapath.read_text()
+                    symlink = metadatapath.parent / metadatapath.read_text(encoding='utf-8')
                     if symlink.is_file():
                         with symlink.open("r", encoding="utf-8") as s:
                             data = yaml.load(s, Loader=SafeLoader)
@@ -740,7 +740,7 @@ def parse_metadata(metadatapath):
         app.id = name
 
     if metadatapath.suffix == '.yml':
-        with metadatapath.open('r') as mf:
+        with metadatapath.open('r', encoding='utf-8') as mf:
             parse_yaml_metadata(mf, app)
     else:
         _warn_or_exception(_('Unknown metadata format: {path} (use: *.yml)')

fdroidserver/publish.py

@@ -88,10 +88,10 @@ def read_fingerprints_from_keystore():
     if p.returncode != 0:
         raise FDroidException('could not read keystore {}'.format(config['keystore']))
 
-    realias = re.compile('Alias name: (?P<alias>.+)\n')
-    resha256 = re.compile(r'\s+SHA256: (?P<sha256>[:0-9A-F]{95})\n')
+    realias = re.compile('Alias name: (?P<alias>.+)' + os.linesep)
+    resha256 = re.compile(r'\s+SHA256: (?P<sha256>[:0-9A-F]{95})' + os.linesep)
     fps = {}
-    for block in p.output.split(('*' * 43) + '\n' + '*' * 43):
+    for block in p.output.split(('*' * 43) + os.linesep + '*' * 43):
         s_alias = realias.search(block)
         s_sha256 = resha256.search(block)
         if s_alias and s_sha256:

fdroidserver/rewritemeta.py

@@ -36,7 +36,7 @@ def proper_format(app):
     s = io.StringIO()
     # TODO: currently reading entire file again, should reuse first
     # read in metadata.py
-    cur_content = Path(app.metadatapath).read_text()
+    cur_content = Path(app.metadatapath).read_text(encoding='utf-8')
     if Path(app.metadatapath).suffix == '.yml':
         metadata.write_yaml(s, app)
     content = s.getvalue()

fdroidserver/update.py

@@ -1021,7 +1021,7 @@ def copy_triple_t_store_metadata(apps):
         sg_list = glob.glob(os.path.join('build', packageName, 'settings.gradle*'))
         if sg_list:
             settings_gradle = sg_list[0]
-            with open(settings_gradle) as fp:
+            with open(settings_gradle, encoding='utf-8') as fp:
                 data = fp.read()
             for matches in setting_gradle_pattern.findall(data):
                 for m in matches:

tests/build.TestCase

@@ -284,7 +284,7 @@ class BuildTest(unittest.TestCase):
                     onserver=False,
                     refresh=False,
                 )
-            # now run `fdroid buid --onserver`
+            # now run `fdroid build --onserver`
             self.assertTrue('r21e' not in config['ndk_paths'])
             fdroidserver.build.build_local(
                 app,
@@ -339,12 +339,12 @@ class BuildTest(unittest.TestCase):
 
         os.mkdir('build')
         os.mkdir('build/reports')
-        with open('build.gradle', 'w') as fp:
+        with open('build.gradle', 'w', encoding='utf-8') as fp:
             fp.write('// placeholder')
 
         os.mkdir('bin')
         os.mkdir('gen')
-        with open('build.xml', 'w') as fp:
+        with open('build.xml', 'w', encoding='utf-8') as fp:
             fp.write(
                 textwrap.dedent(
                     """<?xml version="1.0" encoding="UTF-8" standalone="no"?>

tests/common.TestCase

@@ -315,14 +315,15 @@ class CommonTest(unittest.TestCase):
         fdroidserver.common.prepare_source(FakeVcs(), app, build,
                                            fdroidclient_testdir, fdroidclient_testdir, fdroidclient_testdir)
 
-        with open(os.path.join(fdroidclient_testdir, 'build.gradle'), 'r') as f:
-            filedata = f.read()
+        fdroidclient_testdir = Path(fdroidclient_testdir)
+        build_gradle = fdroidclient_testdir / 'build.gradle'
+        filedata = build_gradle.read_text(encoding='utf-8')
         self.assertIsNotNone(
             re.search(r"\s+compileSdkVersion %s\s+" % testint, filedata)
         )
 
-        with open(os.path.join(fdroidclient_testdir, 'AndroidManifest.xml')) as f:
-            filedata = f.read()
+        androidmanifest_xml = fdroidclient_testdir / 'AndroidManifest.xml'
+        filedata = androidmanifest_xml.read_text(encoding='utf-8')
         self.assertIsNone(re.search('android:debuggable', filedata))
         self.assertIsNotNone(
             re.search('android:versionName="%s"' % build.versionName, filedata)
@@ -331,6 +332,7 @@ class CommonTest(unittest.TestCase):
             re.search('android:versionCode="%s"' % build.versionCode, filedata)
         )
 
+    @unittest.skipIf(os.name == 'nt', "`fdroid build` assumes POSIX scripting")
     def test_prepare_sources_with_prebuild_subdir(self):
         testdir = tempfile.mkdtemp(
             prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
@@ -342,10 +344,10 @@ class CommonTest(unittest.TestCase):
         )
 
         subdir = 'baz/bar'
-        subdir_path = os.path.join(app_build_dir, subdir)
-        os.makedirs(subdir_path)
-        with open(os.path.join(subdir_path, 'build.gradle'), 'w') as fp:
-            fp.write('// just a test placeholder')
+        subdir_path = Path(app_build_dir) / subdir
+        subdir_path.mkdir(parents=True, exist_ok=True)
+        build_gradle = subdir_path / 'build.gradle'
+        build_gradle.write_text('// just a test placeholder', encoding='utf-8')
 
         config = dict()
         fdroidserver.common.fill_config_defaults(config)
@@ -921,7 +923,7 @@ class CommonTest(unittest.TestCase):
         self.assertNotEqual(0, len(files))
         for f in files:
             appid, versionCode = os.path.splitext(os.path.basename(f))[0][12:].split('_')
-            with open(f) as fp:
+            with open(f, encoding='utf-8') as fp:
                 m = fdroidserver.common.APK_ID_TRIPLET_REGEX.match(fp.read())
                 if m:
                     self.assertEqual(appid, m.group(1))
@@ -962,6 +964,7 @@ class CommonTest(unittest.TestCase):
 
     def test_get_sdkversions_androguard(self):
         """This is a sanity test that androguard isn't broken"""
+
         def get_minSdkVersion(apkfile):
             apk = fdroidserver.common._get_androguard_APK(apkfile)
             return fdroidserver.common.get_min_sdk_version(apk)
@@ -1685,6 +1688,34 @@ class CommonTest(unittest.TestCase):
         config = fdroidserver.common.read_config(fdroidserver.common.options)
         self.assertEqual('yml', config.get('apksigner'))
 
+    def test_with_config_yml_utf8(self):
+        """Make sure it is possible to use config.yml in UTF-8 encoding."""
+        testdir = tempfile.mkdtemp(
+            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
+        )
+        os.chdir(testdir)
+        teststr = '/πÇÇ现代通用字-български-عربي1/ö/yml'
+        with open('config.yml', 'w', encoding='utf-8') as fp:
+            fp.write('apksigner: ' + teststr)
+        self.assertTrue(os.path.exists('config.yml'))
+        self.assertFalse(os.path.exists('config.py'))
+        config = fdroidserver.common.read_config(fdroidserver.common.options)
+        self.assertEqual(teststr, config.get('apksigner'))
+
+    def test_with_config_yml_utf8_as_ascii(self):
+        """Make sure it is possible to use config.yml Unicode encoded as ASCII."""
+        testdir = tempfile.mkdtemp(
+            prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
+        )
+        os.chdir(testdir)
+        teststr = '/πÇÇ现代通用字-български-عربي1/ö/yml'
+        with open('config.yml', 'w') as fp:
+            yaml.dump({'apksigner': teststr}, fp)
+        self.assertTrue(os.path.exists('config.yml'))
+        self.assertFalse(os.path.exists('config.py'))
+        config = fdroidserver.common.read_config(fdroidserver.common.options)
+        self.assertEqual(teststr, config.get('apksigner'))
+
     def test_with_config_yml_with_env_var(self):
         """Make sure it is possible to use config.yml alone."""
         testdir = tempfile.mkdtemp(
@@ -1852,9 +1883,9 @@ class CommonTest(unittest.TestCase):
         self.assertEqual([],
                          data['fdroiddata']['untrackedFiles'])
         dirtyfile = 'dirtyfile'
-        with open(dirtyfile, 'w') as fp:
+        with open(dirtyfile, 'w', encoding='utf-8') as fp:
             fp.write('this is just a test')
-        with open(file_in_git, 'a') as fp:
+        with open(file_in_git, 'a', encoding='utf-8') as fp:
             fp.write('\nappend some stuff')
         self.assertEqual([],
                          data['fdroiddata']['modifiedFiles'])

tests/init.TestCase

@@ -54,6 +54,7 @@ class InitTest(unittest.TestCase):
         config = fdroidserver.common.read_config(fdroidserver.common.options)
         self.assertIsNone(config.get('keypass'))
 
+    @unittest.skipIf(os.name == 'nt', "calling main() like this hangs on Windows")
     def test_main_in_empty_dir(self):
         """Test that `fdroid init` will find apksigner and add it to the config"""
         testdir = tempfile.mkdtemp(

tests/metadata.TestCase

@@ -231,8 +231,8 @@ class MetadataTest(unittest.TestCase):
             self.maxDiff = None
             file_name = 'fake.ota.update.yml'
             self.assertEqual(
-                (testdir / file_name).read_text(),
-                (Path('metadata-rewrite-yml') / file_name).read_text(),
+                (testdir / file_name).read_text(encoding='utf-8'),
+                (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'),
             )
 
     def test_rewrite_yaml_fdroidclient(self):
@@ -253,8 +253,8 @@ class MetadataTest(unittest.TestCase):
             self.maxDiff = None
             file_name = 'org.fdroid.fdroid.yml'
             self.assertEqual(
-                (testdir / file_name).read_text(),
-                (Path('metadata-rewrite-yml') / file_name).read_text(),
+                (testdir / file_name).read_text(encoding='utf-8'),
+                (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'),
             )
 
     def test_rewrite_yaml_special_build_params(self):
@@ -274,8 +274,8 @@ class MetadataTest(unittest.TestCase):
             self.maxDiff = None
             file_name = 'app.with.special.build.params.yml'
             self.assertEqual(
-                (testdir / file_name).read_text(),
-                (Path('metadata-rewrite-yml') / file_name).read_text(),
+                (testdir / file_name).read_text(encoding='utf-8'),
+                (Path('metadata-rewrite-yml') / file_name).read_text(encoding='utf-8'),
             )
 
     def test_post_parse_yaml_metadata(self):
@@ -1084,6 +1084,32 @@ class MetadataTest(unittest.TestCase):
             },
         )
 
+    def test_build_ndk_path(self):
+        """"""
+        config = {'ndk_paths': {}, 'sdk_path': tempfile.mkdtemp(prefix='android-sdk-')}
+        fdroidserver.common.config = config
+
+        build = fdroidserver.metadata.Build()
+        build.ndk = 'r10e'
+        self.assertEqual('', build.ndk_path())
+
+        correct = '/fake/path/ndk/r21b'
+        config['ndk_paths'] = {'r21b': correct}
+        self.assertEqual('', build.ndk_path())
+        config['ndk_paths'] = {'r10e': correct}
+        self.assertEqual(correct, build.ndk_path())
+
+        r10e = '/fake/path/ndk/r10e'
+        r22b = '/fake/path/ndk/r22e'
+        config['ndk_paths'] = {'r10e': r10e, 'r22b': r22b}
+        self.assertEqual(r10e, build.ndk_path())
+
+        build.ndk = ['r10e', 'r22b']
+        self.assertEqual(r10e, build.ndk_path())
+
+        build.ndk = ['r22b', 'r10e']
+        self.assertEqual(r22b, build.ndk_path())
+
 
 if __name__ == "__main__":
     parser = optparse.OptionParser()

tests/rewritemeta.TestCase

@@ -48,7 +48,7 @@ class RewriteMetaTest(unittest.TestCase):
             rewritemeta.main()
 
             self.assertEqual(
-                Path('metadata/a.yml').read_text(),
+                Path('metadata/a.yml').read_text(encoding='utf-8'),
                 textwrap.dedent(
                     '''\
                     License: Unknown
@@ -62,7 +62,7 @@ class RewriteMetaTest(unittest.TestCase):
             )
 
             self.assertEqual(
-                Path('metadata/b.yml').read_text(),
+                Path('metadata/b.yml').read_text(encoding='utf-8'),
                 textwrap.dedent(
                     '''\
                     License: Unknown
@@ -90,7 +90,7 @@ class RewriteMetaTest(unittest.TestCase):
                     rewritemeta.main()
 
             self.assertEqual(
-                Path('metadata/a.yml').read_text(), 'AutoName: a'
+                Path('metadata/a.yml').read_text(encoding='utf-8'), 'AutoName: a'
             )
 
 

tests/scanner.TestCase

@@ -76,7 +76,7 @@ class ScannerTest(unittest.TestCase):
             build = fdroidserver.metadata.Build()
             build.gradle = [flavor]
             regexs = fdroidserver.scanner.get_gradle_compile_commands(build)
-            with open(f) as fp:
+            with open(f, encoding='utf-8') as fp:
                 for line in fp.readlines():
                     for regex in regexs:
                         m = regex.match(line)
@@ -93,7 +93,7 @@ class ScannerTest(unittest.TestCase):
         fdroidserver.scanner.config = None
         fdroidserver.scanner.options = mock.Mock()
         fdroidserver.scanner.options.json = True
-        with open('build.gradle', 'w') as fp:
+        with open('build.gradle', 'w', encoding='utf-8') as fp:
             fp.write(
                 textwrap.dedent(
                     """
@@ -233,7 +233,7 @@ class ScannerTest(unittest.TestCase):
                 fp.write('placeholder')
             self.assertTrue(os.path.exists(f))
 
-        with open('build.xml', 'w') as fp:
+        with open('build.xml', 'w', encoding='utf-8') as fp:
             fp.write(
                 textwrap.dedent(
                     """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
@@ -288,7 +288,7 @@ class ScannerTest(unittest.TestCase):
         fdroidserver.scanner.options = mock.Mock()
         build = fdroidserver.metadata.Build()
         build.scandelete = ['build.gradle']
-        with open('build.gradle', 'w') as fp:
+        with open('build.gradle', 'w', encoding='utf-8') as fp:
             fp.write(
                 textwrap.dedent(
                     """

tests/update.TestCase

@@ -411,7 +411,7 @@ class UpdateTest(unittest.TestCase):
 
     def javagetsig(self, apkfile):
         getsig_dir = 'getsig'
-        if not os.path.exists(getsig_dir + "/getsig.class"):
+        if not os.path.exists(os.path.join(getsig_dir, "getsig.class")):
             logging.critical("getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir)
             sys.exit(1)
         # FDroidPopen needs some config to work