checkupdates.py: use pathlib and support Windows

fdroidserver/__main__.py

@@ -21,7 +21,6 @@
 import re
 import sys
 import os
-import locale
 import pkgutil
 import logging
 
@@ -198,7 +197,7 @@ def main():
     else:
         mod = __import__(available_plugins[command]['name'], None, None, [command])
 
-    system_langcode, system_encoding = locale.getdefaultlocale()
+    system_encoding = sys.getdefaultencoding()
     if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'):
         logging.warning(_("Encoding is set to '{enc}' fdroid might run "
                           "into encoding issues. Please set it to 'UTF-8' "

fdroidserver/checkupdates.py

@@ -31,6 +31,7 @@ from distutils.version import LooseVersion
 import logging
 import copy
 import urllib.parse
+from pathlib import Path
 
 from . import _
 from . import common
@@ -62,7 +63,7 @@ def check_http(app):
                 raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
 
         vercode = None
-        if len(urlcode) > 0:
+        if urlcode:
             logging.debug("...requesting {0}".format(urlcode))
             req = urllib.request.Request(urlcode, None, headers=net.HEADERS)
             resp = urllib.request.urlopen(req, None, 20)  # nosec B310 scheme is filtered above
@@ -74,7 +75,7 @@ def check_http(app):
             vercode = m.group(1).strip()
 
         version = "??"
-        if len(urlver) > 0:
+        if urlver:
             if urlver != '.':
                 logging.debug("...requesting {0}".format(urlver))
                 req = urllib.request.Request(urlver, None)
@@ -109,10 +110,10 @@ def check_tags(app, pattern):
     try:
 
         if app.RepoType == 'srclib':
-            build_dir = os.path.join('build', 'srclib', app.Repo)
+            build_dir = Path('build/srclib') / app.Repo
             repotype = common.getsrclibvcs(app.Repo)
         else:
-            build_dir = os.path.join('build', app.id)
+            build_dir = Path('build') / app.id
             repotype = app.RepoType
 
         if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
@@ -161,8 +162,8 @@ def check_tags(app, pattern):
             if app.UpdateCheckData:
                 filecode, codeex, filever, verex = app.UpdateCheckData.split('|')
                 vercode = None
-                if len(filecode) > 0:
+                if filecode:
-                    filecontent = open(os.path.join(build_dir, filecode)).read()
+                    filecontent = (build_dir / filecode).read_text()
 
                     m = re.search(codeex, filecontent)
                     if not m:
@@ -170,9 +171,9 @@ def check_tags(app, pattern):
                     vercode = m.group(1).strip()
 
                 version = "??"
-                if len(filever) > 0:
+                if filever:
                     if filever != '.':
-                        filecontent = open(os.path.join(build_dir, filever)).read()
+                        filecontent = (build_dir / filever).read_text()
 
                     m = re.search(verex, filecontent)
                     if not m:
@@ -189,12 +190,9 @@ def check_tags(app, pattern):
                             hver = version
             else:
                 for subdir in possible_subdirs(app):
-                    if subdir == '.':
+                    root_dir = build_dir / subdir
-                        root_dir = build_dir
-                    else:
-                        root_dir = os.path.join(build_dir, subdir)
                     paths = common.manifest_paths(root_dir, last_build.gradle)
-                    version, vercode, package = common.parse_androidmanifests(paths, app)
+                    version, vercode, _package = common.parse_androidmanifests(paths, app)
                     if vercode:
                         logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
                                       .format(subdir, version, vercode))
@@ -227,10 +225,10 @@ def check_repomanifest(app, branch=None):
     try:
 
         if app.RepoType == 'srclib':
-            build_dir = os.path.join('build', 'srclib', app.Repo)
+            build_dir = Path('build/srclib') / app.Repo
             repotype = common.getsrclibvcs(app.Repo)
         else:
-            build_dir = os.path.join('build', app.id)
+            build_dir = Path('build') / app.id
             repotype = app.RepoType
 
         # Set up vcs interface and make sure we have the latest code...
@@ -248,7 +246,7 @@ def check_repomanifest(app, branch=None):
             vcs.gotorevision(None)
 
         last_build = metadata.Build()
-        if len(app.get('Builds', [])) > 0:
+        if app.get('Builds', []):
             last_build = app.get('Builds', [])[-1]
 
         try_init_submodules(app, last_build, vcs)
@@ -257,10 +255,7 @@ def check_repomanifest(app, branch=None):
         hver = None
         hcode = "0"
         for subdir in possible_subdirs(app):
-            if subdir == '.':
+            root_dir = build_dir / subdir
-                root_dir = build_dir
-            else:
-                root_dir = os.path.join(build_dir, subdir)
             paths = common.manifest_paths(root_dir, last_build.gradle)
             version, vercode, package = common.parse_androidmanifests(paths, app)
             if vercode:
@@ -290,10 +285,10 @@ def check_repotrunk(app):
 
     try:
         if app.RepoType == 'srclib':
-            build_dir = os.path.join('build', 'srclib', app.Repo)
+            build_dir = Path('build/srclib') / app.Repo
             repotype = common.getsrclibvcs(app.Repo)
         else:
-            build_dir = os.path.join('build', app.id)
+            build_dir = Path('build') / app.id
             repotype = app.RepoType
 
         if repotype not in ('git-svn', ):
@@ -360,10 +355,11 @@ def try_init_submodules(app, last_build, vcs):
 # Return all directories under startdir that contain any of the manifest
 # files, and thus are probably an Android project.
 def dirs_with_manifest(startdir):
-    for root, dirs, files in os.walk(startdir):
+    # TODO: Python3.6: Accepts a path-like object.
+    for root, _dirs, files in os.walk(str(startdir)):
         if any(m in files for m in [
                 'AndroidManifest.xml', 'pom.xml', 'build.gradle', 'build.gradle.kts']):
-            yield root
+            yield Path(root)
 
 
 # Tries to find a new subdir starting from the root build_dir. Returns said
@@ -371,9 +367,9 @@ def dirs_with_manifest(startdir):
 def possible_subdirs(app):
 
     if app.RepoType == 'srclib':
-        build_dir = os.path.join('build', 'srclib', app.Repo)
+        build_dir = Path('build/srclib') / app.Repo
     else:
-        build_dir = os.path.join('build', app.id)
+        build_dir = Path('build') / app.id
 
     last_build = app.get_last_build()
 
@@ -381,7 +377,7 @@ def possible_subdirs(app):
         m_paths = common.manifest_paths(d, last_build.gradle)
         package = common.parse_androidmanifests(m_paths, app)[2]
         if package is not None:
-            subdir = os.path.relpath(d, build_dir)
+            subdir = d.relative_to(build_dir)
             logging.debug("Adding possible subdir %s" % subdir)
             yield subdir
 
@@ -401,9 +397,9 @@ def fetch_autoname(app, tag):
         return None
 
     if app.RepoType == 'srclib':
-        build_dir = os.path.join('build', 'srclib', app.Repo)
+        build_dir = Path('build/srclib') / app.Repo
     else:
-        build_dir = os.path.join('build', app.id)
+        build_dir = Path('build') / app.id
 
     try:
         vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
@@ -413,13 +409,10 @@ def fetch_autoname(app, tag):
 
     last_build = app.get_last_build()
 
-    logging.debug("...fetch auto name from " + build_dir)
+    logging.debug("...fetch auto name from " + str(build_dir))
     new_name = None
     for subdir in possible_subdirs(app):
-        if subdir == '.':
+        root_dir = build_dir / subdir
-            root_dir = build_dir
-        else:
-            root_dir = os.path.join(build_dir, subdir)
         new_name = common.fetch_real_name(root_dir, last_build.gradle)
         if new_name is not None:
             break

fdroidserver/common.py

@@ -51,6 +51,7 @@ import yaml
 import zipfile
 import tempfile
 import json
+from pathlib import Path
 
 # TODO change to only import defusedxml once its installed everywhere
 try:
@@ -119,13 +120,13 @@ orig_path = None
 default_config = {
     'sdk_path': "$ANDROID_HOME",
     'ndk_paths': {},
-    'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
+    'cachedir': str(Path.home() / '.cache/fdroidserver'),
     'java_paths': None,
     'scan_binary': False,
     'ant': "ant",
     'mvn3': "mvn",
     'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'),
-    'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'),
+    'gradle_version_dir': str(Path.home() / '.cache/fdroidserver/gradle'),
     'sync_from_local_copy_dir': False,
     'allow_disabled_algorithms': False,
     'per_app_repos': False,
@@ -884,6 +885,8 @@ def setup_vcs(app):
 
 
 def getvcs(vcstype, remote, local):
+    # TODO: Remove this in Python3.6
+    local = str(local)
     if vcstype == 'git':
         return vcs_git(remote, local)
     if vcstype == 'git-svn':
@@ -1082,13 +1085,13 @@ class vcs_git(vcs):
 
         p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
         result = p.output.rstrip()
-        if not result.endswith(self.local):
+        if Path(result) != Path(self.local).resolve():
             raise VCSException('Repository mismatch')
 
     def gotorevisionx(self, rev):
         if not os.path.exists(self.local):
             # Brand new checkout
-            p = self.git(['clone', '--', self.remote, self.local])
+            p = self.git(['clone', '--', self.remote, str(self.local)])
             if p.returncode != 0:
                 self.clone_failed = True
                 raise VCSException("Git clone failed", p.output)
@@ -1197,7 +1200,7 @@ class vcs_gitsvn(vcs):
         """
         p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
         result = p.output.rstrip()
-        if not result.endswith(self.local):
+        if Path(result) != Path(self.local).resolve():
             raise VCSException('Repository mismatch')
 
     def git(self, args, envs=dict(), cwd=None, output=True):
@@ -1258,7 +1261,7 @@ class vcs_gitsvn(vcs):
                 raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ')
                                    .format(before=remote, after=location))
 
-            gitsvn_args.extend(['--', remote, self.local])
+            gitsvn_args.extend(['--', remote, str(self.local)])
             p = self.git(gitsvn_args)
             if p.returncode != 0:
                 self.clone_failed = True
@@ -1358,7 +1361,7 @@ class vcs_hg(vcs):
 
     def gotorevisionx(self, rev):
         if not os.path.exists(self.local):
-            p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local],
+            p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, str(self.local)],
                             output=False)
             if p.returncode != 0:
                 self.clone_failed = True
@@ -1416,7 +1419,7 @@ class vcs_bzr(vcs):
 
     def gotorevisionx(self, rev):
         if not os.path.exists(self.local):
-            p = self.bzr(['branch', self.remote, self.local], output=False)
+            p = self.bzr(['branch', self.remote, str(self.local)], output=False)
             if p.returncode != 0:
                 self.clone_failed = True
                 raise VCSException("Bzr branch failed", p.output)
@@ -1494,6 +1497,8 @@ def retrieve_string_singleline(app_dir, string, xmlfiles=None):
 def manifest_paths(app_dir, flavours):
     '''Return list of existing files that will be used to find the highest vercode'''
 
+    # TODO: Remove this in Python3.6
+    app_dir = str(app_dir)
     possible_manifests = \
         [os.path.join(app_dir, 'AndroidManifest.xml'),
          os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
@@ -1513,6 +1518,8 @@ def manifest_paths(app_dir, flavours):
 
 def fetch_real_name(app_dir, flavours):
     '''Retrieve the package name. Returns the name, or None if not found.'''
+    # TODO: Remove this in Python3.6
+    app_dir = str(app_dir)
     for path in manifest_paths(app_dir, flavours):
         if not path.endswith('.xml') or not os.path.isfile(path):
             continue

tests/checkupdates.TestCase

@@ -2,19 +2,17 @@
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
-import inspect
 import logging
 import optparse
 import os
 import sys
 import unittest
 from unittest import mock
+from pathlib import Path
 
 
-localmodule = os.path.realpath(
+localmodule = Path(__file__).resolve().parent.parent
-    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
+print('localmodule: ' + str(localmodule))
-)
-print('localmodule: ' + localmodule)
 if localmodule not in sys.path:
     sys.path.insert(0, localmodule)
 
@@ -28,11 +26,11 @@ class CheckupdatesTest(unittest.TestCase):
 
     def setUp(self):
         logging.basicConfig(level=logging.DEBUG)
-        self.basedir = os.path.join(localmodule, 'tests')
+        self.basedir = localmodule / 'tests'
-        self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
+        self.tmpdir = localmodule / '.testfiles'
-        if not os.path.exists(self.tmpdir):
+        self.tmpdir.mkdir(exist_ok=True)
-            os.makedirs(self.tmpdir)
+        # TODO: Python3.6: Accepts a path-like object.
-        os.chdir(self.basedir)
+        os.chdir(str(self.basedir))
 
     def test_autoupdatemode_no_suffix(self):
         fdroidserver.checkupdates.options = mock.Mock()
@@ -52,7 +50,9 @@ class CheckupdatesTest(unittest.TestCase):
         build.versionName = app.CurrentVersion
         app['Builds'].append(build)
 
-        with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)):
+        with mock.patch(
+            'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
+        ):
             with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
                 with mock.patch('subprocess.call', lambda cmd: 0):
                     fdroidserver.checkupdates.checkupdates_app(app)
@@ -71,14 +71,16 @@ class CheckupdatesTest(unittest.TestCase):
         app.CurrentVersion = '1.1.8-fdroid'
         app.CurrentVersionCode = 10108
         app.UpdateCheckMode = 'HTTP'
-        app.AutoUpdateMode = 'Version +.%c-fdroid v%v_%c'
+        app.AutoUpdateMode = r'Version +.%c-fdroid v%v_%c'
 
         build = fdroidserver.metadata.Build()
         build.versionCode = app.CurrentVersionCode
         build.versionName = app.CurrentVersion
         app['Builds'].append(build)
 
-        with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)):
+        with mock.patch(
+            'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
+        ):
             with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
                 with mock.patch('subprocess.call', lambda cmd: 0):
                     fdroidserver.checkupdates.checkupdates_app(app)
@@ -98,11 +100,17 @@ class CheckupdatesTest(unittest.TestCase):
         app.UpdateCheckMode = 'HTTP'
         app.UpdateCheckData = 'mock'
 
-        with mock.patch('fdroidserver.checkupdates.check_http', lambda app: (None, 'bla')):
+        with mock.patch(
+            'fdroidserver.checkupdates.check_http', lambda app: (None, 'bla')
+        ):
             fdroidserver.checkupdates.checkupdates_app(app)
 
-        with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)):
+        with mock.patch(
-            with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()) as wrmock:
+            'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
+        ):
+            with mock.patch(
+                'fdroidserver.metadata.write_metadata', mock.Mock()
+            ) as wrmock:
                 with mock.patch('subprocess.call', lambda cmd: 0):
                     fdroidserver.checkupdates.checkupdates_app(app)
                 wrmock.assert_called_with(app.metadatapath, app)
@@ -125,14 +133,17 @@ class CheckupdatesTest(unittest.TestCase):
         build.versionName = app.CurrentVersion
         app['Builds'].append(build)
 
-        with mock.patch('fdroidserver.checkupdates.check_tags', lambda app,
+        with mock.patch(
-                        pattern: (None, 'bla', None)):
+            'fdroidserver.checkupdates.check_tags',
+            lambda app, pattern: (None, 'bla', None),
+        ):
             fdroidserver.checkupdates.checkupdates_app(app)
 
-        with mock.patch('fdroidserver.checkupdates.check_tags', lambda app,
+        with mock.patch(
-                        pattern: ('1.1.9', 10109, 'v1.1.9')):
+            'fdroidserver.checkupdates.check_tags',
-            with mock.patch('fdroidserver.metadata.write_metadata',
+            lambda app, pattern: ('1.1.9', 10109, 'v1.1.9'),
-                            mock.Mock()):
+        ):
+            with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
                 with mock.patch('subprocess.call', lambda cmd: 0):
                     fdroidserver.checkupdates.checkupdates_app(app)
                 build = app['Builds'][-1]
@@ -200,16 +211,17 @@ class CheckupdatesTest(unittest.TestCase):
         vcs = mock.Mock()
         vcs.latesttags.return_value = ['1.1.8', '1.1.9']
         with mock.patch(
-            'builtins.open', mock.mock_open(read_data='v1.1.9\nc10109')
+            'pathlib.Path.read_text', lambda a: 'v1.1.9\nc10109'
         ) as _ignored, mock.patch('fdroidserver.common.getvcs', return_value=vcs):
             _ignored  # silence the linters
-            vername, vercode, tag = fdroidserver.checkupdates.check_tags(app, None)
+            vername, vercode, _tag = fdroidserver.checkupdates.check_tags(app, None)
             self.assertEqual(vername, '1.1.9')
             self.assertEqual(vercode, '10109')
 
 
 if __name__ == "__main__":
-    os.chdir(os.path.dirname(__file__))
+    # TODO: Python3.6: Accept path-like object.
+    os.chdir(str(Path(__file__).parent))
 
     parser = optparse.OptionParser()
     parser.add_option(