diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index d90a4316..c07afb9d 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -21,7 +21,6 @@ import re import sys import os -import locale import pkgutil import logging @@ -198,7 +197,7 @@ def main(): else: mod = __import__(available_plugins[command]['name'], None, None, [command]) - system_langcode, system_encoding = locale.getdefaultlocale() + system_encoding = sys.getdefaultencoding() if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'): logging.warning(_("Encoding is set to '{enc}' fdroid might run " "into encoding issues. Please set it to 'UTF-8' " diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 32bfdb84..1437b47c 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -31,6 +31,7 @@ from distutils.version import LooseVersion import logging import copy import urllib.parse +from pathlib import Path from . import _ from . import common @@ -62,7 +63,7 @@ def check_http(app): raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode)) vercode = None - if len(urlcode) > 0: + if urlcode: logging.debug("...requesting {0}".format(urlcode)) req = urllib.request.Request(urlcode, None, headers=net.HEADERS) resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above @@ -74,7 +75,7 @@ def check_http(app): vercode = m.group(1).strip() version = "??" - if len(urlver) > 0: + if urlver: if urlver != '.': logging.debug("...requesting {0}".format(urlver)) req = urllib.request.Request(urlver, None) @@ -109,10 +110,10 @@ def check_tags(app, pattern): try: if app.RepoType == 'srclib': - build_dir = os.path.join('build', 'srclib', app.Repo) + build_dir = Path('build/srclib') / app.Repo repotype = common.getsrclibvcs(app.Repo) else: - build_dir = os.path.join('build', app.id) + build_dir = Path('build') / app.id repotype = app.RepoType if repotype not in ('git', 'git-svn', 'hg', 'bzr'): @@ -161,8 +162,8 @@ def check_tags(app, pattern): if app.UpdateCheckData: filecode, codeex, filever, verex = app.UpdateCheckData.split('|') vercode = None - if len(filecode) > 0: - filecontent = open(os.path.join(build_dir, filecode)).read() + if filecode: + filecontent = (build_dir / filecode).read_text() m = re.search(codeex, filecontent) if not m: @@ -170,9 +171,9 @@ def check_tags(app, pattern): vercode = m.group(1).strip() version = "??" - if len(filever) > 0: + if filever: if filever != '.': - filecontent = open(os.path.join(build_dir, filever)).read() + filecontent = (build_dir / filever).read_text() m = re.search(verex, filecontent) if not m: @@ -189,12 +190,9 @@ def check_tags(app, pattern): hver = version else: for subdir in possible_subdirs(app): - if subdir == '.': - root_dir = build_dir - else: - root_dir = os.path.join(build_dir, subdir) + root_dir = build_dir / subdir paths = common.manifest_paths(root_dir, last_build.gradle) - version, vercode, package = common.parse_androidmanifests(paths, app) + version, vercode, _package = common.parse_androidmanifests(paths, app) if vercode: logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" .format(subdir, version, vercode)) @@ -227,10 +225,10 @@ def check_repomanifest(app, branch=None): try: if app.RepoType == 'srclib': - build_dir = os.path.join('build', 'srclib', app.Repo) + build_dir = Path('build/srclib') / app.Repo repotype = common.getsrclibvcs(app.Repo) else: - build_dir = os.path.join('build', app.id) + build_dir = Path('build') / app.id repotype = app.RepoType # Set up vcs interface and make sure we have the latest code... @@ -248,7 +246,7 @@ def check_repomanifest(app, branch=None): vcs.gotorevision(None) last_build = metadata.Build() - if len(app.get('Builds', [])) > 0: + if app.get('Builds', []): last_build = app.get('Builds', [])[-1] try_init_submodules(app, last_build, vcs) @@ -257,10 +255,7 @@ def check_repomanifest(app, branch=None): hver = None hcode = "0" for subdir in possible_subdirs(app): - if subdir == '.': - root_dir = build_dir - else: - root_dir = os.path.join(build_dir, subdir) + root_dir = build_dir / subdir paths = common.manifest_paths(root_dir, last_build.gradle) version, vercode, package = common.parse_androidmanifests(paths, app) if vercode: @@ -290,10 +285,10 @@ def check_repotrunk(app): try: if app.RepoType == 'srclib': - build_dir = os.path.join('build', 'srclib', app.Repo) + build_dir = Path('build/srclib') / app.Repo repotype = common.getsrclibvcs(app.Repo) else: - build_dir = os.path.join('build', app.id) + build_dir = Path('build') / app.id repotype = app.RepoType if repotype not in ('git-svn', ): @@ -360,10 +355,11 @@ def try_init_submodules(app, last_build, vcs): # Return all directories under startdir that contain any of the manifest # files, and thus are probably an Android project. def dirs_with_manifest(startdir): - for root, dirs, files in os.walk(startdir): + # TODO: Python3.6: Accepts a path-like object. + for root, _dirs, files in os.walk(str(startdir)): if any(m in files for m in [ 'AndroidManifest.xml', 'pom.xml', 'build.gradle', 'build.gradle.kts']): - yield root + yield Path(root) # Tries to find a new subdir starting from the root build_dir. Returns said @@ -371,9 +367,9 @@ def dirs_with_manifest(startdir): def possible_subdirs(app): if app.RepoType == 'srclib': - build_dir = os.path.join('build', 'srclib', app.Repo) + build_dir = Path('build/srclib') / app.Repo else: - build_dir = os.path.join('build', app.id) + build_dir = Path('build') / app.id last_build = app.get_last_build() @@ -381,7 +377,7 @@ def possible_subdirs(app): m_paths = common.manifest_paths(d, last_build.gradle) package = common.parse_androidmanifests(m_paths, app)[2] if package is not None: - subdir = os.path.relpath(d, build_dir) + subdir = d.relative_to(build_dir) logging.debug("Adding possible subdir %s" % subdir) yield subdir @@ -401,9 +397,9 @@ def fetch_autoname(app, tag): return None if app.RepoType == 'srclib': - build_dir = os.path.join('build', 'srclib', app.Repo) + build_dir = Path('build/srclib') / app.Repo else: - build_dir = os.path.join('build', app.id) + build_dir = Path('build') / app.id try: vcs = common.getvcs(app.RepoType, app.Repo, build_dir) @@ -413,13 +409,10 @@ def fetch_autoname(app, tag): last_build = app.get_last_build() - logging.debug("...fetch auto name from " + build_dir) + logging.debug("...fetch auto name from " + str(build_dir)) new_name = None for subdir in possible_subdirs(app): - if subdir == '.': - root_dir = build_dir - else: - root_dir = os.path.join(build_dir, subdir) + root_dir = build_dir / subdir new_name = common.fetch_real_name(root_dir, last_build.gradle) if new_name is not None: break diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 63be8332..a0b858a0 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -51,6 +51,7 @@ import yaml import zipfile import tempfile import json +from pathlib import Path # TODO change to only import defusedxml once its installed everywhere try: @@ -119,13 +120,13 @@ orig_path = None default_config = { 'sdk_path': "$ANDROID_HOME", 'ndk_paths': {}, - 'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), + 'cachedir': str(Path.home() / '.cache/fdroidserver'), 'java_paths': None, 'scan_binary': False, 'ant': "ant", 'mvn3': "mvn", 'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'), - 'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'), + 'gradle_version_dir': str(Path.home() / '.cache/fdroidserver/gradle'), 'sync_from_local_copy_dir': False, 'allow_disabled_algorithms': False, 'per_app_repos': False, @@ -884,6 +885,8 @@ def setup_vcs(app): def getvcs(vcstype, remote, local): + # TODO: Remove this in Python3.6 + local = str(local) if vcstype == 'git': return vcs_git(remote, local) if vcstype == 'git-svn': @@ -1082,13 +1085,13 @@ class vcs_git(vcs): p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False) result = p.output.rstrip() - if not result.endswith(self.local): + if Path(result) != Path(self.local).resolve(): raise VCSException('Repository mismatch') def gotorevisionx(self, rev): if not os.path.exists(self.local): # Brand new checkout - p = self.git(['clone', '--', self.remote, self.local]) + p = self.git(['clone', '--', self.remote, str(self.local)]) if p.returncode != 0: self.clone_failed = True raise VCSException("Git clone failed", p.output) @@ -1197,7 +1200,7 @@ class vcs_gitsvn(vcs): """ p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False) result = p.output.rstrip() - if not result.endswith(self.local): + if Path(result) != Path(self.local).resolve(): raise VCSException('Repository mismatch') def git(self, args, envs=dict(), cwd=None, output=True): @@ -1258,7 +1261,7 @@ class vcs_gitsvn(vcs): raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ') .format(before=remote, after=location)) - gitsvn_args.extend(['--', remote, self.local]) + gitsvn_args.extend(['--', remote, str(self.local)]) p = self.git(gitsvn_args) if p.returncode != 0: self.clone_failed = True @@ -1358,7 +1361,7 @@ class vcs_hg(vcs): def gotorevisionx(self, rev): if not os.path.exists(self.local): - p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local], + p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, str(self.local)], output=False) if p.returncode != 0: self.clone_failed = True @@ -1416,7 +1419,7 @@ class vcs_bzr(vcs): def gotorevisionx(self, rev): if not os.path.exists(self.local): - p = self.bzr(['branch', self.remote, self.local], output=False) + p = self.bzr(['branch', self.remote, str(self.local)], output=False) if p.returncode != 0: self.clone_failed = True raise VCSException("Bzr branch failed", p.output) @@ -1494,6 +1497,8 @@ def retrieve_string_singleline(app_dir, string, xmlfiles=None): def manifest_paths(app_dir, flavours): '''Return list of existing files that will be used to find the highest vercode''' + # TODO: Remove this in Python3.6 + app_dir = str(app_dir) possible_manifests = \ [os.path.join(app_dir, 'AndroidManifest.xml'), os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'), @@ -1513,6 +1518,8 @@ def manifest_paths(app_dir, flavours): def fetch_real_name(app_dir, flavours): '''Retrieve the package name. Returns the name, or None if not found.''' + # TODO: Remove this in Python3.6 + app_dir = str(app_dir) for path in manifest_paths(app_dir, flavours): if not path.endswith('.xml') or not os.path.isfile(path): continue diff --git a/tests/checkupdates.TestCase b/tests/checkupdates.TestCase index 2f44f235..c431cae4 100755 --- a/tests/checkupdates.TestCase +++ b/tests/checkupdates.TestCase @@ -2,19 +2,17 @@ # http://www.drdobbs.com/testing/unit-testing-with-python/240165163 -import inspect import logging import optparse import os import sys import unittest from unittest import mock +from pathlib import Path -localmodule = os.path.realpath( - os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') -) -print('localmodule: ' + localmodule) +localmodule = Path(__file__).resolve().parent.parent +print('localmodule: ' + str(localmodule)) if localmodule not in sys.path: sys.path.insert(0, localmodule) @@ -28,11 +26,11 @@ class CheckupdatesTest(unittest.TestCase): def setUp(self): logging.basicConfig(level=logging.DEBUG) - self.basedir = os.path.join(localmodule, 'tests') - self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles')) - if not os.path.exists(self.tmpdir): - os.makedirs(self.tmpdir) - os.chdir(self.basedir) + self.basedir = localmodule / 'tests' + self.tmpdir = localmodule / '.testfiles' + self.tmpdir.mkdir(exist_ok=True) + # TODO: Python3.6: Accepts a path-like object. + os.chdir(str(self.basedir)) def test_autoupdatemode_no_suffix(self): fdroidserver.checkupdates.options = mock.Mock() @@ -52,7 +50,9 @@ class CheckupdatesTest(unittest.TestCase): build.versionName = app.CurrentVersion app['Builds'].append(build) - with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)): + with mock.patch( + 'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109) + ): with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()): with mock.patch('subprocess.call', lambda cmd: 0): fdroidserver.checkupdates.checkupdates_app(app) @@ -71,14 +71,16 @@ class CheckupdatesTest(unittest.TestCase): app.CurrentVersion = '1.1.8-fdroid' app.CurrentVersionCode = 10108 app.UpdateCheckMode = 'HTTP' - app.AutoUpdateMode = 'Version +.%c-fdroid v%v_%c' + app.AutoUpdateMode = r'Version +.%c-fdroid v%v_%c' build = fdroidserver.metadata.Build() build.versionCode = app.CurrentVersionCode build.versionName = app.CurrentVersion app['Builds'].append(build) - with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)): + with mock.patch( + 'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109) + ): with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()): with mock.patch('subprocess.call', lambda cmd: 0): fdroidserver.checkupdates.checkupdates_app(app) @@ -98,11 +100,17 @@ class CheckupdatesTest(unittest.TestCase): app.UpdateCheckMode = 'HTTP' app.UpdateCheckData = 'mock' - with mock.patch('fdroidserver.checkupdates.check_http', lambda app: (None, 'bla')): + with mock.patch( + 'fdroidserver.checkupdates.check_http', lambda app: (None, 'bla') + ): fdroidserver.checkupdates.checkupdates_app(app) - with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)): - with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()) as wrmock: + with mock.patch( + 'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109) + ): + with mock.patch( + 'fdroidserver.metadata.write_metadata', mock.Mock() + ) as wrmock: with mock.patch('subprocess.call', lambda cmd: 0): fdroidserver.checkupdates.checkupdates_app(app) wrmock.assert_called_with(app.metadatapath, app) @@ -125,14 +133,17 @@ class CheckupdatesTest(unittest.TestCase): build.versionName = app.CurrentVersion app['Builds'].append(build) - with mock.patch('fdroidserver.checkupdates.check_tags', lambda app, - pattern: (None, 'bla', None)): + with mock.patch( + 'fdroidserver.checkupdates.check_tags', + lambda app, pattern: (None, 'bla', None), + ): fdroidserver.checkupdates.checkupdates_app(app) - with mock.patch('fdroidserver.checkupdates.check_tags', lambda app, - pattern: ('1.1.9', 10109, 'v1.1.9')): - with mock.patch('fdroidserver.metadata.write_metadata', - mock.Mock()): + with mock.patch( + 'fdroidserver.checkupdates.check_tags', + lambda app, pattern: ('1.1.9', 10109, 'v1.1.9'), + ): + with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()): with mock.patch('subprocess.call', lambda cmd: 0): fdroidserver.checkupdates.checkupdates_app(app) build = app['Builds'][-1] @@ -200,16 +211,17 @@ class CheckupdatesTest(unittest.TestCase): vcs = mock.Mock() vcs.latesttags.return_value = ['1.1.8', '1.1.9'] with mock.patch( - 'builtins.open', mock.mock_open(read_data='v1.1.9\nc10109') + 'pathlib.Path.read_text', lambda a: 'v1.1.9\nc10109' ) as _ignored, mock.patch('fdroidserver.common.getvcs', return_value=vcs): _ignored # silence the linters - vername, vercode, tag = fdroidserver.checkupdates.check_tags(app, None) + vername, vercode, _tag = fdroidserver.checkupdates.check_tags(app, None) self.assertEqual(vername, '1.1.9') self.assertEqual(vercode, '10109') if __name__ == "__main__": - os.chdir(os.path.dirname(__file__)) + # TODO: Python3.6: Accept path-like object. + os.chdir(str(Path(__file__).parent)) parser = optparse.OptionParser() parser.add_option(