diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a0b858a0..44dd32f9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -862,10 +862,8 @@ def write_status_json(output, pretty=False, name=None): def get_head_commit_id(git_repo): """Get git commit ID for HEAD as a str - - repo.head.commit.binsha is a bytearray stored in a str """ - return hexlify(bytearray(git_repo.head.commit.binsha)).decode() + return git_repo.head.commit.hexsha def setup_vcs(app): @@ -914,6 +912,8 @@ class vcs: def __init__(self, remote, local): + # TODO: Remove this in Python3.6 + local = str(local) # svn, git-svn and bzr may require auth self.username = None if self.repotype() in ('git-svn', 'bzr'): @@ -1611,7 +1611,8 @@ def parse_androidmanifests(paths, app): max_package = None for path in paths: - + # TODO: Remove this in Python3.6 + path = str(path) if not os.path.isfile(path): continue @@ -1803,11 +1804,12 @@ def is_strict_application_id(name): def get_all_gradle_and_manifests(build_dir): paths = [] - for root, dirs, files in os.walk(build_dir): + # TODO: Python3.6: Accepts a path-like object. + for root, dirs, files in os.walk(str(build_dir)): for f in sorted(files): if f == 'AndroidManifest.xml' \ or f.endswith('.gradle') or f.endswith('.gradle.kts'): - full = os.path.join(root, f) + full = Path(root) / f paths.append(full) return paths @@ -1817,22 +1819,18 @@ def get_gradle_subdir(build_dir, paths): first_gradle_dir = None for path in paths: if not first_gradle_dir: - first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir) - if os.path.exists(path) and SETTINGS_GRADLE_REGEX.match(os.path.basename(path)): - with open(path) as fp: - for m in GRADLE_SUBPROJECT_REGEX.finditer(fp.read()): - for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')): - with open(f) as fp: - while True: - line = fp.readline() - if not line: - break - if ANDROID_PLUGIN_REGEX.match(line): - return os.path.relpath(os.path.dirname(f), build_dir) - if first_gradle_dir and first_gradle_dir != '.': + first_gradle_dir = path.parent.relative_to(build_dir) + if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)): + for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text()): + for f in (path.parent / m.group(1)).glob('build.gradle*'): + with f.open() as fp: + for line in fp.readlines(): + if ANDROID_PLUGIN_REGEX.match(line): + return f.parent.relative_to(build_dir) + if first_gradle_dir and first_gradle_dir != Path('.'): return first_gradle_dir - return '' + return def getrepofrompage(url): @@ -4324,3 +4322,10 @@ NDKS = [ "url": "https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip" } ] + + +def handle_retree_error_on_windows(function, path, excinfo): + """Python can't remove a readonly file on Windows so chmod first""" + if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError: + os.chmod(path, stat.S_IWRITE) + function(path) diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 7c608e58..b86d99d6 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -20,12 +20,12 @@ import configparser import git import json -import os import shutil import sys import yaml from argparse import ArgumentParser import logging +from pathlib import Path, PurePosixPath try: from yaml import CSafeLoader as SafeLoader @@ -44,15 +44,15 @@ options = None # WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py! -def clone_to_tmp_dir(app): - tmp_dir = 'tmp' - if not os.path.isdir(tmp_dir): - logging.info(_("Creating temporary directory")) - os.makedirs(tmp_dir) - tmp_dir = os.path.join(tmp_dir, 'importer') - if os.path.exists(tmp_dir): - shutil.rmtree(tmp_dir) +def clone_to_tmp_dir(app): + tmp_dir = Path('tmp') + tmp_dir.mkdir(exist_ok=True) + + tmp_dir = tmp_dir / 'importer' + + if tmp_dir.exists(): + shutil.rmtree(str(tmp_dir), onerror=common.handle_retree_error_on_windows) vcs = common.getvcs(app.RepoType, app.Repo, tmp_dir) vcs.gotorevision(options.rev) @@ -61,8 +61,8 @@ def clone_to_tmp_dir(app): def check_for_kivy_buildozer(tmp_importer_dir, app, build): versionCode = None - buildozer_spec = os.path.join(tmp_importer_dir, 'buildozer.spec') - if os.path.exists(buildozer_spec): + buildozer_spec = tmp_importer_dir / 'buildozer.spec' + if buildozer_spec.exists(): config = configparser.ConfigParser() config.read(buildozer_spec) import pprint @@ -132,15 +132,16 @@ def main(): raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0]) build = metadata.Build() - if options.url is None and os.path.isdir('.git'): + if options.url is None and Path('.git').is_dir(): app = metadata.App() - app.AutoName = os.path.basename(os.getcwd()) + app.AutoName = Path.cwd().name app.RepoType = 'git' - if os.path.exists('build.gradle') or os.path.exists('build.gradle.kts'): + if Path('build.gradle').exists() or Path('build.gradle.kts').exists(): build.gradle = ['yes'] - git_repo = git.repo.Repo(os.getcwd()) + # TODO: Python3.6: Should accept path-like + git_repo = git.Repo(str(Path.cwd())) for remote in git.Remote.iter_items(git_repo): if remote.name == 'origin': url = git_repo.remotes.origin.url @@ -152,7 +153,9 @@ def main(): elif options.url: app = common.get_app_from_url(options.url) tmp_importer_dir = clone_to_tmp_dir(app) - git_repo = git.repo.Repo(tmp_importer_dir) + # TODO: Python3.6: Should accept path-like + git_repo = git.Repo(str(tmp_importer_dir)) + if not options.omit_disable: build.disable = 'Generated by import.py - check/set version fields and commit id' write_local_file = False @@ -189,37 +192,36 @@ def main(): build.subdir = options.subdir build.gradle = ['yes'] elif subdir: - build.subdir = subdir + build.subdir = str(PurePosixPath(subdir)) build.gradle = ['yes'] if options.license: app.License = options.license if options.categories: app.Categories = options.categories.split(',') - if os.path.exists(os.path.join(subdir, 'jni')): + if (subdir / 'jni').exists(): build.buildjni = ['yes'] - if os.path.exists(os.path.join(subdir, 'build.gradle')) \ - or os.path.exists(os.path.join(subdir, 'build.gradle')): + if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle').exists(): build.gradle = ['yes'] - package_json = os.path.join(tmp_importer_dir, 'package.json') # react-native - pubspec_yaml = os.path.join(tmp_importer_dir, 'pubspec.yaml') # flutter - if os.path.exists(package_json): + package_json = tmp_importer_dir / 'package.json' # react-native + pubspec_yaml = tmp_importer_dir / 'pubspec.yaml' # flutter + if package_json.exists(): build.sudo = ['apt-get update || apt-get update', 'apt-get install -t stretch-backports npm', 'npm install -g react-native-cli'] build.init = ['npm install'] - with open(package_json) as fp: + with package_json.open() as fp: data = json.load(fp) app.AutoName = data.get('name', app.AutoName) app.License = data.get('license', app.License) app.Description = data.get('description', app.Description) app.WebSite = data.get('homepage', app.WebSite) - app_json = os.path.join(tmp_importer_dir, 'app.json') - if os.path.exists(app_json): - with open(app_json) as fp: + app_json = tmp_importer_dir / 'app.json' + if app_json.exists(): + with app_json.open() as fp: data = json.load(fp) app.AutoName = data.get('name', app.AutoName) - if os.path.exists(pubspec_yaml): - with open(pubspec_yaml) as fp: + if pubspec_yaml.exists(): + with pubspec_yaml.open() as fp: data = yaml.load(fp, Loader=SafeLoader) app.AutoName = data.get('name', app.AutoName) app.License = data.get('license', app.License) @@ -232,8 +234,8 @@ def main(): '$$flutter$$/bin/flutter build apk', ] - git_modules = os.path.join(tmp_importer_dir, '.gitmodules') - if os.path.exists(git_modules): + git_modules = tmp_importer_dir / '.gitmodules' + if git_modules.exists(): build.submodules = True metadata.post_metadata_parse(app) @@ -241,24 +243,25 @@ def main(): app['Builds'].append(build) if write_local_file: - metadata.write_metadata('.fdroid.yml', app) + metadata.write_metadata(Path('.fdroid.yml'), app) else: # Keep the repo directory to save bandwidth... - if not os.path.exists('build'): - os.mkdir('build') - build_dir = os.path.join('build', appid) - if os.path.exists(build_dir): + Path('build').mkdir(exist_ok=True) + build_dir = Path('build') / appid + if build_dir.exists(): logging.warning(_('{path} already exists, ignoring import results!') .format(path=build_dir)) sys.exit(1) - elif tmp_importer_dir is not None: - shutil.move(tmp_importer_dir, build_dir) - with open('build/.fdroidvcs-' + appid, 'w') as f: - f.write(app.RepoType + ' ' + app.Repo) + elif tmp_importer_dir: + # For Windows: Close the repo or a git.exe instance holds handles to repo + git_repo.close() + # TODO: Python3.9: Accepts a path-like object for both src and dst. + shutil.move(str(tmp_importer_dir), str(build_dir)) + Path('build/.fdroidvcs-' + appid).write_text(app.RepoType + ' ' + app.Repo) - metadatapath = os.path.join('metadata', appid + '.yml') + metadatapath = Path('metadata') / (appid + '.yml') metadata.write_metadata(metadatapath, app) - logging.info("Wrote " + metadatapath) + logging.info("Wrote " + str(metadatapath)) if __name__ == "__main__": diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index f6524864..cc2e7876 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -978,6 +978,8 @@ build_cont = re.compile(r'^[ \t]') def write_metadata(metadatapath, app): + # TODO: Remove this + metadatapath = str(metadatapath) if metadatapath.endswith('.yml'): if importlib.util.find_spec('ruamel.yaml'): with open(metadatapath, 'w') as mf: diff --git a/tests/common.TestCase b/tests/common.TestCase index a47fe75b..2da0ecfa 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -22,6 +22,7 @@ import yaml import gzip from zipfile import ZipFile from unittest import mock +from pathlib import Path localmodule = os.path.realpath( @@ -1223,26 +1224,28 @@ class CommonTest(unittest.TestCase): def test_get_all_gradle_and_manifests(self): """Test whether the function works with relative and absolute paths""" - a = fdroidserver.common.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat')) + a = fdroidserver.common.get_all_gradle_and_manifests(Path('source-files/cn.wildfirechat.chat')) paths = [ - os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'), + 'avenginekit/build.gradle', + 'build.gradle', + 'chat/build.gradle', + 'client/build.gradle', + 'client/src/main/AndroidManifest.xml', + 'emojilibrary/build.gradle', + 'gradle/build_libraries.gradle', + 'imagepicker/build.gradle', + 'mars-core-release/build.gradle', + 'push/build.gradle', + 'settings.gradle', ] + paths = [Path('source-files/cn.wildfirechat.chat') / path for path in paths] self.assertEqual(sorted(paths), sorted(a)) - abspath = os.path.join(self.basedir, 'source-files', 'realm') + abspath = Path(self.basedir) / 'source-files/realm' p = fdroidserver.common.get_all_gradle_and_manifests(abspath) self.assertEqual(1, len(p)) - self.assertTrue(p[0].startswith(abspath)) + # TODO: Pathon3.9: self.assertTrue(p[0].is_relative_to(abspath)) + self.assertTrue(abspath in p[0].parents) def test_get_gradle_subdir(self): subdirs = { @@ -1251,17 +1254,12 @@ class CommonTest(unittest.TestCase): 'org.tasks': 'app', 'ut.ewh.audiometrytest': 'app', } - for f in ( - 'cn.wildfirechat.chat', - 'com.anpmech.launcher', - 'org.tasks', - 'ut.ewh.audiometrytest', - ): - build_dir = os.path.join('source-files', f) + for k, v in subdirs.items(): + build_dir = Path('source-files') / k paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir) logging.info(paths) subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths) - self.assertEqual(subdirs[f], subdir) + self.assertEqual(v, str(subdir)) def test_parse_srclib_spec_good(self): self.assertEqual(fdroidserver.common.parse_srclib_spec('osmand-external-skia@android/oreo'), diff --git a/tests/import.TestCase b/tests/import.TestCase index 361e27e3..9803648d 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -2,41 +2,41 @@ # http://www.drdobbs.com/testing/unit-testing-with-python/240165163 -import inspect import logging import optparse import os -import requests import shutil import sys import tempfile import unittest from unittest import mock +from pathlib import Path -localmodule = os.path.realpath( - os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') -) -print('localmodule: ' + localmodule) -if localmodule not in sys.path: - sys.path.insert(0, localmodule) - -import fdroidserver.common -import fdroidserver.metadata +import requests +from testcommon import TmpCwd # work around the syntax error from: import fdroidserver.import import import_proxy +localmodule = Path(__file__).resolve().parent.parent +print('localmodule: ' + str(localmodule)) +if localmodule not in sys.path: + sys.path.insert(0, str(localmodule)) + +import fdroidserver.common +import fdroidserver.metadata + class ImportTest(unittest.TestCase): '''fdroid import''' def setUp(self): logging.basicConfig(level=logging.DEBUG) - self.basedir = os.path.join(localmodule, 'tests') - self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles')) - if not os.path.exists(self.tmpdir): - os.makedirs(self.tmpdir) - os.chdir(self.basedir) + self.basedir = localmodule / 'tests' + self.tmpdir = localmodule / '.testfiles' + self.tmpdir.mkdir(exist_ok=True) + # TODO: Python3.6: Accepts a path-like object. + os.chdir(str(self.basedir)) def test_import_gitlab(self): # FDroidPopen needs some config to work @@ -57,59 +57,72 @@ class ImportTest(unittest.TestCase): self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git') def test_get_app_from_url(self): - testdir = tempfile.mkdtemp( - prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir - ) - os.chdir(testdir) - os.mkdir(os.path.join(testdir, 'tmp')) - tmp_importer = os.path.join(testdir, 'tmp', 'importer') - data = ( - ( - 'cn.wildfirechat.chat', - 'https://github.com/wildfirechat/android-chat', - '0.6.9', - '23', - ), - ( - 'com.anpmech.launcher', - 'https://github.com/KeikaiLauncher/KeikaiLauncher', - 'Unknown', - None, - ), - ( - 'ut.ewh.audiometrytest', - 'https://github.com/ReeceStevens/ut_ewh_audiometer_2014', - '1.65', - '14', - ), - ) - for appid, url, vn, vc in data: - shutil.rmtree(tmp_importer, ignore_errors=True) - shutil.copytree( - os.path.join(self.basedir, 'source-files', appid), tmp_importer + # TODO: Pytohn3.6: The dir parameter now accepts a path-like object. + with tempfile.TemporaryDirectory(dir=str(self.tmpdir)) as testdir, TmpCwd( + testdir + ): + testdir = Path(testdir) + (testdir / 'tmp').mkdir() + tmp_importer = testdir / 'tmp/importer' + data = ( + ( + 'cn.wildfirechat.chat', + 'https://github.com/wildfirechat/android-chat', + '0.6.9', + '23', + ), + ( + 'com.anpmech.launcher', + 'https://github.com/KeikaiLauncher/KeikaiLauncher', + 'Unknown', + None, + ), + ( + 'ut.ewh.audiometrytest', + 'https://github.com/ReeceStevens/ut_ewh_audiometer_2014', + '1.65', + '14', + ), ) + for appid, url, vn, vc in data: + # TODO: Python3.6: Accepts a path-like object. + shutil.rmtree( + str(tmp_importer), + onerror=fdroidserver.common.handle_retree_error_on_windows, + ) + shutil.copytree( + str(self.basedir / 'source-files' / appid), str(tmp_importer) + ) - app = fdroidserver.common.get_app_from_url(url) - with mock.patch('fdroidserver.common.getvcs', - lambda a, b, c: fdroidserver.common.vcs(url, testdir)): - with mock.patch('fdroidserver.common.vcs.gotorevision', - lambda s, rev: None): - with mock.patch('shutil.rmtree', lambda a: None): - build_dir = import_proxy.clone_to_tmp_dir(app) - self.assertEqual('git', app.RepoType) - self.assertEqual(url, app.Repo) - self.assertEqual(url, app.SourceCode) - logging.info(build_dir) - paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir) - self.assertNotEqual(paths, []) - versionName, versionCode, package = fdroidserver.common.parse_androidmanifests(paths, app) - self.assertEqual(vn, versionName) - self.assertEqual(vc, versionCode) - self.assertEqual(appid, package) + app = fdroidserver.common.get_app_from_url(url) + with mock.patch( + 'fdroidserver.common.getvcs', + lambda a, b, c: fdroidserver.common.vcs(url, testdir), + ), mock.patch( + 'fdroidserver.common.vcs.gotorevision', lambda s, rev: None + ), mock.patch( + 'shutil.rmtree', lambda a, onerror=None: None + ): + build_dir = import_proxy.clone_to_tmp_dir(app) + self.assertEqual('git', app.RepoType) + self.assertEqual(url, app.Repo) + self.assertEqual(url, app.SourceCode) + logging.info(build_dir) + paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir) + self.assertNotEqual(paths, []) + ( + versionName, + versionCode, + package, + ) = fdroidserver.common.parse_androidmanifests(paths, app) + self.assertEqual(vn, versionName) + self.assertEqual(vc, versionCode) + self.assertEqual(appid, package) if __name__ == "__main__": - os.chdir(os.path.dirname(__file__)) + # TODO: Python3.6: Support added to accept objects implementing the os.PathLike interface. + os.chdir(str(Path(__file__).parent)) parser = optparse.OptionParser() parser.add_option( diff --git a/tests/import_proxy.py b/tests/import_proxy.py index f230fdb1..5a6dfced 100644 --- a/tests/import_proxy.py +++ b/tests/import_proxy.py @@ -1,20 +1,21 @@ # workaround the syntax error from: import fdroidserver.import import inspect -import os import sys +from pathlib import Path -localmodule = os.path.realpath( - os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) -print('localmodule: ' + localmodule) +localmodule = Path(__file__).resolve().parent.parent +print('localmodule: ' + str(localmodule)) if localmodule not in sys.path: - sys.path.insert(0, localmodule) + sys.path.insert(0, str(localmodule)) + class Options: def __init__(self): self.rev = None self.subdir = None + module = __import__('fdroidserver.import') for name, obj in inspect.getmembers(module): if name == 'import':