diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 87e26d3e..dcd53b59 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -101,9 +101,6 @@ VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[ re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') -SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') -GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''') - MAX_VERSION_CODE = 0x7fffffff # Java's Integer.MAX_VALUE (2147483647) XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}' @@ -2120,37 +2117,6 @@ def is_strict_application_id(name): and '.' in name -def get_all_gradle_and_manifests(build_dir): - paths = [] - # TODO: Python3.6: Accepts a path-like object. - for root, dirs, files in os.walk(str(build_dir)): - for f in sorted(files): - if f == 'AndroidManifest.xml' \ - or f.endswith('.gradle') or f.endswith('.gradle.kts'): - full = Path(root) / f - paths.append(full) - return paths - - -def get_gradle_subdir(build_dir, paths): - """Get the subdir where the gradle build is based.""" - first_gradle_dir = None - for path in paths: - if not first_gradle_dir: - first_gradle_dir = path.parent.relative_to(build_dir) - if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)): - for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')): - for f in (path.parent / m.group(1)).glob('build.gradle*'): - with f.open(encoding='utf-8') as fp: - for line in fp.readlines(): - if ANDROID_PLUGIN_REGEX.match(line): - return f.parent.relative_to(build_dir) - if first_gradle_dir and first_gradle_dir != Path('.'): - return first_gradle_dir - - return - - def parse_srclib_spec(spec): if type(spec) != str: diff --git a/fdroidserver/import_subcommand.py b/fdroidserver/import_subcommand.py index 3902250e..b9f9a4c4 100644 --- a/fdroidserver/import_subcommand.py +++ b/fdroidserver/import_subcommand.py @@ -18,34 +18,64 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import json +import logging import os import re -import stat -import urllib - -import git -import json import shutil +import stat import sys -import yaml +import urllib from argparse import ArgumentParser -import logging from pathlib import Path from typing import Optional +import git +import yaml + try: from yaml import CSafeLoader as SafeLoader except ImportError: from yaml import SafeLoader -from . import _ -from . import common -from . import metadata +from . import _, common, metadata from .exception import FDroidException - config = None +SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') +GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''') +APPLICATION_ID_REGEX = re.compile(r'''\s*applicationId\s=?\s?['"].*['"]''') + + +def get_all_gradle_and_manifests(build_dir): + paths = [] + for root, dirs, files in os.walk(build_dir): + for f in sorted(files): + if f == 'AndroidManifest.xml' or f.endswith(('.gradle', '.gradle.kts')): + full = Path(root) / f + paths.append(full) + return paths + + +def get_gradle_subdir(build_dir, paths): + """Get the subdir where the gradle build is based.""" + first_gradle_dir = None + for path in paths: + if not first_gradle_dir: + first_gradle_dir = path.parent.relative_to(build_dir) + if path.exists() and SETTINGS_GRADLE_REGEX.match(path.name): + for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')): + for f in (path.parent / m.group(1)).glob('build.gradle*'): + with f.open(encoding='utf-8') as fp: + for line in fp: + if common.ANDROID_PLUGIN_REGEX.match( + line + ) or APPLICATION_ID_REGEX.match(line): + return f.parent.relative_to(build_dir) + if first_gradle_dir and first_gradle_dir != Path('.'): + return first_gradle_dir + def handle_retree_error_on_windows(function, path, excinfo): """Python can't remove a readonly file on Windows so chmod first.""" @@ -100,6 +130,7 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]: The found repository type or None if an error occured. address_or_reason The address to the found repository or the reason if an error occured. + """ if not url.startswith('http'): return (None, _('{url} does not start with "http"!'.format(url=url))) @@ -122,7 +153,7 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]: index = page.find('hg clone') if index != -1: repotype = 'hg' - repo = page[index + 9:] + repo = page[index + 9 :] index = repo.find('<') if index == -1: return (None, _("Error while getting repo address")) @@ -134,7 +165,7 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]: index = page.find('git clone') if index != -1: repotype = 'git' - repo = page[index + 10:] + repo = page[index + 10 :] index = repo.find('<') if index == -1: return (None, _("Error while getting repo address")) @@ -168,6 +199,7 @@ def get_app_from_url(url: str) -> metadata.App: If the VCS type could not be determined. :exc:`ValueError` If the URL is invalid. + """ parsed = urllib.parse.urlparse(url) invalid_url = False @@ -243,18 +275,29 @@ def main(): # Parse command line... parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument("-u", "--url", default=None, - help=_("Project URL to import from.")) - parser.add_argument("-s", "--subdir", default=None, - help=_("Path to main Android project subdirectory, if not in root.")) - parser.add_argument("-c", "--categories", default=None, - help=_("Comma separated list of categories.")) - parser.add_argument("-l", "--license", default=None, - help=_("Overall license of the project.")) - parser.add_argument("--omit-disable", action="store_true", default=False, - help=_("Do not add 'disable:' to the generated build entries")) - parser.add_argument("--rev", default=None, - help=_("Allows a different revision (or git branch) to be specified for the initial import")) + parser.add_argument("-u", "--url", help=_("Project URL to import from.")) + parser.add_argument( + "-s", + "--subdir", + help=_("Path to main Android project subdirectory, if not in root."), + ) + parser.add_argument( + "-c", + "--categories", + help=_("Comma separated list of categories."), + ) + parser.add_argument("-l", "--license", help=_("Overall license of the project.")) + parser.add_argument( + "--omit-disable", + action="store_true", + help=_("Do not add 'disable:' to the generated build entries"), + ) + parser.add_argument( + "--rev", + help=_( + "Allows a different revision (or git branch) to be specified for the initial import" + ), + ) metadata.add_metadata_arguments(parser) options = common.parse_args(parser) metadata.warnings_action = options.W @@ -268,24 +311,20 @@ def main(): local_metadata_files = common.get_local_metadata_files() if local_metadata_files: - raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0]) + raise FDroidException( + _("This repo already has local metadata: %s") % local_metadata_files[0] + ) build = metadata.Build() + app = metadata.App() if options.url is None and Path('.git').is_dir(): - app = metadata.App() - app.AutoName = Path.cwd().name app.RepoType = 'git' - - if Path('build.gradle').exists() or Path('build.gradle.kts').exists(): - build.gradle = ['yes'] - - git_repo = git.Repo(Path.cwd()) + tmp_importer_dir = Path.cwd() + git_repo = git.Repo(tmp_importer_dir) for remote in git.Remote.iter_items(git_repo): if remote.name == 'origin': url = git_repo.remotes.origin.url - if url.startswith('https://git'): # github, gitlab - app.SourceCode = url.rstrip('.git') - app.Repo = url + app = get_app_from_url(url) break write_local_file = True elif options.url: @@ -294,17 +333,20 @@ def main(): git_repo = git.Repo(tmp_importer_dir) if not options.omit_disable: - build.disable = 'Generated by `fdroid import` - check version fields and commitid' + build.disable = ( + 'Generated by `fdroid import` - check version fields and commitid' + ) write_local_file = False else: raise FDroidException("Specify project url.") + app.AutoUpdateMode = 'Version' app.UpdateCheckMode = 'Tags' build.commit = common.get_head_commit_id(git_repo) # Extract some information... - paths = common.get_all_gradle_and_manifests(tmp_importer_dir) - subdir = common.get_gradle_subdir(tmp_importer_dir, paths) + paths = get_all_gradle_and_manifests(tmp_importer_dir) + gradle_subdir = get_gradle_subdir(tmp_importer_dir, paths) if paths: versionName, versionCode, appid = common.parse_androidmanifests(paths, app) if not appid: @@ -322,16 +364,15 @@ def main(): # Create a build line... build.versionName = versionName or 'Unknown' + app.CurrentVersion = build.versionName build.versionCode = versionCode or 0 + app.CurrentVersionCode = build.versionCode if options.subdir: build.subdir = options.subdir - build.gradle = ['yes'] - elif subdir: - build.subdir = subdir.as_posix() - build.gradle = ['yes'] - else: - # subdir might be None - subdir = Path() + elif gradle_subdir: + build.subdir = gradle_subdir.as_posix() + # subdir might be None + subdir = Path(tmp_importer_dir / build.subdir) if build.subdir else tmp_importer_dir if options.license: app.License = options.license @@ -339,23 +380,23 @@ def main(): app.Categories = options.categories.split(',') if (subdir / 'jni').exists(): build.buildjni = ['yes'] - if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle').exists(): + if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle.kts').exists(): build.gradle = ['yes'] + app.AutoName = common.fetch_real_name(subdir, build.gradle) + package_json = tmp_importer_dir / 'package.json' # react-native pubspec_yaml = tmp_importer_dir / 'pubspec.yaml' # flutter if package_json.exists(): build.sudo = [ 'sysctl fs.inotify.max_user_watches=524288 || true', - 'curl -Lo node.tar.gz https://nodejs.org/download/release/v19.3.0/node-v19.3.0-linux-x64.tar.gz', - 'echo "b525028ae5bb71b5b32cb7fce903ccce261dbfef4c7dd0f3e0ffc27cd6fc0b3f node.tar.gz" | sha256sum -c -', - 'tar xzf node.tar.gz --strip-components=1 -C /usr/local/', - 'npm -g install yarn', + 'apt-get update', + 'apt-get install -y npm', ] build.init = ['npm install --build-from-source'] with package_json.open() as fp: data = json.load(fp) - app.AutoName = data.get('name', app.AutoName) + app.AutoName = app.AutoName or data.get('name') app.License = data.get('license', app.License) app.Description = data.get('description', app.Description) app.WebSite = data.get('homepage', app.WebSite) @@ -365,11 +406,11 @@ def main(): if app_json.exists(): with app_json.open() as fp: data = json.load(fp) - app.AutoName = data.get('name', app.AutoName) + app.AutoName = app.AutoName or data.get('name') if pubspec_yaml.exists(): with pubspec_yaml.open() as fp: data = yaml.load(fp, Loader=SafeLoader) - app.AutoName = data.get('name', app.AutoName) + app.AutoName = app.AutoName or data.get('name') app.License = data.get('license', app.License) app.Description = data.get('description', app.Description) app.UpdateCheckData = 'pubspec.yaml|version:\\s.+\\+(\\d+)|.|version:\\s(.+)\\+' @@ -405,8 +446,11 @@ def main(): Path('build').mkdir(exist_ok=True) build_dir = Path('build') / appid if build_dir.exists(): - logging.warning(_('{path} already exists, ignoring import results!') - .format(path=build_dir)) + logging.warning( + _('{path} already exists, ignoring import results!').format( + path=build_dir + ) + ) sys.exit(1) elif tmp_importer_dir: # For Windows: Close the repo or a git.exe instance holds handles to repo diff --git a/pyproject.toml b/pyproject.toml index a64bae4e..a6262ae2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,3 @@ - # We ignore the following PEP8 warnings # * E123: closing bracket does not match indentation of opening bracket's line # - Broken if multiple indentation levels start on a single line @@ -38,7 +37,6 @@ force-exclude = '''( | fdroidserver/build\.py | fdroidserver/checkupdates\.py | fdroidserver/common\.py - | fdroidserver/import_subcommand\.py | fdroidserver/index\.py | fdroidserver/metadata\.py | fdroidserver/nightly\.py diff --git a/tests/common.TestCase b/tests/common.TestCase index 48e668fc..cf961f8d 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1522,45 +1522,6 @@ class CommonTest(unittest.TestCase): self.assertEqual(('2021-06-30', 34, 'de.varengold.activeTAN'), fdroidserver.common.parse_androidmanifests(paths, app)) - def test_get_all_gradle_and_manifests(self): - """Test whether the function works with relative and absolute paths""" - a = fdroidserver.common.get_all_gradle_and_manifests(Path('source-files/cn.wildfirechat.chat')) - paths = [ - 'avenginekit/build.gradle', - 'build.gradle', - 'chat/build.gradle', - 'client/build.gradle', - 'client/src/main/AndroidManifest.xml', - 'emojilibrary/build.gradle', - 'gradle/build_libraries.gradle', - 'imagepicker/build.gradle', - 'mars-core-release/build.gradle', - 'push/build.gradle', - 'settings.gradle', - ] - paths = [Path('source-files/cn.wildfirechat.chat') / path for path in paths] - self.assertEqual(sorted(paths), sorted(a)) - - abspath = Path(self.basedir) / 'source-files/realm' - p = fdroidserver.common.get_all_gradle_and_manifests(abspath) - self.assertEqual(1, len(p)) - self.assertTrue(p[0].is_relative_to(abspath)) - - def test_get_gradle_subdir(self): - subdirs = { - 'cn.wildfirechat.chat': 'chat', - 'com.anpmech.launcher': 'app', - 'org.tasks': 'app', - 'ut.ewh.audiometrytest': 'app', - 'org.noise_planet.noisecapture': 'app', - } - for k, v in subdirs.items(): - build_dir = Path('source-files') / k - paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir) - logging.info(paths) - subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths) - self.assertEqual(v, str(subdir)) - def test_parse_srclib_spec_good(self): self.assertEqual(fdroidserver.common.parse_srclib_spec('osmand-external-skia@android/oreo'), ('osmand-external-skia', 'android/oreo', None, None)) diff --git a/tests/import_subcommand.TestCase b/tests/import_subcommand.TestCase index b37c2f37..411f500d 100755 --- a/tests/import_subcommand.TestCase +++ b/tests/import_subcommand.TestCase @@ -2,29 +2,30 @@ # http://www.drdobbs.com/testing/unit-testing-with-python/240165163 -import git import logging import os import shutil import sys import tempfile import unittest -import yaml -from unittest import mock from pathlib import Path +from unittest import mock +import git import requests +import yaml localmodule = Path(__file__).resolve().parent.parent print('localmodule: ' + str(localmodule)) if localmodule not in sys.path: sys.path.insert(0, str(localmodule)) +from testcommon import TmpCwd, mkdtemp, parse_args_for_test + import fdroidserver.common import fdroidserver.import_subcommand import fdroidserver.metadata from fdroidserver.exception import FDroidException -from testcommon import TmpCwd, mkdtemp, parse_args_for_test class ImportTest(unittest.TestCase): @@ -41,6 +42,49 @@ class ImportTest(unittest.TestCase): os.chdir(self.basedir) self._td.cleanup() + def test_get_all_gradle_and_manifests(self): + """Test whether the function works with relative and absolute paths""" + a = fdroidserver.import_subcommand.get_all_gradle_and_manifests( + Path('source-files/cn.wildfirechat.chat') + ) + paths = [ + 'avenginekit/build.gradle', + 'build.gradle', + 'chat/build.gradle', + 'client/build.gradle', + 'client/src/main/AndroidManifest.xml', + 'emojilibrary/build.gradle', + 'gradle/build_libraries.gradle', + 'imagepicker/build.gradle', + 'mars-core-release/build.gradle', + 'push/build.gradle', + 'settings.gradle', + ] + paths = [Path('source-files/cn.wildfirechat.chat') / path for path in paths] + self.assertEqual(sorted(paths), sorted(a)) + + abspath = Path(self.basedir) / 'source-files/realm' + p = fdroidserver.import_subcommand.get_all_gradle_and_manifests(abspath) + self.assertEqual(1, len(p)) + self.assertTrue(p[0].is_relative_to(abspath)) + + def test_get_gradle_subdir(self): + subdirs = { + 'cn.wildfirechat.chat': 'chat', + 'com.anpmech.launcher': 'app', + 'org.tasks': 'app', + 'ut.ewh.audiometrytest': 'app', + 'org.noise_planet.noisecapture': 'app', + } + for k, v in subdirs.items(): + build_dir = Path('source-files') / k + paths = fdroidserver.import_subcommand.get_all_gradle_and_manifests( + build_dir + ) + logging.info(paths) + subdir = fdroidserver.import_subcommand.get_gradle_subdir(build_dir, paths) + self.assertEqual(v, str(subdir)) + def test_import_gitlab(self): with tempfile.TemporaryDirectory() as testdir, TmpCwd(testdir): # FDroidPopen needs some config to work @@ -106,7 +150,9 @@ class ImportTest(unittest.TestCase): self.assertEqual(url, app.Repo) self.assertEqual(url, app.SourceCode) logging.info(build_dir) - paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir) + paths = fdroidserver.import_subcommand.get_all_gradle_and_manifests( + build_dir + ) self.assertNotEqual(paths, []) ( versionName,