diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d8a9f06..ed1998ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -254,8 +254,7 @@ black: tests/build.TestCase tests/deploy.TestCase tests/exception.TestCase - tests/import.TestCase - tests/import_proxy.py + tests/import_subcommand.TestCase tests/init.TestCase tests/install.TestCase tests/key-tricks.py diff --git a/MANIFEST.in b/MANIFEST.in index 2d330a4a..33266a64 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -567,8 +567,7 @@ include tests/gnupghome/secring.gpg include tests/gnupghome/trustdb.gpg include tests/gradle-maven-blocks.yaml include tests/gradle-release-checksums.py -include tests/import_proxy.py -include tests/import.TestCase +include tests/import_subcommand.TestCase include tests/index.TestCase include tests/init.TestCase include tests/install.TestCase diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 037c33d5..e470099c 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -40,7 +40,7 @@ COMMANDS = OrderedDict([ ("deploy", _("Interact with the repo HTTP server")), ("verify", _("Verify the integrity of downloaded packages")), ("checkupdates", _("Check for updates to applications")), - ("import", _("Add a new application from its source code")), + ("import", _("Extract application metadata from a source repository")), ("install", _("Install built packages on devices")), ("readmeta", _("Read all the metadata files and exit")), ("rewritemeta", _("Rewrite all the metadata files")), @@ -197,6 +197,8 @@ def main(): del sys.argv[1] if command in COMMANDS.keys(): + # import is named import_subcommand internally b/c import is reserved by Python + command = 'import_subcommand' if command == 'import' else command mod = __import__('fdroidserver.' + command, None, None, [command]) else: mod = __import__(available_plugins[command]['name'], None, None, [command]) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index b4217f94..1eaa2e2f 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -45,8 +45,6 @@ import logging import hashlib import socket import base64 -import urllib.parse -import urllib.request import yaml import zipfile import tempfile @@ -1944,121 +1942,6 @@ def get_gradle_subdir(build_dir, paths): return -def getrepofrompage(url): - """Get the repo type and address from the given web page. - - The page is scanned in a rather naive manner for 'git clone xxxx', - 'hg clone xxxx', etc, and when one of these is found it's assumed - that's the information we want. Returns repotype, address, or - None, reason - - """ - if not url.startswith('http'): - return (None, _('{url} does not start with "http"!'.format(url=url))) - req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out - if req.getcode() != 200: - return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) - page = req.read().decode(req.headers.get_content_charset()) - - # Works for BitBucket - m = re.search('data-fetch-url="(.*)"', page) - if m is not None: - repo = m.group(1) - - if repo.endswith('.git'): - return ('git', repo) - - return ('hg', repo) - - # Works for BitBucket (obsolete) - index = page.find('hg clone') - if index != -1: - repotype = 'hg' - repo = page[index + 9:] - index = repo.find('<') - if index == -1: - return (None, _("Error while getting repo address")) - repo = repo[:index] - repo = repo.split('"')[0] - return (repotype, repo) - - # Works for BitBucket (obsolete) - index = page.find('git clone') - if index != -1: - repotype = 'git' - repo = page[index + 10:] - index = repo.find('<') - if index == -1: - return (None, _("Error while getting repo address")) - repo = repo[:index] - repo = repo.split('"')[0] - return (repotype, repo) - - return (None, _("No information found.") + page) - - -def get_app_from_url(url): - """Guess basic app metadata from the URL. - - The URL must include a network hostname, unless it is an lp:, - file:, or git/ssh URL. This throws ValueError on bad URLs to - match urlparse(). - - """ - parsed = urllib.parse.urlparse(url) - invalid_url = False - if not parsed.scheme or not parsed.path: - invalid_url = True - - app = fdroidserver.metadata.App() - app.Repo = url - if url.startswith('git://') or url.startswith('git@'): - app.RepoType = 'git' - elif parsed.netloc == 'github.com': - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'gitlab.com' or parsed.netloc == 'framagit.org': - # git can be fussy with gitlab URLs unless they end in .git - if url.endswith('.git'): - url = url[:-4] - app.Repo = url + '.git' - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'notabug.org': - if url.endswith('.git'): - url = url[:-4] - app.Repo = url + '.git' - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'bitbucket.org': - if url.endswith('/'): - url = url[:-1] - app.SourceCode = url + '/src' - app.IssueTracker = url + '/issues' - # Figure out the repo type and adddress... - app.RepoType, app.Repo = getrepofrompage(url) - elif parsed.netloc == 'codeberg.org': - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif url.startswith('https://') and url.endswith('.git'): - app.RepoType = 'git' - - if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'): - invalid_url = True - - if invalid_url: - raise ValueError(_('"{url}" is not a valid URL!'.format(url=url))) - - if not app.RepoType: - raise FDroidException("Unable to determine vcs type. " + app.Repo) - - return app - - def parse_srclib_spec(spec): if type(spec) != str: @@ -4609,10 +4492,3 @@ NDKS = [ "url": "https://dl.google.com/android/repository/android-ndk-r25b-linux.zip" } ] - - -def handle_retree_error_on_windows(function, path, excinfo): - """Python can't remove a readonly file on Windows so chmod first.""" - if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError: - os.chmod(path, stat.S_IWRITE) - function(path) diff --git a/fdroidserver/import.py b/fdroidserver/import_subcommand.py similarity index 71% rename from fdroidserver/import.py rename to fdroidserver/import_subcommand.py index d524632b..e32a9f90 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import_subcommand.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# import.py - part of the FDroid server tools +# import_subcommand.py - part of the FDroid server tools # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com # Copyright (C) 2013-2014 Daniel Martí # @@ -18,6 +18,11 @@ # along with this program. If not, see . import configparser +import os +import re +import stat +import urllib + import git import json import shutil @@ -42,7 +47,11 @@ config = None options = None -# WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py! +def handle_retree_error_on_windows(function, path, excinfo): + """Python can't remove a readonly file on Windows so chmod first.""" + if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError: + os.chmod(path, stat.S_IWRITE) + function(path) def clone_to_tmp_dir(app): @@ -52,13 +61,128 @@ def clone_to_tmp_dir(app): tmp_dir = tmp_dir / 'importer' if tmp_dir.exists(): - shutil.rmtree(str(tmp_dir), onerror=common.handle_retree_error_on_windows) + shutil.rmtree(str(tmp_dir), onerror=handle_retree_error_on_windows) vcs = common.getvcs(app.RepoType, app.Repo, tmp_dir) vcs.gotorevision(options.rev) return tmp_dir +def getrepofrompage(url): + """Get the repo type and address from the given web page. + + The page is scanned in a rather naive manner for 'git clone xxxx', + 'hg clone xxxx', etc, and when one of these is found it's assumed + that's the information we want. Returns repotype, address, or + None, reason + + """ + if not url.startswith('http'): + return (None, _('{url} does not start with "http"!'.format(url=url))) + req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out + if req.getcode() != 200: + return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) + page = req.read().decode(req.headers.get_content_charset()) + + # Works for BitBucket + m = re.search('data-fetch-url="(.*)"', page) + if m is not None: + repo = m.group(1) + + if repo.endswith('.git'): + return ('git', repo) + + return ('hg', repo) + + # Works for BitBucket (obsolete) + index = page.find('hg clone') + if index != -1: + repotype = 'hg' + repo = page[index + 9:] + index = repo.find('<') + if index == -1: + return (None, _("Error while getting repo address")) + repo = repo[:index] + repo = repo.split('"')[0] + return (repotype, repo) + + # Works for BitBucket (obsolete) + index = page.find('git clone') + if index != -1: + repotype = 'git' + repo = page[index + 10:] + index = repo.find('<') + if index == -1: + return (None, _("Error while getting repo address")) + repo = repo[:index] + repo = repo.split('"')[0] + return (repotype, repo) + + return (None, _("No information found.") + page) + + +def get_app_from_url(url): + """Guess basic app metadata from the URL. + + The URL must include a network hostname, unless it is an lp:, + file:, or git/ssh URL. This throws ValueError on bad URLs to + match urlparse(). + + """ + parsed = urllib.parse.urlparse(url) + invalid_url = False + if not parsed.scheme or not parsed.path: + invalid_url = True + + app = metadata.App() + app.Repo = url + if url.startswith('git://') or url.startswith('git@'): + app.RepoType = 'git' + elif parsed.netloc == 'github.com': + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'gitlab.com' or parsed.netloc == 'framagit.org': + # git can be fussy with gitlab URLs unless they end in .git + if url.endswith('.git'): + url = url[:-4] + app.Repo = url + '.git' + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'notabug.org': + if url.endswith('.git'): + url = url[:-4] + app.Repo = url + '.git' + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'bitbucket.org': + if url.endswith('/'): + url = url[:-1] + app.SourceCode = url + '/src' + app.IssueTracker = url + '/issues' + # Figure out the repo type and adddress... + app.RepoType, app.Repo = getrepofrompage(url) + elif parsed.netloc == 'codeberg.org': + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif url.startswith('https://') and url.endswith('.git'): + app.RepoType = 'git' + + if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'): + invalid_url = True + + if invalid_url: + raise ValueError(_('"{url}" is not a valid URL!'.format(url=url))) + + if not app.RepoType: + raise FDroidException("Unable to determine vcs type. " + app.Repo) + + return app + + def check_for_kivy_buildozer(tmp_importer_dir, app, build): versionCode = None buildozer_spec = tmp_importer_dir / 'buildozer.spec' @@ -151,7 +275,7 @@ def main(): break write_local_file = True elif options.url: - app = common.get_app_from_url(options.url) + app = get_app_from_url(options.url) tmp_importer_dir = clone_to_tmp_dir(app) # TODO: Python3.6: Should accept path-like git_repo = git.Repo(str(tmp_importer_dir)) diff --git a/tests/common.TestCase b/tests/common.TestCase index e80ecd35..549167fc 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -1419,19 +1419,6 @@ class CommonTest(unittest.TestCase): with self.assertRaises(MetaDataException): self.assertEqual(fdroidserver.common.parse_srclib_spec('@multi@at-signs@')) - def test_bad_urls(self): - for url in ( - 'asdf', - 'file://thing.git', - 'https:///github.com/my/project', - 'git:///so/many/slashes', - 'ssh:/notabug.org/missing/a/slash', - 'git:notabug.org/missing/some/slashes', - 'https//github.com/bar/baz', - ): - with self.assertRaises(ValueError): - fdroidserver.common.get_app_from_url(url) - def test_remove_signing_keys(self): testdir = tempfile.mkdtemp( prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir diff --git a/tests/import_proxy.py b/tests/import_proxy.py deleted file mode 100644 index 5a6dfced..00000000 --- a/tests/import_proxy.py +++ /dev/null @@ -1,27 +0,0 @@ -# workaround the syntax error from: import fdroidserver.import - -import inspect -import sys -from pathlib import Path - -localmodule = Path(__file__).resolve().parent.parent -print('localmodule: ' + str(localmodule)) -if localmodule not in sys.path: - sys.path.insert(0, str(localmodule)) - - -class Options: - def __init__(self): - self.rev = None - self.subdir = None - - -module = __import__('fdroidserver.import') -for name, obj in inspect.getmembers(module): - if name == 'import': - clone_to_tmp_dir = obj.clone_to_tmp_dir - obj.options = Options() - options = obj.options - break - -globals().update(vars(module)) diff --git a/tests/import.TestCase b/tests/import_subcommand.TestCase similarity index 81% rename from tests/import.TestCase rename to tests/import_subcommand.TestCase index 81aeac39..8c2d7792 100755 --- a/tests/import.TestCase +++ b/tests/import_subcommand.TestCase @@ -15,15 +15,13 @@ from pathlib import Path import requests from testcommon import TmpCwd -# work around the syntax error from: import fdroidserver.import -import import_proxy - localmodule = Path(__file__).resolve().parent.parent print('localmodule: ' + str(localmodule)) if localmodule not in sys.path: sys.path.insert(0, str(localmodule)) import fdroidserver.common +import fdroidserver.import_subcommand import fdroidserver.metadata @@ -37,6 +35,8 @@ class ImportTest(unittest.TestCase): self.tmpdir.mkdir(exist_ok=True) # TODO: Python3.6: Accepts a path-like object. os.chdir(str(self.basedir)) + fdroidserver.import_subcommand.options = mock.Mock() + fdroidserver.import_subcommand.options.rev = None def test_import_gitlab(self): # FDroidPopen needs some config to work @@ -51,8 +51,8 @@ class ImportTest(unittest.TestCase): print('Skipping ImportTest!') return - app = fdroidserver.common.get_app_from_url(url) - import_proxy.clone_to_tmp_dir(app) + app = fdroidserver.import_subcommand.get_app_from_url(url) + fdroidserver.import_subcommand.clone_to_tmp_dir(app) self.assertEqual(app.RepoType, 'git') self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git') @@ -88,13 +88,13 @@ class ImportTest(unittest.TestCase): # TODO: Python3.6: Accepts a path-like object. shutil.rmtree( str(tmp_importer), - onerror=fdroidserver.common.handle_retree_error_on_windows, + onerror=fdroidserver.import_subcommand.handle_retree_error_on_windows, ) shutil.copytree( str(self.basedir / 'source-files' / appid), str(tmp_importer) ) - app = fdroidserver.common.get_app_from_url(url) + app = fdroidserver.import_subcommand.get_app_from_url(url) with mock.patch( 'fdroidserver.common.getvcs', lambda a, b, c: fdroidserver.common.vcs(url, testdir), @@ -103,7 +103,7 @@ class ImportTest(unittest.TestCase): ), mock.patch( 'shutil.rmtree', lambda a, onerror=None: None ): - build_dir = import_proxy.clone_to_tmp_dir(app) + build_dir = fdroidserver.import_subcommand.clone_to_tmp_dir(app) self.assertEqual('git', app.RepoType) self.assertEqual(url, app.Repo) self.assertEqual(url, app.SourceCode) @@ -119,6 +119,19 @@ class ImportTest(unittest.TestCase): self.assertEqual(vc, versionCode) self.assertEqual(appid, package) + def test_bad_urls(self): + for url in ( + 'asdf', + 'file://thing.git', + 'https:///github.com/my/project', + 'git:///so/many/slashes', + 'ssh:/notabug.org/missing/a/slash', + 'git:notabug.org/missing/some/slashes', + 'https//github.com/bar/baz', + ): + with self.assertRaises(ValueError): + fdroidserver.import_subcommand.get_app_from_url(url) + if __name__ == "__main__": parser = optparse.OptionParser()