mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
Merge branch 'remove_import_hack' into 'master'
[import] Rename to import_subcommand internally See merge request fdroid/fdroidserver!1142
This commit is contained in:
commit
10528b832a
@ -254,8 +254,7 @@ black:
|
||||
tests/build.TestCase
|
||||
tests/deploy.TestCase
|
||||
tests/exception.TestCase
|
||||
tests/import.TestCase
|
||||
tests/import_proxy.py
|
||||
tests/import_subcommand.TestCase
|
||||
tests/init.TestCase
|
||||
tests/install.TestCase
|
||||
tests/key-tricks.py
|
||||
|
@ -567,8 +567,7 @@ include tests/gnupghome/secring.gpg
|
||||
include tests/gnupghome/trustdb.gpg
|
||||
include tests/gradle-maven-blocks.yaml
|
||||
include tests/gradle-release-checksums.py
|
||||
include tests/import_proxy.py
|
||||
include tests/import.TestCase
|
||||
include tests/import_subcommand.TestCase
|
||||
include tests/index.TestCase
|
||||
include tests/init.TestCase
|
||||
include tests/install.TestCase
|
||||
|
@ -40,7 +40,7 @@ COMMANDS = OrderedDict([
|
||||
("deploy", _("Interact with the repo HTTP server")),
|
||||
("verify", _("Verify the integrity of downloaded packages")),
|
||||
("checkupdates", _("Check for updates to applications")),
|
||||
("import", _("Add a new application from its source code")),
|
||||
("import", _("Extract application metadata from a source repository")),
|
||||
("install", _("Install built packages on devices")),
|
||||
("readmeta", _("Read all the metadata files and exit")),
|
||||
("rewritemeta", _("Rewrite all the metadata files")),
|
||||
@ -197,6 +197,8 @@ def main():
|
||||
|
||||
del sys.argv[1]
|
||||
if command in COMMANDS.keys():
|
||||
# import is named import_subcommand internally b/c import is reserved by Python
|
||||
command = 'import_subcommand' if command == 'import' else command
|
||||
mod = __import__('fdroidserver.' + command, None, None, [command])
|
||||
else:
|
||||
mod = __import__(available_plugins[command]['name'], None, None, [command])
|
||||
|
@ -45,8 +45,6 @@ import logging
|
||||
import hashlib
|
||||
import socket
|
||||
import base64
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import yaml
|
||||
import zipfile
|
||||
import tempfile
|
||||
@ -1944,121 +1942,6 @@ def get_gradle_subdir(build_dir, paths):
|
||||
return
|
||||
|
||||
|
||||
def getrepofrompage(url):
|
||||
"""Get the repo type and address from the given web page.
|
||||
|
||||
The page is scanned in a rather naive manner for 'git clone xxxx',
|
||||
'hg clone xxxx', etc, and when one of these is found it's assumed
|
||||
that's the information we want. Returns repotype, address, or
|
||||
None, reason
|
||||
|
||||
"""
|
||||
if not url.startswith('http'):
|
||||
return (None, _('{url} does not start with "http"!'.format(url=url)))
|
||||
req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
|
||||
if req.getcode() != 200:
|
||||
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
|
||||
page = req.read().decode(req.headers.get_content_charset())
|
||||
|
||||
# Works for BitBucket
|
||||
m = re.search('data-fetch-url="(.*)"', page)
|
||||
if m is not None:
|
||||
repo = m.group(1)
|
||||
|
||||
if repo.endswith('.git'):
|
||||
return ('git', repo)
|
||||
|
||||
return ('hg', repo)
|
||||
|
||||
# Works for BitBucket (obsolete)
|
||||
index = page.find('hg clone')
|
||||
if index != -1:
|
||||
repotype = 'hg'
|
||||
repo = page[index + 9:]
|
||||
index = repo.find('<')
|
||||
if index == -1:
|
||||
return (None, _("Error while getting repo address"))
|
||||
repo = repo[:index]
|
||||
repo = repo.split('"')[0]
|
||||
return (repotype, repo)
|
||||
|
||||
# Works for BitBucket (obsolete)
|
||||
index = page.find('git clone')
|
||||
if index != -1:
|
||||
repotype = 'git'
|
||||
repo = page[index + 10:]
|
||||
index = repo.find('<')
|
||||
if index == -1:
|
||||
return (None, _("Error while getting repo address"))
|
||||
repo = repo[:index]
|
||||
repo = repo.split('"')[0]
|
||||
return (repotype, repo)
|
||||
|
||||
return (None, _("No information found.") + page)
|
||||
|
||||
|
||||
def get_app_from_url(url):
|
||||
"""Guess basic app metadata from the URL.
|
||||
|
||||
The URL must include a network hostname, unless it is an lp:,
|
||||
file:, or git/ssh URL. This throws ValueError on bad URLs to
|
||||
match urlparse().
|
||||
|
||||
"""
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
invalid_url = False
|
||||
if not parsed.scheme or not parsed.path:
|
||||
invalid_url = True
|
||||
|
||||
app = fdroidserver.metadata.App()
|
||||
app.Repo = url
|
||||
if url.startswith('git://') or url.startswith('git@'):
|
||||
app.RepoType = 'git'
|
||||
elif parsed.netloc == 'github.com':
|
||||
app.RepoType = 'git'
|
||||
app.SourceCode = url
|
||||
app.IssueTracker = url + '/issues'
|
||||
elif parsed.netloc == 'gitlab.com' or parsed.netloc == 'framagit.org':
|
||||
# git can be fussy with gitlab URLs unless they end in .git
|
||||
if url.endswith('.git'):
|
||||
url = url[:-4]
|
||||
app.Repo = url + '.git'
|
||||
app.RepoType = 'git'
|
||||
app.SourceCode = url
|
||||
app.IssueTracker = url + '/issues'
|
||||
elif parsed.netloc == 'notabug.org':
|
||||
if url.endswith('.git'):
|
||||
url = url[:-4]
|
||||
app.Repo = url + '.git'
|
||||
app.RepoType = 'git'
|
||||
app.SourceCode = url
|
||||
app.IssueTracker = url + '/issues'
|
||||
elif parsed.netloc == 'bitbucket.org':
|
||||
if url.endswith('/'):
|
||||
url = url[:-1]
|
||||
app.SourceCode = url + '/src'
|
||||
app.IssueTracker = url + '/issues'
|
||||
# Figure out the repo type and adddress...
|
||||
app.RepoType, app.Repo = getrepofrompage(url)
|
||||
elif parsed.netloc == 'codeberg.org':
|
||||
app.RepoType = 'git'
|
||||
app.SourceCode = url
|
||||
app.IssueTracker = url + '/issues'
|
||||
elif url.startswith('https://') and url.endswith('.git'):
|
||||
app.RepoType = 'git'
|
||||
|
||||
if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
|
||||
invalid_url = True
|
||||
|
||||
if invalid_url:
|
||||
raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
|
||||
|
||||
if not app.RepoType:
|
||||
raise FDroidException("Unable to determine vcs type. " + app.Repo)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def parse_srclib_spec(spec):
|
||||
|
||||
if type(spec) != str:
|
||||
@ -4609,10 +4492,3 @@ NDKS = [
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r25b-linux.zip"
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def handle_retree_error_on_windows(function, path, excinfo):
|
||||
"""Python can't remove a readonly file on Windows so chmod first."""
|
||||
if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError:
|
||||
os.chmod(path, stat.S_IWRITE)
|
||||
function(path)
|
||||
|
@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# import.py - part of the FDroid server tools
|
||||
# import_subcommand.py - part of the FDroid server tools
|
||||
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
|
||||
# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
|
||||
#
|
||||
@ -18,6 +18,11 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import configparser
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import urllib
|
||||
|
||||
import git
|
||||
import json
|
||||
import shutil
|
||||
@ -42,7 +47,11 @@ config = None
|
||||
options = None
|
||||
|
||||
|
||||
# WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py!
|
||||
def handle_retree_error_on_windows(function, path, excinfo):
|
||||
"""Python can't remove a readonly file on Windows so chmod first."""
|
||||
if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError:
|
||||
os.chmod(path, stat.S_IWRITE)
|
||||
function(path)
|
||||
|
||||
|
||||
def clone_to_tmp_dir(app):
|
||||
@ -52,13 +61,128 @@ def clone_to_tmp_dir(app):
|
||||
tmp_dir = tmp_dir / 'importer'
|
||||
|
||||
if tmp_dir.exists():
|
||||
shutil.rmtree(str(tmp_dir), onerror=common.handle_retree_error_on_windows)
|
||||
shutil.rmtree(str(tmp_dir), onerror=handle_retree_error_on_windows)
|
||||
vcs = common.getvcs(app.RepoType, app.Repo, tmp_dir)
|
||||
vcs.gotorevision(options.rev)
|
||||
|
||||
return tmp_dir
|
||||
|
||||
|
||||
def getrepofrompage(url):
|
||||
"""Get the repo type and address from the given web page.
|
||||
|
||||
The page is scanned in a rather naive manner for 'git clone xxxx',
|
||||
'hg clone xxxx', etc, and when one of these is found it's assumed
|
||||
that's the information we want. Returns repotype, address, or
|
||||
None, reason
|
||||
|
||||
"""
|
||||
if not url.startswith('http'):
|
||||
return (None, _('{url} does not start with "http"!'.format(url=url)))
|
||||
req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
|
||||
if req.getcode() != 200:
|
||||
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
|
||||
page = req.read().decode(req.headers.get_content_charset())
|
||||
|
||||
# Works for BitBucket
|
||||
m = re.search('data-fetch-url="(.*)"', page)
|
||||
if m is not None:
|
||||
repo = m.group(1)
|
||||
|
||||
if repo.endswith('.git'):
|
||||
return ('git', repo)
|
||||
|
||||
return ('hg', repo)
|
||||
|
||||
# Works for BitBucket (obsolete)
|
||||
index = page.find('hg clone')
|
||||
if index != -1:
|
||||
repotype = 'hg'
|
||||
repo = page[index + 9:]
|
||||
index = repo.find('<')
|
||||
if index == -1:
|
||||
return (None, _("Error while getting repo address"))
|
||||
repo = repo[:index]
|
||||
repo = repo.split('"')[0]
|
||||
return (repotype, repo)
|
||||
|
||||
# Works for BitBucket (obsolete)
|
||||
index = page.find('git clone')
|
||||
if index != -1:
|
||||
repotype = 'git'
|
||||
repo = page[index + 10:]
|
||||
index = repo.find('<')
|
||||
if index == -1:
|
||||
return (None, _("Error while getting repo address"))
|
||||
repo = repo[:index]
|
||||
repo = repo.split('"')[0]
|
||||
return (repotype, repo)
|
||||
|
||||
return (None, _("No information found.") + page)
|
||||
|
||||
|
||||
def get_app_from_url(url):
|
||||
"""Guess basic app metadata from the URL.
|
||||
|
||||
The URL must include a network hostname, unless it is an lp:,
|
||||
file:, or git/ssh URL. This throws ValueError on bad URLs to
|
||||
match urlparse().
|
||||
|
||||
"""
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
invalid_url = False
|
||||
if not parsed.scheme or not parsed.path:
|
||||
invalid_url = True
|
||||
|
||||
app = metadata.App()
|
||||
app.Repo = url
|
||||
if url.startswith('git://') or url.startswith('git@'):
|
||||
app.RepoType = 'git'
|
||||
elif parsed.netloc == 'github.com':
|
||||
app.RepoType = 'git'
|
||||
app.SourceCode = url
|
||||
app.IssueTracker = url + '/issues'
|
||||
elif parsed.netloc == 'gitlab.com' or parsed.netloc == 'framagit.org':
|
||||
# git can be fussy with gitlab URLs unless they end in .git
|
||||
if url.endswith('.git'):
|
||||
url = url[:-4]
|
||||
app.Repo = url + '.git'
|
||||
app.RepoType = 'git'
|
||||
app.SourceCode = url
|
||||
app.IssueTracker = url + '/issues'
|
||||
elif parsed.netloc == 'notabug.org':
|
||||
if url.endswith('.git'):
|
||||
url = url[:-4]
|
||||
app.Repo = url + '.git'
|
||||
app.RepoType = 'git'
|
||||
app.SourceCode = url
|
||||
app.IssueTracker = url + '/issues'
|
||||
elif parsed.netloc == 'bitbucket.org':
|
||||
if url.endswith('/'):
|
||||
url = url[:-1]
|
||||
app.SourceCode = url + '/src'
|
||||
app.IssueTracker = url + '/issues'
|
||||
# Figure out the repo type and adddress...
|
||||
app.RepoType, app.Repo = getrepofrompage(url)
|
||||
elif parsed.netloc == 'codeberg.org':
|
||||
app.RepoType = 'git'
|
||||
app.SourceCode = url
|
||||
app.IssueTracker = url + '/issues'
|
||||
elif url.startswith('https://') and url.endswith('.git'):
|
||||
app.RepoType = 'git'
|
||||
|
||||
if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
|
||||
invalid_url = True
|
||||
|
||||
if invalid_url:
|
||||
raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
|
||||
|
||||
if not app.RepoType:
|
||||
raise FDroidException("Unable to determine vcs type. " + app.Repo)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def check_for_kivy_buildozer(tmp_importer_dir, app, build):
|
||||
versionCode = None
|
||||
buildozer_spec = tmp_importer_dir / 'buildozer.spec'
|
||||
@ -151,7 +275,7 @@ def main():
|
||||
break
|
||||
write_local_file = True
|
||||
elif options.url:
|
||||
app = common.get_app_from_url(options.url)
|
||||
app = get_app_from_url(options.url)
|
||||
tmp_importer_dir = clone_to_tmp_dir(app)
|
||||
# TODO: Python3.6: Should accept path-like
|
||||
git_repo = git.Repo(str(tmp_importer_dir))
|
@ -1419,19 +1419,6 @@ class CommonTest(unittest.TestCase):
|
||||
with self.assertRaises(MetaDataException):
|
||||
self.assertEqual(fdroidserver.common.parse_srclib_spec('@multi@at-signs@'))
|
||||
|
||||
def test_bad_urls(self):
|
||||
for url in (
|
||||
'asdf',
|
||||
'file://thing.git',
|
||||
'https:///github.com/my/project',
|
||||
'git:///so/many/slashes',
|
||||
'ssh:/notabug.org/missing/a/slash',
|
||||
'git:notabug.org/missing/some/slashes',
|
||||
'https//github.com/bar/baz',
|
||||
):
|
||||
with self.assertRaises(ValueError):
|
||||
fdroidserver.common.get_app_from_url(url)
|
||||
|
||||
def test_remove_signing_keys(self):
|
||||
testdir = tempfile.mkdtemp(
|
||||
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
|
||||
|
@ -1,27 +0,0 @@
|
||||
# workaround the syntax error from: import fdroidserver.import
|
||||
|
||||
import inspect
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
localmodule = Path(__file__).resolve().parent.parent
|
||||
print('localmodule: ' + str(localmodule))
|
||||
if localmodule not in sys.path:
|
||||
sys.path.insert(0, str(localmodule))
|
||||
|
||||
|
||||
class Options:
|
||||
def __init__(self):
|
||||
self.rev = None
|
||||
self.subdir = None
|
||||
|
||||
|
||||
module = __import__('fdroidserver.import')
|
||||
for name, obj in inspect.getmembers(module):
|
||||
if name == 'import':
|
||||
clone_to_tmp_dir = obj.clone_to_tmp_dir
|
||||
obj.options = Options()
|
||||
options = obj.options
|
||||
break
|
||||
|
||||
globals().update(vars(module))
|
@ -15,15 +15,13 @@ from pathlib import Path
|
||||
import requests
|
||||
from testcommon import TmpCwd
|
||||
|
||||
# work around the syntax error from: import fdroidserver.import
|
||||
import import_proxy
|
||||
|
||||
localmodule = Path(__file__).resolve().parent.parent
|
||||
print('localmodule: ' + str(localmodule))
|
||||
if localmodule not in sys.path:
|
||||
sys.path.insert(0, str(localmodule))
|
||||
|
||||
import fdroidserver.common
|
||||
import fdroidserver.import_subcommand
|
||||
import fdroidserver.metadata
|
||||
|
||||
|
||||
@ -37,6 +35,8 @@ class ImportTest(unittest.TestCase):
|
||||
self.tmpdir.mkdir(exist_ok=True)
|
||||
# TODO: Python3.6: Accepts a path-like object.
|
||||
os.chdir(str(self.basedir))
|
||||
fdroidserver.import_subcommand.options = mock.Mock()
|
||||
fdroidserver.import_subcommand.options.rev = None
|
||||
|
||||
def test_import_gitlab(self):
|
||||
# FDroidPopen needs some config to work
|
||||
@ -51,8 +51,8 @@ class ImportTest(unittest.TestCase):
|
||||
print('Skipping ImportTest!')
|
||||
return
|
||||
|
||||
app = fdroidserver.common.get_app_from_url(url)
|
||||
import_proxy.clone_to_tmp_dir(app)
|
||||
app = fdroidserver.import_subcommand.get_app_from_url(url)
|
||||
fdroidserver.import_subcommand.clone_to_tmp_dir(app)
|
||||
self.assertEqual(app.RepoType, 'git')
|
||||
self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git')
|
||||
|
||||
@ -88,13 +88,13 @@ class ImportTest(unittest.TestCase):
|
||||
# TODO: Python3.6: Accepts a path-like object.
|
||||
shutil.rmtree(
|
||||
str(tmp_importer),
|
||||
onerror=fdroidserver.common.handle_retree_error_on_windows,
|
||||
onerror=fdroidserver.import_subcommand.handle_retree_error_on_windows,
|
||||
)
|
||||
shutil.copytree(
|
||||
str(self.basedir / 'source-files' / appid), str(tmp_importer)
|
||||
)
|
||||
|
||||
app = fdroidserver.common.get_app_from_url(url)
|
||||
app = fdroidserver.import_subcommand.get_app_from_url(url)
|
||||
with mock.patch(
|
||||
'fdroidserver.common.getvcs',
|
||||
lambda a, b, c: fdroidserver.common.vcs(url, testdir),
|
||||
@ -103,7 +103,7 @@ class ImportTest(unittest.TestCase):
|
||||
), mock.patch(
|
||||
'shutil.rmtree', lambda a, onerror=None: None
|
||||
):
|
||||
build_dir = import_proxy.clone_to_tmp_dir(app)
|
||||
build_dir = fdroidserver.import_subcommand.clone_to_tmp_dir(app)
|
||||
self.assertEqual('git', app.RepoType)
|
||||
self.assertEqual(url, app.Repo)
|
||||
self.assertEqual(url, app.SourceCode)
|
||||
@ -119,6 +119,19 @@ class ImportTest(unittest.TestCase):
|
||||
self.assertEqual(vc, versionCode)
|
||||
self.assertEqual(appid, package)
|
||||
|
||||
def test_bad_urls(self):
|
||||
for url in (
|
||||
'asdf',
|
||||
'file://thing.git',
|
||||
'https:///github.com/my/project',
|
||||
'git:///so/many/slashes',
|
||||
'ssh:/notabug.org/missing/a/slash',
|
||||
'git:notabug.org/missing/some/slashes',
|
||||
'https//github.com/bar/baz',
|
||||
):
|
||||
with self.assertRaises(ValueError):
|
||||
fdroidserver.import_subcommand.get_app_from_url(url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = optparse.OptionParser()
|
Loading…
Reference in New Issue
Block a user