kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-07-04 16:30:12 +02:00
Code Activity

Merge branch 'remove_import_hack' into 'master'

Browse Source 
[import] Rename to import_subcommand internally

See merge request fdroid/fdroidserver!1142
This commit is contained in:
Hans-Christoph Steiner 2022-08-24 20:35:42 +00:00
commit 10528b832a
8 changed files with 154 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitlab-ci.yml
View File

@ -254,8 +254,7 @@ black:
tests/build.TestCase
tests/deploy.TestCase
tests/exception.TestCase
tests/import.TestCase
tests/import_proxy.py
tests/import_subcommand.TestCase
tests/init.TestCase
tests/install.TestCase
tests/key-tricks.py

3
MANIFEST.in
View File

@ -567,8 +567,7 @@ include tests/gnupghome/secring.gpg
include tests/gnupghome/trustdb.gpg
include tests/gradle-maven-blocks.yaml
include tests/gradle-release-checksums.py
include tests/import_proxy.py
include tests/import.TestCase
include tests/import_subcommand.TestCase
include tests/index.TestCase
include tests/init.TestCase
include tests/install.TestCase

4
fdroidserver/__main__.py
View File

@ -40,7 +40,7 @@ COMMANDS = OrderedDict([
("deploy", _("Interact with the repo HTTP server")),
("verify", _("Verify the integrity of downloaded packages")),
("checkupdates", _("Check for updates to applications")),
("import", _("Add a new application from its source code")),
("import", _("Extract application metadata from a source repository")),
("install", _("Install built packages on devices")),
("readmeta", _("Read all the metadata files and exit")),
("rewritemeta", _("Rewrite all the metadata files")),
@ -197,6 +197,8 @@ def main():
del sys.argv[1]
if command in COMMANDS.keys():
# import is named import_subcommand internally b/c import is reserved by Python
command = 'import_subcommand' if command == 'import' else command
mod = __import__('fdroidserver.' + command, None, None, [command])
else:
mod = __import__(available_plugins[command]['name'], None, None, [command])

124
fdroidserver/common.py
View File

@ -45,8 +45,6 @@ import logging
import hashlib
import socket
import base64
import urllib.parse
import urllib.request
import yaml
import zipfile
import tempfile
@ -1944,121 +1942,6 @@ def get_gradle_subdir(build_dir, paths):
return
def getrepofrompage(url):
"""Get the repo type and address from the given web page.
The page is scanned in a rather naive manner for 'git clone xxxx',
'hg clone xxxx', etc, and when one of these is found it's assumed
that's the information we want. Returns repotype, address, or
None, reason
"""
if not url.startswith('http'):
return (None, _('{url} does not start with "http"!'.format(url=url)))
req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
if req.getcode() != 200:
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
page = req.read().decode(req.headers.get_content_charset())
# Works for BitBucket
m = re.search('data-fetch-url="(.*)"', page)
if m is not None:
repo = m.group(1)
if repo.endswith('.git'):
return ('git', repo)
return ('hg', repo)
# Works for BitBucket (obsolete)
index = page.find('hg clone')
if index != -1:
repotype = 'hg'
repo = page[index + 9:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
# Works for BitBucket (obsolete)
index = page.find('git clone')
if index != -1:
repotype = 'git'
repo = page[index + 10:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
return (None, _("No information found.") + page)
def get_app_from_url(url):
"""Guess basic app metadata from the URL.
The URL must include a network hostname, unless it is an lp:,
file:, or git/ssh URL. This throws ValueError on bad URLs to
match urlparse().
"""
parsed = urllib.parse.urlparse(url)
invalid_url = False
if not parsed.scheme or not parsed.path:
invalid_url = True
app = fdroidserver.metadata.App()
app.Repo = url
if url.startswith('git://') or url.startswith('git@'):
app.RepoType = 'git'
elif parsed.netloc == 'github.com':
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'gitlab.com' or parsed.netloc == 'framagit.org':
# git can be fussy with gitlab URLs unless they end in .git
if url.endswith('.git'):
url = url[:-4]
app.Repo = url + '.git'
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'notabug.org':
if url.endswith('.git'):
url = url[:-4]
app.Repo = url + '.git'
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'bitbucket.org':
if url.endswith('/'):
url = url[:-1]
app.SourceCode = url + '/src'
app.IssueTracker = url + '/issues'
# Figure out the repo type and adddress...
app.RepoType, app.Repo = getrepofrompage(url)
elif parsed.netloc == 'codeberg.org':
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif url.startswith('https://') and url.endswith('.git'):
app.RepoType = 'git'
if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
invalid_url = True
if invalid_url:
raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
if not app.RepoType:
raise FDroidException("Unable to determine vcs type. " + app.Repo)
return app
def parse_srclib_spec(spec):
if type(spec) != str:
@ -4609,10 +4492,3 @@ NDKS = [
"url": "https://dl.google.com/android/repository/android-ndk-r25b-linux.zip"
}
]
def handle_retree_error_on_windows(function, path, excinfo):
"""Python can't remove a readonly file on Windows so chmod first."""
if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError:
os.chmod(path, stat.S_IWRITE)
function(path)

132
fdroidserver/import.py → fdroidserver/import_subcommand.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python3
#
# import.py - part of the FDroid server tools
# import_subcommand.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
#
@ -18,6 +18,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import configparser
import os
import re
import stat
import urllib
import git
import json
import shutil
@ -42,7 +47,11 @@ config = None
options = None
# WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py!
def handle_retree_error_on_windows(function, path, excinfo):
"""Python can't remove a readonly file on Windows so chmod first."""
if function in (os.unlink, os.rmdir, os.remove) and excinfo[0] == PermissionError:
os.chmod(path, stat.S_IWRITE)
function(path)
def clone_to_tmp_dir(app):
@ -52,13 +61,128 @@ def clone_to_tmp_dir(app):
tmp_dir = tmp_dir / 'importer'
if tmp_dir.exists():
shutil.rmtree(str(tmp_dir), onerror=common.handle_retree_error_on_windows)
shutil.rmtree(str(tmp_dir), onerror=handle_retree_error_on_windows)
vcs = common.getvcs(app.RepoType, app.Repo, tmp_dir)
vcs.gotorevision(options.rev)
return tmp_dir
def getrepofrompage(url):
"""Get the repo type and address from the given web page.
The page is scanned in a rather naive manner for 'git clone xxxx',
'hg clone xxxx', etc, and when one of these is found it's assumed
that's the information we want. Returns repotype, address, or
None, reason
"""
if not url.startswith('http'):
return (None, _('{url} does not start with "http"!'.format(url=url)))
req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
if req.getcode() != 200:
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
page = req.read().decode(req.headers.get_content_charset())
# Works for BitBucket
m = re.search('data-fetch-url="(.*)"', page)
if m is not None:
repo = m.group(1)
if repo.endswith('.git'):
return ('git', repo)
return ('hg', repo)
# Works for BitBucket (obsolete)
index = page.find('hg clone')
if index != -1:
repotype = 'hg'
repo = page[index + 9:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
# Works for BitBucket (obsolete)
index = page.find('git clone')
if index != -1:
repotype = 'git'
repo = page[index + 10:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
return (None, _("No information found.") + page)
def get_app_from_url(url):
"""Guess basic app metadata from the URL.
The URL must include a network hostname, unless it is an lp:,
file:, or git/ssh URL. This throws ValueError on bad URLs to
match urlparse().
"""
parsed = urllib.parse.urlparse(url)
invalid_url = False
if not parsed.scheme or not parsed.path:
invalid_url = True
app = metadata.App()
app.Repo = url
if url.startswith('git://') or url.startswith('git@'):
app.RepoType = 'git'
elif parsed.netloc == 'github.com':
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'gitlab.com' or parsed.netloc == 'framagit.org':
# git can be fussy with gitlab URLs unless they end in .git
if url.endswith('.git'):
url = url[:-4]
app.Repo = url + '.git'
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'notabug.org':
if url.endswith('.git'):
url = url[:-4]
app.Repo = url + '.git'
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'bitbucket.org':
if url.endswith('/'):
url = url[:-1]
app.SourceCode = url + '/src'
app.IssueTracker = url + '/issues'
# Figure out the repo type and adddress...
app.RepoType, app.Repo = getrepofrompage(url)
elif parsed.netloc == 'codeberg.org':
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif url.startswith('https://') and url.endswith('.git'):
app.RepoType = 'git'
if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
invalid_url = True
if invalid_url:
raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
if not app.RepoType:
raise FDroidException("Unable to determine vcs type. " + app.Repo)
return app
def check_for_kivy_buildozer(tmp_importer_dir, app, build):
versionCode = None
buildozer_spec = tmp_importer_dir / 'buildozer.spec'
@ -151,7 +275,7 @@ def main():
break
write_local_file = True
elif options.url:
app = common.get_app_from_url(options.url)
app = get_app_from_url(options.url)
tmp_importer_dir = clone_to_tmp_dir(app)
# TODO: Python3.6: Should accept path-like
git_repo = git.Repo(str(tmp_importer_dir))

13
tests/common.TestCase
View File

@ -1419,19 +1419,6 @@ class CommonTest(unittest.TestCase):
with self.assertRaises(MetaDataException):
self.assertEqual(fdroidserver.common.parse_srclib_spec('@multi@at-signs@'))
def test_bad_urls(self):
for url in (
'asdf',
'file://thing.git',
'https:///github.com/my/project',
'git:///so/many/slashes',
'ssh:/notabug.org/missing/a/slash',
'git:notabug.org/missing/some/slashes',
'https//github.com/bar/baz',
):
with self.assertRaises(ValueError):
fdroidserver.common.get_app_from_url(url)
def test_remove_signing_keys(self):
testdir = tempfile.mkdtemp(
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir

27
tests/import_proxy.py
View File

@ -1,27 +0,0 @@
# workaround the syntax error from: import fdroidserver.import
import inspect
import sys
from pathlib import Path
localmodule = Path(__file__).resolve().parent.parent
print('localmodule: ' + str(localmodule))
if localmodule not in sys.path:
sys.path.insert(0, str(localmodule))
class Options:
def __init__(self):
self.rev = None
self.subdir = None
module = __import__('fdroidserver.import')
for name, obj in inspect.getmembers(module):
if name == 'import':
clone_to_tmp_dir = obj.clone_to_tmp_dir
obj.options = Options()
options = obj.options
break
globals().update(vars(module))

29
tests/import.TestCase → tests/import_subcommand.TestCase
View File

@ -15,15 +15,13 @@ from pathlib import Path
import requests
from testcommon import TmpCwd
# work around the syntax error from: import fdroidserver.import
import import_proxy
localmodule = Path(__file__).resolve().parent.parent
print('localmodule: ' + str(localmodule))
if localmodule not in sys.path:
sys.path.insert(0, str(localmodule))
import fdroidserver.common
import fdroidserver.import_subcommand
import fdroidserver.metadata
@ -37,6 +35,8 @@ class ImportTest(unittest.TestCase):
self.tmpdir.mkdir(exist_ok=True)
# TODO: Python3.6: Accepts a path-like object.
os.chdir(str(self.basedir))
fdroidserver.import_subcommand.options = mock.Mock()
fdroidserver.import_subcommand.options.rev = None
def test_import_gitlab(self):
# FDroidPopen needs some config to work
@ -51,8 +51,8 @@ class ImportTest(unittest.TestCase):
print('Skipping ImportTest!')
return
app = fdroidserver.common.get_app_from_url(url)
import_proxy.clone_to_tmp_dir(app)
app = fdroidserver.import_subcommand.get_app_from_url(url)
fdroidserver.import_subcommand.clone_to_tmp_dir(app)
self.assertEqual(app.RepoType, 'git')
self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git')
@ -88,13 +88,13 @@ class ImportTest(unittest.TestCase):
# TODO: Python3.6: Accepts a path-like object.
shutil.rmtree(
str(tmp_importer),
onerror=fdroidserver.common.handle_retree_error_on_windows,
onerror=fdroidserver.import_subcommand.handle_retree_error_on_windows,
)
shutil.copytree(
str(self.basedir / 'source-files' / appid), str(tmp_importer)
)
app = fdroidserver.common.get_app_from_url(url)
app = fdroidserver.import_subcommand.get_app_from_url(url)
with mock.patch(
'fdroidserver.common.getvcs',
lambda a, b, c: fdroidserver.common.vcs(url, testdir),
@ -103,7 +103,7 @@ class ImportTest(unittest.TestCase):
), mock.patch(
'shutil.rmtree', lambda a, onerror=None: None
):
build_dir = import_proxy.clone_to_tmp_dir(app)
build_dir = fdroidserver.import_subcommand.clone_to_tmp_dir(app)
self.assertEqual('git', app.RepoType)
self.assertEqual(url, app.Repo)
self.assertEqual(url, app.SourceCode)
@ -119,6 +119,19 @@ class ImportTest(unittest.TestCase):
self.assertEqual(vc, versionCode)
self.assertEqual(appid, package)
def test_bad_urls(self):
for url in (
'asdf',
'file://thing.git',
'https:///github.com/my/project',
'git:///so/many/slashes',
'ssh:/notabug.org/missing/a/slash',
'git:notabug.org/missing/some/slashes',
'https//github.com/bar/baz',
):
with self.assertRaises(ValueError):
fdroidserver.import_subcommand.get_app_from_url(url)
if __name__ == "__main__":
parser = optparse.OptionParser()