From c80c1bf0170f928825b94b9ed103311bd6fba5b8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 5 Aug 2015 20:42:58 +0200 Subject: [PATCH 1/4] import: split out URL handling into its own function This is preparation to add other import methods, like checking if the command was run in a currently checked out git repo. --- fdroidserver/import.py | 112 ++++++++++++++++++++------------------- fdroidserver/metadata.py | 7 ++- 2 files changed, 63 insertions(+), 56 deletions(-) diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 9323f9bd..5c6fbe48 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -70,71 +70,43 @@ config = None options = None -def main(): - - global config, options - - # Parse command line... - parser = ArgumentParser() - parser.add_argument("-v", "--verbose", action="store_true", default=False, - help="Spew out even more information than normal") - parser.add_argument("-q", "--quiet", action="store_true", default=False, - help="Restrict output to warnings and errors") - parser.add_argument("-u", "--url", default=None, - help="Project URL to import from.") - parser.add_argument("-s", "--subdir", default=None, - help="Path to main android project subdirectory, if not in root.") - parser.add_argument("--rev", default=None, - help="Allows a different revision (or git branch) to be specified for the initial import") - options = parser.parse_args() - - config = common.read_config(options) - - if not options.url: - logging.error("Specify project url.") - sys.exit(1) - url = options.url +def get_metadata_from_url(app, url): tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): logging.info("Creating temporary directory") os.makedirs(tmp_dir) - # Get all apps... - apps = metadata.read_metadata() - # Figure out what kind of project it is... projecttype = None - issuetracker = None - license = None - website = url # by default, we might override it + app['Web Site'] = url # by default, we might override it if url.startswith('git://'): projecttype = 'git' repo = url repotype = 'git' - sourcecode = "" - website = "" + app['Source Code'] = "" + app['Web Site'] = "" elif url.startswith('https://github.com'): projecttype = 'github' repo = url repotype = 'git' - sourcecode = url - issuetracker = url + '/issues' - website = "" + app['Source Code'] = url + app['issuetracker'] = url + '/issues' + app['Web Site'] = "" elif url.startswith('https://gitlab.com/'): projecttype = 'gitlab' repo = url repotype = 'git' - sourcecode = url + '/tree/HEAD' - issuetracker = url + '/issues' + app['Source Code'] = url + '/tree/HEAD' + app['issuetracker'] = url + '/issues' elif url.startswith('https://bitbucket.org/'): if url.endswith('/'): url = url[:-1] projecttype = 'bitbucket' - sourcecode = url + '/src' - issuetracker = url + '/issues' + app['Source Code'] = url + '/src' + app['issuetracker'] = url + '/issues' # Figure out the repo type and adddress... - repotype, repo = getrepofrompage(sourcecode) + repotype, repo = getrepofrompage(app['Source Code']) if not repotype: logging.error("Unable to determine vcs type. " + repo) sys.exit(1) @@ -166,6 +138,49 @@ def main(): else: root_dir = src_dir + app['Repo Type'] = repotype + app['Repo'] = repo + + return root_dir, src_dir + + +config = None +options = None + + +def main(): + + global config, options + + # Parse command line... + parser = ArgumentParser() + parser.add_argument("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + parser.add_argument("-q", "--quiet", action="store_true", default=False, + help="Restrict output to warnings and errors") + parser.add_argument("-u", "--url", default=None, + help="Project URL to import from.") + parser.add_argument("-s", "--subdir", default=None, + help="Path to main android project subdirectory, if not in root.") + parser.add_argument("--rev", default=None, + help="Allows a different revision (or git branch) to be specified for the initial import") + options = parser.parse_args() + + config = common.read_config(options) + + apps = metadata.read_metadata() + package, app = metadata.get_default_app_info_list(apps) + app['Update Check Mode'] = "Tags" + + if os.path.isdir('.git'): + if options.url: + app['Web Site'] = options.url + elif options.url: + root_dir, src_dir = get_metadata_from_url(app, options.url) + else: + logging.error("Specify project url.") + sys.exit(1) + # Extract some information... paths = common.manifest_paths(root_dir, []) if paths: @@ -197,18 +212,6 @@ def main(): logging.error("Package " + package + " already exists") sys.exit(1) - # Construct the metadata... - app = metadata.parse_txt_metadata(None)[1] - app['Web Site'] = website - app['Source Code'] = sourcecode - if issuetracker: - app['Issue Tracker'] = issuetracker - if license: - app['License'] = license - app['Repo Type'] = repotype - app['Repo'] = repo - app['Update Check Mode'] = "Tags" - # Create a build line... build = {} build['version'] = version or '?' @@ -230,9 +233,10 @@ def main(): # Keep the repo directory to save bandwidth... if not os.path.exists('build'): os.mkdir('build') - shutil.move(src_dir, os.path.join('build', package)) + if src_dir is not None: + shutil.move(src_dir, os.path.join('build', package)) with open('build/.fdroidvcs-' + package, 'w') as f: - f.write(repotype + ' ' + repo) + f.write(app['Repo Type'] + ' ' + app['Repo']) metadatapath = os.path.join('metadata', package + '.txt') metadata.write_metadata(metadatapath, app) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 303aab6c..2afb590b 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -579,8 +579,11 @@ def split_list_values(s): return [v for v in l if v] -def get_default_app_info_list(apps, metadatapath): - appid = os.path.splitext(os.path.basename(metadatapath))[0] +def get_default_app_info_list(apps, metadatapath=None): + if metadatapath is None: + appid = None + else: + appid = os.path.splitext(os.path.basename(metadatapath))[0] if appid in apps: logging.critical("'%s' is a duplicate! '%s' is already provided by '%s'" % (metadatapath, appid, apps[appid]['metadatapath'])) From 48397bfa3cc1d8a9fec8b1de991fd72d9e2f121c Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 5 Aug 2015 21:54:25 +0200 Subject: [PATCH 2/4] import: append .git to gitlab URLs, git sometimes freaks out otherwise From my experience, gitlab URLs need to end in .git for git to always be happy using them. --- fdroidserver/import.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 5c6fbe48..1debd216 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -95,7 +95,11 @@ def get_metadata_from_url(app, url): app['Web Site'] = "" elif url.startswith('https://gitlab.com/'): projecttype = 'gitlab' - repo = url + # git can be fussy with gitlab URLs unless they end in .git + if url.endswith('.git'): + repo = url + else: + repo = url + '.git' repotype = 'git' app['Source Code'] = url + '/tree/HEAD' app['issuetracker'] = url + '/issues' From 707930ce0a708ab495f865ff8904b3ba2f1b8cb5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 6 Aug 2015 00:55:16 +0200 Subject: [PATCH 3/4] add a basic test of `fdroid import` --- tests/import.TestCase | 50 +++++++++++++++++++++++++++++++++++++++++++ tests/import_proxy.py | 26 ++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100755 tests/import.TestCase create mode 100644 tests/import_proxy.py diff --git a/tests/import.TestCase b/tests/import.TestCase new file mode 100755 index 00000000..adcbe1a2 --- /dev/null +++ b/tests/import.TestCase @@ -0,0 +1,50 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +# http://www.drdobbs.com/testing/unit-testing-with-python/240165163 + +import inspect +import optparse +import os +import sys +import unittest + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +import fdroidserver.common +import fdroidserver.metadata +# work around the syntax error from: import fdroidserver.import +import import_proxy + + +class ImportTest(unittest.TestCase): + '''fdroid import''' + + def test_import_gitlab(self): + # FDroidPopen needs some config to work + fdroidserver.common.config = dict() + fdroidserver.common.config['sdk_path'] = '/fake/path/to/android-sdk' + + url = 'https://gitlab.com/fdroid/fdroidclient' + apps = dict() + appid, app = fdroidserver.metadata.get_default_app_info_list(apps) + app['Update Check Mode'] = "Tags" + root_dir, src_dir = import_proxy.get_metadata_from_url(app, url) + self.assertEquals(app['Repo Type'], 'git') + self.assertEquals(app['Web Site'], 'https://gitlab.com/fdroid/fdroidclient') + self.assertEquals(app['Repo'], 'https://gitlab.com/fdroid/fdroidclient.git') + + +if __name__ == "__main__": + parser = optparse.OptionParser() + parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") + (fdroidserver.common.options, args) = parser.parse_args(['--verbose']) + + newSuite = unittest.TestSuite() + newSuite.addTest(unittest.makeSuite(ImportTest)) + unittest.main() diff --git a/tests/import_proxy.py b/tests/import_proxy.py new file mode 100644 index 00000000..ce24a50f --- /dev/null +++ b/tests/import_proxy.py @@ -0,0 +1,26 @@ +# workaround the syntax error from: import fdroidserver.import + +import inspect +import os +import sys + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +class Options: + def __init__(self): + self.rev = None + self.subdir = None + +module = __import__('fdroidserver.import') +for name, obj in inspect.getmembers(module): + if name == 'import': + get_metadata_from_url = obj.get_metadata_from_url + obj.options = Options() + options = obj.options + break + +globals().update(vars(module)) From 491d2d75a280b2e5b6b9d9f5be40326a937458ce Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 27 Aug 2015 19:34:37 +0200 Subject: [PATCH 4/4] comment out things that are default values in examples/config.py Since these are already defaults, there is no need to set them as defaults in the default config. --- examples/config.py | 54 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/examples/config.py b/examples/config.py index fac101d8..147b7386 100644 --- a/examples/config.py +++ b/examples/config.py @@ -16,23 +16,23 @@ # } # Build tools version to be used -build_tools = "23.0.1" +# build_tools = "23.0.1" # Command or path to binary for running Ant -ant = "ant" +# ant = "ant" # Command or path to binary for running maven 3 -mvn3 = "mvn" +# mvn3 = "mvn" # Command or path to binary for running Gradle -gradle = "gradle" +# gradle = "gradle" # Set the maximum age (in days) of an index that a client should accept from # this repo. Setting it to 0 or not setting it at all disables this # functionality. If you do set this to a non-zero value, you need to ensure # that your index is updated much more frequently than the specified interval. # The same policy is applied to the archive repo, if there is one. -repo_maxage = 0 +# repo_maxage = 0 repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo" repo_name = "My First F-Droid Repo Demo" @@ -116,15 +116,15 @@ The repository of older versions of applications from the main demo repository. # keypass = "password2" # The distinguished name used for all keys. -keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US" +# keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US" # Use this to override the auto-generated key aliases with specific ones # for particular applications. Normally, just leave it empty. -keyaliases = {} -keyaliases['com.example.app'] = 'example' +# keyaliases = {} +# keyaliases['com.example.app'] = 'example' # You can also force an app to use the same key alias as another one, using # the @ prefix. -keyaliases['com.example.another.plugin'] = '@com.example.another' +# keyaliases['com.example.another.plugin'] = '@com.example.another' # The full path to the root of the repository. It must be specified in @@ -182,36 +182,36 @@ keyaliases['com.example.another.plugin'] = '@com.example.another' # nonstandardwebroot = False -# Wiki details -wiki_protocol = "http" -wiki_server = "server" -wiki_path = "/wiki/" -wiki_user = "login" -wiki_password = "1234" +# The build logs can be posted to a mediawiki instance, like on f-droid.org. +# wiki_protocol = "http" +# wiki_server = "server" +# wiki_path = "/wiki/" +# wiki_user = "login" +# wiki_password = "1234" # Only set this to true when running a repository where you want to generate # stats, and only then on the master build servers, not a development # machine. -update_stats = False +# update_stats = True # When used with stats, this is a list of IP addresses that are ignored for # calculation purposes. -stats_ignore = [] +# stats_ignore = [] # Server stats logs are retrieved from. Required when update_stats is True. -stats_server = "example.com" +# stats_server = "example.com" # User stats logs are retrieved from. Required when update_stats is True. -stats_user = "bob" +# stats_user = "bob" # Use the following to push stats to a Carbon instance: -stats_to_carbon = False -carbon_host = '0.0.0.0' -carbon_port = 2003 +# stats_to_carbon = False +# carbon_host = '0.0.0.0' +# carbon_port = 2003 # Set this to true to always use a build server. This saves specifying the # --server option on dedicated secure build server hosts. -build_server_always = False +# build_server_always = True # By default, fdroid will use YAML and the custom .txt metadata formats. It # is also possible to have metadata in JSON and XML by adding 'json' and @@ -220,7 +220,7 @@ build_server_always = False # Limit in number of characters that fields can take up # Only the fields listed here are supported, defaults shown -char_limits = { - 'Summary': 80, - 'Description': 4000, -} +# char_limits = { +# 'Summary': 80, +# 'Description': 4000, +# }