Merge branch 'import-tweaks-and-tests' into 'master'

Import tweaks and tests

This makes the code for `fdroid import` a bit more flexible and adds tests for the first time.  It also comments out options in `examples/config.py` that just mirror the defaults to make it clear that they are defaults, and help illustrate other options (this is standard procedure in default config files).

More details in the commit messages.

See merge request !76

examples/config.py

@@ -16,23 +16,23 @@
 # }
 
 # Build tools version to be used
-build_tools = "23.0.1"
+# build_tools = "23.0.1"
 
 # Command or path to binary for running Ant
-ant = "ant"
+# ant = "ant"
 
 # Command or path to binary for running maven 3
-mvn3 = "mvn"
+# mvn3 = "mvn"
 
 # Command or path to binary for running Gradle
-gradle = "gradle"
+# gradle = "gradle"
 
 # Set the maximum age (in days) of an index that a client should accept from
 # this repo. Setting it to 0 or not setting it at all disables this
 # functionality. If you do set this to a non-zero value, you need to ensure
 # that your index is updated much more frequently than the specified interval.
 # The same policy is applied to the archive repo, if there is one.
-repo_maxage = 0
+# repo_maxage = 0
 
 repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo"
 repo_name = "My First F-Droid Repo Demo"
@@ -116,15 +116,15 @@ The repository of older versions of applications from the main demo repository.
 # keypass = "password2"
 
 # The distinguished name used for all keys.
-keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"
+# keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"
 
 # Use this to override the auto-generated key aliases with specific ones
 # for particular applications. Normally, just leave it empty.
-keyaliases = {}
-keyaliases['com.example.app'] = 'example'
+# keyaliases = {}
+# keyaliases['com.example.app'] = 'example'
 # You can also force an app to use the same key alias as another one, using
 # the @ prefix.
-keyaliases['com.example.another.plugin'] = '@com.example.another'
+# keyaliases['com.example.another.plugin'] = '@com.example.another'
 
 
 # The full path to the root of the repository. It must be specified in
@@ -182,36 +182,36 @@ keyaliases['com.example.another.plugin'] = '@com.example.another'
 # nonstandardwebroot = False
 
 
-# Wiki details
-wiki_protocol = "http"
-wiki_server = "server"
-wiki_path = "/wiki/"
-wiki_user = "login"
-wiki_password = "1234"
+# The build logs can be posted to a mediawiki instance, like on f-droid.org.
+# wiki_protocol = "http"
+# wiki_server = "server"
+# wiki_path = "/wiki/"
+# wiki_user = "login"
+# wiki_password = "1234"
 
 # Only set this to true when running a repository where you want to generate
 # stats, and only then on the master build servers, not a development
 # machine.
-update_stats = False
+# update_stats = True
 
 # When used with stats, this is a list of IP addresses that are ignored for
 # calculation purposes.
-stats_ignore = []
+# stats_ignore = []
 
 # Server stats logs are retrieved from. Required when update_stats is True.
-stats_server = "example.com"
+# stats_server = "example.com"
 
 # User stats logs are retrieved from. Required when update_stats is True.
-stats_user = "bob"
+# stats_user = "bob"
 
 # Use the following to push stats to a Carbon instance:
-stats_to_carbon = False
-carbon_host = '0.0.0.0'
-carbon_port = 2003
+# stats_to_carbon = False
+# carbon_host = '0.0.0.0'
+# carbon_port = 2003
 
 # Set this to true to always use a build server. This saves specifying the
 # --server option on dedicated secure build server hosts.
-build_server_always = False
+# build_server_always = True
 
 # By default, fdroid will use YAML and the custom .txt metadata formats. It
 # is also possible to have metadata in JSON and XML by adding 'json' and
@@ -220,7 +220,7 @@ build_server_always = False
 
 # Limit in number of characters that fields can take up
 # Only the fields listed here are supported, defaults shown
-char_limits = {
-    'Summary': 80,
-    'Description': 4000,
-}
+# char_limits = {
+#     'Summary': 80,
+#     'Description': 4000,
+# }

fdroidserver/import.py

@@ -70,71 +70,47 @@ config = None
 options = None
 
 
-def main():
-
-    global config, options
-
-    # Parse command line...
-    parser = ArgumentParser()
-    parser.add_argument("-v", "--verbose", action="store_true", default=False,
-                        help="Spew out even more information than normal")
-    parser.add_argument("-q", "--quiet", action="store_true", default=False,
-                        help="Restrict output to warnings and errors")
-    parser.add_argument("-u", "--url", default=None,
-                        help="Project URL to import from.")
-    parser.add_argument("-s", "--subdir", default=None,
-                        help="Path to main android project subdirectory, if not in root.")
-    parser.add_argument("--rev", default=None,
-                        help="Allows a different revision (or git branch) to be specified for the initial import")
-    options = parser.parse_args()
-
-    config = common.read_config(options)
-
-    if not options.url:
-        logging.error("Specify project url.")
-        sys.exit(1)
-    url = options.url
+def get_metadata_from_url(app, url):
 
     tmp_dir = 'tmp'
     if not os.path.isdir(tmp_dir):
         logging.info("Creating temporary directory")
         os.makedirs(tmp_dir)
 
-    # Get all apps...
-    apps = metadata.read_metadata()
-
     # Figure out what kind of project it is...
     projecttype = None
-    issuetracker = None
-    license = None
-    website = url  # by default, we might override it
+    app['Web Site'] = url  # by default, we might override it
     if url.startswith('git://'):
         projecttype = 'git'
         repo = url
         repotype = 'git'
-        sourcecode = ""
-        website = ""
+        app['Source Code'] = ""
+        app['Web Site'] = ""
     elif url.startswith('https://github.com'):
         projecttype = 'github'
         repo = url
         repotype = 'git'
-        sourcecode = url
-        issuetracker = url + '/issues'
-        website = ""
+        app['Source Code'] = url
+        app['issuetracker'] = url + '/issues'
+        app['Web Site'] = ""
     elif url.startswith('https://gitlab.com/'):
         projecttype = 'gitlab'
-        repo = url
+        # git can be fussy with gitlab URLs unless they end in .git
+        if url.endswith('.git'):
+            repo = url
+        else:
+            repo = url + '.git'
         repotype = 'git'
-        sourcecode = url + '/tree/HEAD'
-        issuetracker = url + '/issues'
+        app['Source Code'] = url + '/tree/HEAD'
+        app['issuetracker'] = url + '/issues'
     elif url.startswith('https://bitbucket.org/'):
         if url.endswith('/'):
             url = url[:-1]
         projecttype = 'bitbucket'
-        sourcecode = url + '/src'
-        issuetracker = url + '/issues'
+        app['Source Code'] = url + '/src'
+        app['issuetracker'] = url + '/issues'
         # Figure out the repo type and adddress...
-        repotype, repo = getrepofrompage(sourcecode)
+        repotype, repo = getrepofrompage(app['Source Code'])
         if not repotype:
             logging.error("Unable to determine vcs type. " + repo)
             sys.exit(1)
@@ -166,6 +142,49 @@ def main():
     else:
         root_dir = src_dir
 
+    app['Repo Type'] = repotype
+    app['Repo'] = repo
+
+    return root_dir, src_dir
+
+
+config = None
+options = None
+
+
+def main():
+
+    global config, options
+
+    # Parse command line...
+    parser = ArgumentParser()
+    parser.add_argument("-v", "--verbose", action="store_true", default=False,
+                        help="Spew out even more information than normal")
+    parser.add_argument("-q", "--quiet", action="store_true", default=False,
+                        help="Restrict output to warnings and errors")
+    parser.add_argument("-u", "--url", default=None,
+                        help="Project URL to import from.")
+    parser.add_argument("-s", "--subdir", default=None,
+                        help="Path to main android project subdirectory, if not in root.")
+    parser.add_argument("--rev", default=None,
+                        help="Allows a different revision (or git branch) to be specified for the initial import")
+    options = parser.parse_args()
+
+    config = common.read_config(options)
+
+    apps = metadata.read_metadata()
+    package, app = metadata.get_default_app_info_list(apps)
+    app['Update Check Mode'] = "Tags"
+
+    if os.path.isdir('.git'):
+        if options.url:
+            app['Web Site'] = options.url
+    elif options.url:
+        root_dir, src_dir = get_metadata_from_url(app, options.url)
+    else:
+        logging.error("Specify project url.")
+        sys.exit(1)
+
     # Extract some information...
     paths = common.manifest_paths(root_dir, [])
     if paths:
@@ -197,18 +216,6 @@ def main():
         logging.error("Package " + package + " already exists")
         sys.exit(1)
 
-    # Construct the metadata...
-    app = metadata.parse_txt_metadata(None)[1]
-    app['Web Site'] = website
-    app['Source Code'] = sourcecode
-    if issuetracker:
-        app['Issue Tracker'] = issuetracker
-    if license:
-        app['License'] = license
-    app['Repo Type'] = repotype
-    app['Repo'] = repo
-    app['Update Check Mode'] = "Tags"
-
     # Create a build line...
     build = {}
     build['version'] = version or '?'
@@ -230,9 +237,10 @@ def main():
     # Keep the repo directory to save bandwidth...
     if not os.path.exists('build'):
         os.mkdir('build')
-    shutil.move(src_dir, os.path.join('build', package))
+    if src_dir is not None:
+        shutil.move(src_dir, os.path.join('build', package))
     with open('build/.fdroidvcs-' + package, 'w') as f:
-        f.write(repotype + ' ' + repo)
+        f.write(app['Repo Type'] + ' ' + app['Repo'])
 
     metadatapath = os.path.join('metadata', package + '.txt')
     metadata.write_metadata(metadatapath, app)

fdroidserver/metadata.py

@@ -579,8 +579,11 @@ def split_list_values(s):
     return [v for v in l if v]
 
 
-def get_default_app_info_list(apps, metadatapath):
-    appid = os.path.splitext(os.path.basename(metadatapath))[0]
+def get_default_app_info_list(apps, metadatapath=None):
+    if metadatapath is None:
+        appid = None
+    else:
+        appid = os.path.splitext(os.path.basename(metadatapath))[0]
     if appid in apps:
         logging.critical("'%s' is a duplicate! '%s' is already provided by '%s'"
                          % (metadatapath, appid, apps[appid]['metadatapath']))

tests/import.TestCase

@@ -0,0 +1,50 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
+
+import inspect
+import optparse
+import os
+import sys
+import unittest
+
+localmodule = os.path.realpath(
+    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+    sys.path.insert(0, localmodule)
+
+import fdroidserver.common
+import fdroidserver.metadata
+# work around the syntax error from: import fdroidserver.import
+import import_proxy
+
+
+class ImportTest(unittest.TestCase):
+    '''fdroid import'''
+
+    def test_import_gitlab(self):
+        # FDroidPopen needs some config to work
+        fdroidserver.common.config = dict()
+        fdroidserver.common.config['sdk_path'] = '/fake/path/to/android-sdk'
+
+        url = 'https://gitlab.com/fdroid/fdroidclient'
+        apps = dict()
+        appid, app = fdroidserver.metadata.get_default_app_info_list(apps)
+        app['Update Check Mode'] = "Tags"
+        root_dir, src_dir = import_proxy.get_metadata_from_url(app, url)
+        self.assertEquals(app['Repo Type'], 'git')
+        self.assertEquals(app['Web Site'], 'https://gitlab.com/fdroid/fdroidclient')
+        self.assertEquals(app['Repo'], 'https://gitlab.com/fdroid/fdroidclient.git')
+
+
+if __name__ == "__main__":
+    parser = optparse.OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
+
+    newSuite = unittest.TestSuite()
+    newSuite.addTest(unittest.makeSuite(ImportTest))
+    unittest.main()

tests/import_proxy.py

@@ -0,0 +1,26 @@
+# workaround the syntax error from: import fdroidserver.import
+
+import inspect
+import os
+import sys
+
+localmodule = os.path.realpath(
+    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+    sys.path.insert(0, localmodule)
+
+class Options:
+    def __init__(self):
+        self.rev = None
+        self.subdir = None
+
+module = __import__('fdroidserver.import')
+for name, obj in inspect.getmembers(module):
+    if name == 'import':
+        get_metadata_from_url = obj.get_metadata_from_url
+        obj.options = Options()
+        options = obj.options
+        break
+
+globals().update(vars(module))