Merge branch 'fdroid-yml-builds' into 'master'

builds straight from source repo using .fdroid.yml

The overarching theme of the merge request is allowing _.fdroid.yml_ to be included in an app's source repo, then letting `fdroid build` build the app straight out of the git repo without requiring a setup like _fdroiddata_ (e.g. _config.py_, _metadata/packagename.txt_, etc.).  _fdroiddata_ repos can then include source repos with a _.fdroid.yml_ by having _metadata/packagename.txt_ that includes just:

```
Repo Type:git
Repo:https://gitlab.com/upstream/app.git
```

Any other metadata fields that are included in _metadata/packagename.txt_ will override what is in _.fdroid.yml_, giving the repo manager the final say about what is included in their repo.  This setup provides a number of benefits:

* CI systems like jenkins, travis, gitlab-ci can build from _.fdroid.yml_
* very easy to start building apps using `fdroid build`, no separate repo needed
* some maintenance can be offloaded to the upstream dev

See merge request !184

.gitignore

@@ -16,7 +16,7 @@ docs/html/
 
 # files generated by tests
 tmp/
-tests/repo/icons*
+/tests/repo/icons*
 
 # files used in manual testing
 /config.py
@@ -24,6 +24,12 @@ tests/repo/icons*
 /logs/
 /metadata/
 makebuildserver.config.py
+/tests/.fdroid.keypass.txt
+/tests/.fdroid.keystorepass.txt
 /tests/config.py
 /tests/fdroid-icon.png
+/tests/keystore.jks
+/tests/OBBMainPatchCurrent.apk
+/tests/OBBMainTwoVersions.apk
+/tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk
 /unsigned/

fdroidserver/build.py

@@ -952,6 +952,40 @@ def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extl
     return True
 
 
+def get_android_tools_versions(sdk_path, ndk_path=None):
+    '''get a list of the versions of all installed Android SDK/NDK components'''
+
+    if sdk_path[-1] != '/':
+        sdk_path += '/'
+    components = []
+    if ndk_path:
+        ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
+        if os.path.isfile(ndk_release_txt):
+            with open(ndk_release_txt, 'r') as fp:
+                components.append((os.path.basename(ndk_path), fp.read()[:-1]))
+
+    pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
+    for root, dirs, files in os.walk(sdk_path):
+        if 'source.properties' in files:
+            source_properties = os.path.join(root, 'source.properties')
+            with open(source_properties, 'r') as fp:
+                m = pattern.search(fp.read())
+                if m:
+                    components.append((root[len(sdk_path):], m.group(1)))
+
+    return components
+
+
+def get_android_tools_version_log(sdk_path, ndk_path):
+    '''get a list of the versions of all installed Android SDK/NDK components'''
+    log = ''
+    components = get_android_tools_versions(sdk_path, ndk_path)
+    for name, version in sorted(components):
+        log += '* ' + name + ' (' + version + ')\n'
+
+    return log
+
+
 def parse_commandline():
     """Parse the command line. Returns options, parser."""
 
@@ -1066,9 +1100,10 @@ def main():
     extlib_dir = os.path.join(build_dir, 'extlib')
 
     # Read all app and srclib metadata
-    allapps = metadata.read_metadata(xref=not options.onserver)
-
+    pkgs = common.read_pkg_args(options.appid, True)
+    allapps = metadata.read_metadata(not options.onserver, pkgs)
     apps = common.read_app_args(options.appid, allapps, True)
+
     for appid, app in list(apps.items()):
         if (app.Disabled and not options.force) or not app.RepoType or not app.builds:
             del apps[appid]
@@ -1099,22 +1134,15 @@ def main():
 
         for build in app.builds:
             wikilog = None
+            tools_version_log = '== Installed Android Tools ==\n\n'
+            tools_version_log += get_android_tools_version_log(config['sdk_path'], build.ndk_path())
             try:
 
                 # For the first build of a particular app, we need to set up
                 # the source repo. We can reuse it on subsequent builds, if
                 # there are any.
                 if first:
-                    if app.RepoType == 'srclib':
-                        build_dir = os.path.join('build', 'srclib', app.Repo)
-                    else:
-                        build_dir = os.path.join('build', appid)
-
-                    # Set up vcs interface and make sure we have the latest code...
-                    logging.debug("Getting {0} vcs interface for {1}"
-                                  .format(app.RepoType, app.Repo))
-                    vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
-
+                    vcs, build_dir = common.setup_vcs(app)
                     first = False
 
                 logging.debug("Checking " + build.version)
@@ -1150,6 +1178,12 @@ def main():
                 wikilog = str(vcse)
             except FDroidException as e:
                 with open(os.path.join(log_dir, appid + '.log'), 'a+') as f:
+                    f.write('\n\n============================================================\n')
+                    f.write('versionCode: %s\nversionName: %s\ncommit: %s\n' %
+                            (build.vercode, build.version, build.commit))
+                    f.write('Build completed at '
+                            + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + '\n')
+                    f.write('\n' + tools_version_log + '\n')
                     f.write(str(e))
                 logging.error("Could not build app %s: %s" % (appid, e))
                 if options.stop:
@@ -1164,6 +1198,9 @@ def main():
                 failed_apps[appid] = e
                 wikilog = str(e)
 
+            if wikilog:
+                wikilog = tools_version_log + '\n\n' + wikilog
+
             if options.wiki and wikilog:
                 try:
                     # Write a page with the last build log for this version code

fdroidserver/common.py

@@ -358,9 +358,11 @@ def get_local_metadata_files():
     return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
 
 
-# Given the arguments in the form of multiple appid:[vc] strings, this returns
-# a dictionary with the set of vercodes specified for each package.
 def read_pkg_args(args, allow_vercodes=False):
+    """
+    Given the arguments in the form of multiple appid:[vc] strings, this returns
+    a dictionary with the set of vercodes specified for each package.
+    """
 
     vercodes = {}
     if not args:
@@ -380,9 +382,11 @@ def read_pkg_args(args, allow_vercodes=False):
     return vercodes
 
 
-# On top of what read_pkg_args does, this returns the whole app metadata, but
-# limiting the builds list to the builds matching the vercodes specified.
 def read_app_args(args, allapps, allow_vercodes=False):
+    """
+    On top of what read_pkg_args does, this returns the whole app metadata, but
+    limiting the builds list to the builds matching the vercodes specified.
+    """
 
     vercodes = read_pkg_args(args, allow_vercodes)
 
@@ -482,6 +486,31 @@ def getcvname(app):
     return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
 
 
+def get_build_dir(app):
+    '''get the dir that this app will be built in'''
+
+    if app.RepoType == 'srclib':
+        return os.path.join('build', 'srclib', app.Repo)
+
+    return os.path.join('build', app.id)
+
+
+def setup_vcs(app):
+    '''checkout code from VCS and return instance of vcs and the build dir'''
+    build_dir = get_build_dir(app)
+
+    # Set up vcs interface and make sure we have the latest code...
+    logging.debug("Getting {0} vcs interface for {1}"
+                  .format(app.RepoType, app.Repo))
+    if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
+        remote = os.getcwd()
+    else:
+        remote = app.Repo
+    vcs = getvcs(app.RepoType, remote, build_dir)
+
+    return vcs, build_dir
+
+
 def getvcs(vcstype, remote, local):
     if vcstype == 'git':
         return vcs_git(remote, local)
@@ -979,8 +1008,8 @@ def retrieve_string_singleline(app_dir, string, xmlfiles=None):
     return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
 
 
-# Return list of existing files that will be used to find the highest vercode
 def manifest_paths(app_dir, flavours):
+    '''Return list of existing files that will be used to find the highest vercode'''
 
     possible_manifests = \
         [os.path.join(app_dir, 'AndroidManifest.xml'),
@@ -997,8 +1026,8 @@ def manifest_paths(app_dir, flavours):
     return [path for path in possible_manifests if os.path.isfile(path)]
 
 
-# Retrieve the package name. Returns the name, or None if not found.
 def fetch_real_name(app_dir, flavours):
+    '''Retrieve the package name. Returns the name, or None if not found.'''
     for path in manifest_paths(app_dir, flavours):
         if not has_extension(path, 'xml') or not os.path.isfile(path):
             continue
@@ -1070,10 +1099,12 @@ def app_matches_packagename(app, package):
     return appid == package
 
 
-# Extract some information from the AndroidManifest.xml at the given path.
-# Returns (version, vercode, package), any or all of which might be None.
-# All values returned are strings.
 def parse_androidmanifests(paths, app):
+    """
+    Extract some information from the AndroidManifest.xml at the given path.
+    Returns (version, vercode, package), any or all of which might be None.
+    All values returned are strings.
+    """
 
     ignoreversions = app.UpdateCheckIgnore
     ignoresearch = re.compile(ignoreversions).search if ignoreversions else None

fdroidserver/metadata.py

@@ -152,24 +152,30 @@ class App():
         self.lastupdated = None
         self._modified = set()
 
-    # Translates human-readable field names to attribute names, e.g.
-    # 'Auto Name' to 'AutoName'
     @classmethod
     def field_to_attr(cls, f):
+        """
+        Translates human-readable field names to attribute names, e.g.
+        'Auto Name' to 'AutoName'
+        """
         return f.replace(' ', '')
 
-    # Translates attribute names to human-readable field names, e.g.
-    # 'AutoName' to 'Auto Name'
     @classmethod
     def attr_to_field(cls, k):
+        """
+        Translates attribute names to human-readable field names, e.g.
+        'AutoName' to 'Auto Name'
+        """
         if k in app_fields:
             return k
         f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
         return f
 
-    # Constructs an old-fashioned dict with the human-readable field
-    # names. Should only be used for tests.
     def field_dict(self):
+        """
+        Constructs an old-fashioned dict with the human-readable field
+        names. Should only be used for tests.
+        """
         d = {}
         for k, v in self.__dict__.items():
             if k == 'builds':
@@ -182,23 +188,23 @@ class App():
                 d[f] = v
         return d
 
-    # Gets the value associated to a field name, e.g. 'Auto Name'
     def get_field(self, f):
+        """Gets the value associated to a field name, e.g. 'Auto Name'"""
         if f not in app_fields:
             warn_or_exception('Unrecognised app field: ' + f)
         k = App.field_to_attr(f)
         return getattr(self, k)
 
-    # Sets the value associated to a field name, e.g. 'Auto Name'
     def set_field(self, f, v):
+        """Sets the value associated to a field name, e.g. 'Auto Name'"""
         if f not in app_fields:
             warn_or_exception('Unrecognised app field: ' + f)
         k = App.field_to_attr(f)
         self.__dict__[k] = v
         self._modified.add(k)
 
-    # Appends to the value associated to a field name, e.g. 'Auto Name'
     def append_field(self, f, v):
+        """Appends to the value associated to a field name, e.g. 'Auto Name'"""
         if f not in app_fields:
             warn_or_exception('Unrecognised app field: ' + f)
         k = App.field_to_attr(f)
@@ -207,8 +213,8 @@ class App():
         else:
             self.__dict__[k].append(v)
 
-    # Like dict.update(), but using human-readable field names
     def update_fields(self, d):
+        '''Like dict.update(), but using human-readable field names'''
         for f, v in d.items():
             if f == 'builds':
                 for b in v:
@@ -218,6 +224,21 @@ class App():
             else:
                 self.set_field(f, v)
 
+    def update(self, d):
+        '''Like dict.update()'''
+        for k, v in d.__dict__.items():
+            if k == '_modified':
+                continue
+            elif k == 'builds':
+                for b in v:
+                    build = Build()
+                    del(b.__dict__['_modified'])
+                    build.update_flags(b.__dict__)
+                    self.builds.append(build)
+            elif v:
+                self.__dict__[k] = v
+                self._modified.add(k)
+
 
 TYPE_UNKNOWN = 0
 TYPE_OBSOLETE = 1
@@ -772,9 +793,13 @@ def read_srclibs():
         srclibs[srclibname] = parse_srclib(metadatapath)
 
 
-# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
-# returned by the parse_txt_metadata function.
-def read_metadata(xref=True):
+def read_metadata(xref=True, check_vcs=[]):
+    """
+    Read all metadata. Returns a list of 'app' objects (which are dictionaries as
+    returned by the parse_txt_metadata function.
+
+    check_vcs is the list of packageNames to check for .fdroid.yml in source
+    """
 
     # Always read the srclibs before the apps, since they can use a srlib as
     # their source repository.
@@ -801,7 +826,7 @@ def read_metadata(xref=True):
         packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
         if packageName in apps:
             warn_or_exception("Found multiple metadata files for " + packageName)
-        app = parse_metadata(metadatapath)
+        app = parse_metadata(metadatapath, packageName in check_vcs)
         check_metadata(app)
         apps[app.id] = app
 
@@ -904,6 +929,11 @@ def post_metadata_parse(app):
             elif ftype == TYPE_STRING:
                 if isinstance(v, bool) and v:
                     build.__dict__[k] = 'yes'
+            elif ftype == TYPE_LIST:
+                if isinstance(v, bool) and v:
+                    build.__dict__[k] = ['yes']
+                elif isinstance(v, str):
+                    build.__dict__[k] = [v]
 
     if not app.Description:
         app.Description = 'No description available'
@@ -949,7 +979,9 @@ def _decode_bool(s):
     warn_or_exception("Invalid bool '%s'" % s)
 
 
-def parse_metadata(metadatapath):
+def parse_metadata(metadatapath, check_vcs=False):
+    '''parse metadata file, optionally checking the git repo for metadata first'''
+
     _, ext = fdroidserver.common.get_extension(metadatapath)
     accepted = fdroidserver.common.config['accepted_formats']
     if ext not in accepted:
@@ -958,7 +990,11 @@ def parse_metadata(metadatapath):
 
     app = App()
     app.metadatapath = metadatapath
-    app.id, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+    name, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+    if name == '.fdroid':
+        check_vcs = False
+    else:
+        app.id = name
 
     with open(metadatapath, 'r', encoding='utf-8') as mf:
         if ext == 'txt':
@@ -972,7 +1008,28 @@ def parse_metadata(metadatapath):
         else:
             warn_or_exception('Unknown metadata format: %s' % metadatapath)
 
+    if check_vcs and app.Repo:
+        build_dir = fdroidserver.common.get_build_dir(app)
+        metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
+        if not os.path.isfile(metadata_in_repo):
+            vcs, build_dir = fdroidserver.common.setup_vcs(app)
+            vcs.gotorevision('HEAD')  # HEAD since we can't know where else to go
+        if os.path.isfile(metadata_in_repo):
+            logging.debug('Including metadata from ' + metadata_in_repo)
+            app.update(parse_metadata(metadata_in_repo))
+
     post_metadata_parse(app)
+
+    if not app.id:
+        if app.builds:
+            build = app.builds[-1]
+            if build.subdir:
+                root_dir = build.subdir
+            else:
+                root_dir = '.'
+            paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
+            _, _, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
+
     return app
 
 

tests/import.TestCase

@@ -25,12 +25,13 @@ class ImportTest(unittest.TestCase):
     '''fdroid import'''
 
     def test_import_gitlab(self):
+        os.chdir(os.path.dirname(__file__))
         # FDroidPopen needs some config to work
         config = dict()
         fdroidserver.common.fill_config_defaults(config)
         fdroidserver.common.config = config
 
-        url = 'https://gitlab.com/fdroid/fdroidclient'
+        url = 'https://gitlab.com/eighthave/ci-test-app'
         r = requests.head(url)
         if r.status_code != 200:
             print("ERROR", url, 'unreachable (', r.status_code, ')')
@@ -41,8 +42,8 @@ class ImportTest(unittest.TestCase):
         app.UpdateCheckMode = "Tags"
         root_dir, src_dir = import_proxy.get_metadata_from_url(app, url)
         self.assertEqual(app.RepoType, 'git')
-        self.assertEqual(app.WebSite, 'https://gitlab.com/fdroid/fdroidclient')
-        self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/fdroidclient.git')
+        self.assertEqual(app.WebSite, 'https://gitlab.com/eighthave/ci-test-app')
+        self.assertEqual(app.Repo, 'https://gitlab.com/eighthave/ci-test-app.git')
 
 
 if __name__ == "__main__":

tests/metadata/info.guardianproject.checkey.txt

@@ -0,0 +1,21 @@
+Categories:Development,GuardianProject
+License:GPLv3
+Web Site:https://dev.guardianproject.info/projects/checkey
+Source Code:https://github.com/guardianproject/checkey
+Issue Tracker:https://dev.guardianproject.info/projects/checkey/issues
+Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+
+Auto Name:Checkey
+Summary:Info on local apps
+Description:
+Checkey is a utility for getting information about the APKs that are installed
+on your device. Starting with a list of all of the apps that you have
+installed on your device, it will show you the APK signature with a single
+touch, and provides links to virustotal.com and androidobservatory.org to
+easily access the profiles of that APK. It will also let you export the
+signing certificate and generate ApkSignaturePin pin files for use with the
+TrustedIntents library.
+.
+
+
+Current Version Code:9999999

tests/metadata/org.fdroid.ci.test.app.txt

@@ -0,0 +1,2 @@
+Repo Type:git
+Repo:https://gitlab.com/eighthave/ci-test-app

tests/metadata/raw.template.txt

@@ -0,0 +1,9 @@
+License:Unknown
+Web Site:
+Source Code:
+Issue Tracker:
+Changelog:
+Summary:Template
+Description:
+Template
+.

tests/run-tests

@@ -100,6 +100,8 @@ echo_header "test python getsig replacement"
 
 cd $WORKSPACE/tests/getsig
 ./make.sh
+
+cd $WORKSPACE/tests
 for testcase in $WORKSPACE/tests/*.TestCase; do
     $testcase
 done
@@ -138,6 +140,19 @@ $fdroid readmeta
 $fdroid update
 
 
+#------------------------------------------------------------------------------#
+echo_header 'run `fdroid build` in fresh git checkout from import.TestCase'
+
+cd $WORKSPACE/tests/tmp/importer
+if [ -d $ANDROID_HOME/platforms/android-23 ]; then
+    echo "build_tools = '`ls -1 $ANDROID_HOME/build-tools/ | sort -n | tail -1`'" > config.py
+    echo "force_build_tools = True" >> config.py
+    $fdroid build --verbose org.fdroid.ci.test.app:300
+else
+    echo 'WARNING: Skipping `fdroid build` test since android-23 is missing!'
+fi
+
+
 #------------------------------------------------------------------------------#
 echo_header "copy tests/repo, generate java/gpg keys, update, and gpgsign"