diff --git a/.gitignore b/.gitignore index 8bff77fa..40e2cd2b 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ docs/html/ # files generated by tests tmp/ -tests/repo/icons* +/tests/repo/icons* # files used in manual testing /config.py @@ -24,6 +24,12 @@ tests/repo/icons* /logs/ /metadata/ makebuildserver.config.py +/tests/.fdroid.keypass.txt +/tests/.fdroid.keystorepass.txt /tests/config.py /tests/fdroid-icon.png +/tests/keystore.jks +/tests/OBBMainPatchCurrent.apk +/tests/OBBMainTwoVersions.apk +/tests/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk /unsigned/ diff --git a/fdroidserver/build.py b/fdroidserver/build.py index ac9ae552..281ad523 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -952,6 +952,40 @@ def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extl return True +def get_android_tools_versions(sdk_path, ndk_path=None): + '''get a list of the versions of all installed Android SDK/NDK components''' + + if sdk_path[-1] != '/': + sdk_path += '/' + components = [] + if ndk_path: + ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT') + if os.path.isfile(ndk_release_txt): + with open(ndk_release_txt, 'r') as fp: + components.append((os.path.basename(ndk_path), fp.read()[:-1])) + + pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE) + for root, dirs, files in os.walk(sdk_path): + if 'source.properties' in files: + source_properties = os.path.join(root, 'source.properties') + with open(source_properties, 'r') as fp: + m = pattern.search(fp.read()) + if m: + components.append((root[len(sdk_path):], m.group(1))) + + return components + + +def get_android_tools_version_log(sdk_path, ndk_path): + '''get a list of the versions of all installed Android SDK/NDK components''' + log = '' + components = get_android_tools_versions(sdk_path, ndk_path) + for name, version in sorted(components): + log += '* ' + name + ' (' + version + ')\n' + + return log + + def parse_commandline(): """Parse the command line. Returns options, parser.""" @@ -1066,9 +1100,10 @@ def main(): extlib_dir = os.path.join(build_dir, 'extlib') # Read all app and srclib metadata - allapps = metadata.read_metadata(xref=not options.onserver) - + pkgs = common.read_pkg_args(options.appid, True) + allapps = metadata.read_metadata(not options.onserver, pkgs) apps = common.read_app_args(options.appid, allapps, True) + for appid, app in list(apps.items()): if (app.Disabled and not options.force) or not app.RepoType or not app.builds: del apps[appid] @@ -1099,22 +1134,15 @@ def main(): for build in app.builds: wikilog = None + tools_version_log = '== Installed Android Tools ==\n\n' + tools_version_log += get_android_tools_version_log(config['sdk_path'], build.ndk_path()) try: # For the first build of a particular app, we need to set up # the source repo. We can reuse it on subsequent builds, if # there are any. if first: - if app.RepoType == 'srclib': - build_dir = os.path.join('build', 'srclib', app.Repo) - else: - build_dir = os.path.join('build', appid) - - # Set up vcs interface and make sure we have the latest code... - logging.debug("Getting {0} vcs interface for {1}" - .format(app.RepoType, app.Repo)) - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) - + vcs, build_dir = common.setup_vcs(app) first = False logging.debug("Checking " + build.version) @@ -1150,6 +1178,12 @@ def main(): wikilog = str(vcse) except FDroidException as e: with open(os.path.join(log_dir, appid + '.log'), 'a+') as f: + f.write('\n\n============================================================\n') + f.write('versionCode: %s\nversionName: %s\ncommit: %s\n' % + (build.vercode, build.version, build.commit)) + f.write('Build completed at ' + + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + '\n') + f.write('\n' + tools_version_log + '\n') f.write(str(e)) logging.error("Could not build app %s: %s" % (appid, e)) if options.stop: @@ -1164,6 +1198,9 @@ def main(): failed_apps[appid] = e wikilog = str(e) + if wikilog: + wikilog = tools_version_log + '\n\n' + wikilog + if options.wiki and wikilog: try: # Write a page with the last build log for this version code diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 80eb8a0e..e61d29be 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -358,9 +358,11 @@ def get_local_metadata_files(): return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') -# Given the arguments in the form of multiple appid:[vc] strings, this returns -# a dictionary with the set of vercodes specified for each package. def read_pkg_args(args, allow_vercodes=False): + """ + Given the arguments in the form of multiple appid:[vc] strings, this returns + a dictionary with the set of vercodes specified for each package. + """ vercodes = {} if not args: @@ -380,9 +382,11 @@ def read_pkg_args(args, allow_vercodes=False): return vercodes -# On top of what read_pkg_args does, this returns the whole app metadata, but -# limiting the builds list to the builds matching the vercodes specified. def read_app_args(args, allapps, allow_vercodes=False): + """ + On top of what read_pkg_args does, this returns the whole app metadata, but + limiting the builds list to the builds matching the vercodes specified. + """ vercodes = read_pkg_args(args, allow_vercodes) @@ -482,6 +486,31 @@ def getcvname(app): return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode) +def get_build_dir(app): + '''get the dir that this app will be built in''' + + if app.RepoType == 'srclib': + return os.path.join('build', 'srclib', app.Repo) + + return os.path.join('build', app.id) + + +def setup_vcs(app): + '''checkout code from VCS and return instance of vcs and the build dir''' + build_dir = get_build_dir(app) + + # Set up vcs interface and make sure we have the latest code... + logging.debug("Getting {0} vcs interface for {1}" + .format(app.RepoType, app.Repo)) + if app.RepoType == 'git' and os.path.exists('.fdroid.yml'): + remote = os.getcwd() + else: + remote = app.Repo + vcs = getvcs(app.RepoType, remote, build_dir) + + return vcs, build_dir + + def getvcs(vcstype, remote, local): if vcstype == 'git': return vcs_git(remote, local) @@ -979,8 +1008,8 @@ def retrieve_string_singleline(app_dir, string, xmlfiles=None): return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip() -# Return list of existing files that will be used to find the highest vercode def manifest_paths(app_dir, flavours): + '''Return list of existing files that will be used to find the highest vercode''' possible_manifests = \ [os.path.join(app_dir, 'AndroidManifest.xml'), @@ -997,8 +1026,8 @@ def manifest_paths(app_dir, flavours): return [path for path in possible_manifests if os.path.isfile(path)] -# Retrieve the package name. Returns the name, or None if not found. def fetch_real_name(app_dir, flavours): + '''Retrieve the package name. Returns the name, or None if not found.''' for path in manifest_paths(app_dir, flavours): if not has_extension(path, 'xml') or not os.path.isfile(path): continue @@ -1070,10 +1099,12 @@ def app_matches_packagename(app, package): return appid == package -# Extract some information from the AndroidManifest.xml at the given path. -# Returns (version, vercode, package), any or all of which might be None. -# All values returned are strings. def parse_androidmanifests(paths, app): + """ + Extract some information from the AndroidManifest.xml at the given path. + Returns (version, vercode, package), any or all of which might be None. + All values returned are strings. + """ ignoreversions = app.UpdateCheckIgnore ignoresearch = re.compile(ignoreversions).search if ignoreversions else None diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index ddfb19b5..986d240a 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -152,24 +152,30 @@ class App(): self.lastupdated = None self._modified = set() - # Translates human-readable field names to attribute names, e.g. - # 'Auto Name' to 'AutoName' @classmethod def field_to_attr(cls, f): + """ + Translates human-readable field names to attribute names, e.g. + 'Auto Name' to 'AutoName' + """ return f.replace(' ', '') - # Translates attribute names to human-readable field names, e.g. - # 'AutoName' to 'Auto Name' @classmethod def attr_to_field(cls, k): + """ + Translates attribute names to human-readable field names, e.g. + 'AutoName' to 'Auto Name' + """ if k in app_fields: return k f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k) return f - # Constructs an old-fashioned dict with the human-readable field - # names. Should only be used for tests. def field_dict(self): + """ + Constructs an old-fashioned dict with the human-readable field + names. Should only be used for tests. + """ d = {} for k, v in self.__dict__.items(): if k == 'builds': @@ -182,23 +188,23 @@ class App(): d[f] = v return d - # Gets the value associated to a field name, e.g. 'Auto Name' def get_field(self, f): + """Gets the value associated to a field name, e.g. 'Auto Name'""" if f not in app_fields: warn_or_exception('Unrecognised app field: ' + f) k = App.field_to_attr(f) return getattr(self, k) - # Sets the value associated to a field name, e.g. 'Auto Name' def set_field(self, f, v): + """Sets the value associated to a field name, e.g. 'Auto Name'""" if f not in app_fields: warn_or_exception('Unrecognised app field: ' + f) k = App.field_to_attr(f) self.__dict__[k] = v self._modified.add(k) - # Appends to the value associated to a field name, e.g. 'Auto Name' def append_field(self, f, v): + """Appends to the value associated to a field name, e.g. 'Auto Name'""" if f not in app_fields: warn_or_exception('Unrecognised app field: ' + f) k = App.field_to_attr(f) @@ -207,8 +213,8 @@ class App(): else: self.__dict__[k].append(v) - # Like dict.update(), but using human-readable field names def update_fields(self, d): + '''Like dict.update(), but using human-readable field names''' for f, v in d.items(): if f == 'builds': for b in v: @@ -218,6 +224,21 @@ class App(): else: self.set_field(f, v) + def update(self, d): + '''Like dict.update()''' + for k, v in d.__dict__.items(): + if k == '_modified': + continue + elif k == 'builds': + for b in v: + build = Build() + del(b.__dict__['_modified']) + build.update_flags(b.__dict__) + self.builds.append(build) + elif v: + self.__dict__[k] = v + self._modified.add(k) + TYPE_UNKNOWN = 0 TYPE_OBSOLETE = 1 @@ -772,9 +793,13 @@ def read_srclibs(): srclibs[srclibname] = parse_srclib(metadatapath) -# Read all metadata. Returns a list of 'app' objects (which are dictionaries as -# returned by the parse_txt_metadata function. -def read_metadata(xref=True): +def read_metadata(xref=True, check_vcs=[]): + """ + Read all metadata. Returns a list of 'app' objects (which are dictionaries as + returned by the parse_txt_metadata function. + + check_vcs is the list of packageNames to check for .fdroid.yml in source + """ # Always read the srclibs before the apps, since they can use a srlib as # their source repository. @@ -801,7 +826,7 @@ def read_metadata(xref=True): packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) if packageName in apps: warn_or_exception("Found multiple metadata files for " + packageName) - app = parse_metadata(metadatapath) + app = parse_metadata(metadatapath, packageName in check_vcs) check_metadata(app) apps[app.id] = app @@ -904,6 +929,11 @@ def post_metadata_parse(app): elif ftype == TYPE_STRING: if isinstance(v, bool) and v: build.__dict__[k] = 'yes' + elif ftype == TYPE_LIST: + if isinstance(v, bool) and v: + build.__dict__[k] = ['yes'] + elif isinstance(v, str): + build.__dict__[k] = [v] if not app.Description: app.Description = 'No description available' @@ -949,7 +979,9 @@ def _decode_bool(s): warn_or_exception("Invalid bool '%s'" % s) -def parse_metadata(metadatapath): +def parse_metadata(metadatapath, check_vcs=False): + '''parse metadata file, optionally checking the git repo for metadata first''' + _, ext = fdroidserver.common.get_extension(metadatapath) accepted = fdroidserver.common.config['accepted_formats'] if ext not in accepted: @@ -958,7 +990,11 @@ def parse_metadata(metadatapath): app = App() app.metadatapath = metadatapath - app.id, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + name, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath)) + if name == '.fdroid': + check_vcs = False + else: + app.id = name with open(metadatapath, 'r', encoding='utf-8') as mf: if ext == 'txt': @@ -972,7 +1008,28 @@ def parse_metadata(metadatapath): else: warn_or_exception('Unknown metadata format: %s' % metadatapath) + if check_vcs and app.Repo: + build_dir = fdroidserver.common.get_build_dir(app) + metadata_in_repo = os.path.join(build_dir, '.fdroid.yml') + if not os.path.isfile(metadata_in_repo): + vcs, build_dir = fdroidserver.common.setup_vcs(app) + vcs.gotorevision('HEAD') # HEAD since we can't know where else to go + if os.path.isfile(metadata_in_repo): + logging.debug('Including metadata from ' + metadata_in_repo) + app.update(parse_metadata(metadata_in_repo)) + post_metadata_parse(app) + + if not app.id: + if app.builds: + build = app.builds[-1] + if build.subdir: + root_dir = build.subdir + else: + root_dir = '.' + paths = fdroidserver.common.manifest_paths(root_dir, build.gradle) + _, _, app.id = fdroidserver.common.parse_androidmanifests(paths, app) + return app diff --git a/tests/import.TestCase b/tests/import.TestCase index 7b846a5f..3cfdb92e 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -25,12 +25,13 @@ class ImportTest(unittest.TestCase): '''fdroid import''' def test_import_gitlab(self): + os.chdir(os.path.dirname(__file__)) # FDroidPopen needs some config to work config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config - url = 'https://gitlab.com/fdroid/fdroidclient' + url = 'https://gitlab.com/eighthave/ci-test-app' r = requests.head(url) if r.status_code != 200: print("ERROR", url, 'unreachable (', r.status_code, ')') @@ -41,8 +42,8 @@ class ImportTest(unittest.TestCase): app.UpdateCheckMode = "Tags" root_dir, src_dir = import_proxy.get_metadata_from_url(app, url) self.assertEqual(app.RepoType, 'git') - self.assertEqual(app.WebSite, 'https://gitlab.com/fdroid/fdroidclient') - self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/fdroidclient.git') + self.assertEqual(app.WebSite, 'https://gitlab.com/eighthave/ci-test-app') + self.assertEqual(app.Repo, 'https://gitlab.com/eighthave/ci-test-app.git') if __name__ == "__main__": diff --git a/tests/metadata/info.guardianproject.checkey.txt b/tests/metadata/info.guardianproject.checkey.txt new file mode 100644 index 00000000..43faaade --- /dev/null +++ b/tests/metadata/info.guardianproject.checkey.txt @@ -0,0 +1,21 @@ +Categories:Development,GuardianProject +License:GPLv3 +Web Site:https://dev.guardianproject.info/projects/checkey +Source Code:https://github.com/guardianproject/checkey +Issue Tracker:https://dev.guardianproject.info/projects/checkey/issues +Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk + +Auto Name:Checkey +Summary:Info on local apps +Description: +Checkey is a utility for getting information about the APKs that are installed +on your device. Starting with a list of all of the apps that you have +installed on your device, it will show you the APK signature with a single +touch, and provides links to virustotal.com and androidobservatory.org to +easily access the profiles of that APK. It will also let you export the +signing certificate and generate ApkSignaturePin pin files for use with the +TrustedIntents library. +. + + +Current Version Code:9999999 diff --git a/tests/metadata/net.osmand.plus.pickle b/tests/metadata/net.osmand.plus.pickle index cf9a39ec..76409710 100644 Binary files a/tests/metadata/net.osmand.plus.pickle and b/tests/metadata/net.osmand.plus.pickle differ diff --git a/tests/metadata/org.adaway.pickle b/tests/metadata/org.adaway.pickle index cc2907ab..4c35fe2e 100644 Binary files a/tests/metadata/org.adaway.pickle and b/tests/metadata/org.adaway.pickle differ diff --git a/tests/metadata/org.fdroid.ci.test.app.txt b/tests/metadata/org.fdroid.ci.test.app.txt new file mode 100644 index 00000000..466b7af6 --- /dev/null +++ b/tests/metadata/org.fdroid.ci.test.app.txt @@ -0,0 +1,2 @@ +Repo Type:git +Repo:https://gitlab.com/eighthave/ci-test-app diff --git a/tests/metadata/org.videolan.vlc.pickle b/tests/metadata/org.videolan.vlc.pickle index a1810e60..8a727bb5 100644 Binary files a/tests/metadata/org.videolan.vlc.pickle and b/tests/metadata/org.videolan.vlc.pickle differ diff --git a/tests/metadata/raw.template.txt b/tests/metadata/raw.template.txt new file mode 100644 index 00000000..ec2fe286 --- /dev/null +++ b/tests/metadata/raw.template.txt @@ -0,0 +1,9 @@ +License:Unknown +Web Site: +Source Code: +Issue Tracker: +Changelog: +Summary:Template +Description: +Template +. diff --git a/tests/run-tests b/tests/run-tests index a4474daa..354e6782 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -100,6 +100,8 @@ echo_header "test python getsig replacement" cd $WORKSPACE/tests/getsig ./make.sh + +cd $WORKSPACE/tests for testcase in $WORKSPACE/tests/*.TestCase; do $testcase done @@ -138,6 +140,19 @@ $fdroid readmeta $fdroid update +#------------------------------------------------------------------------------# +echo_header 'run `fdroid build` in fresh git checkout from import.TestCase' + +cd $WORKSPACE/tests/tmp/importer +if [ -d $ANDROID_HOME/platforms/android-23 ]; then + echo "build_tools = '`ls -1 $ANDROID_HOME/build-tools/ | sort -n | tail -1`'" > config.py + echo "force_build_tools = True" >> config.py + $fdroid build --verbose org.fdroid.ci.test.app:300 +else + echo 'WARNING: Skipping `fdroid build` test since android-23 is missing!' +fi + + #------------------------------------------------------------------------------# echo_header "copy tests/repo, generate java/gpg keys, update, and gpgsign"