From 20afa0e8613c6e6914fd877571dbb08141b2678e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 27 Jun 2017 22:46:19 +0200 Subject: [PATCH 1/6] lint: check all build fields against accepted list This makes sure there are no typoed or wrong build fields in all metadata files. --- fdroidserver/lint.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py index 200bddf6..04ef0f9c 100644 --- a/fdroidserver/lint.py +++ b/fdroidserver/lint.py @@ -299,6 +299,8 @@ def check_bulleted_lists(app): def check_builds(app): + supported_flags = set(metadata.build_flags) + # needed for YAML and JSON for build in app.builds: if build.disable: if build.disable.startswith('Generated by import.py'): @@ -311,6 +313,9 @@ def check_builds(app): ref = srclib.split('@')[1].split('/')[0] if ref.startswith(s): yield "Branch '%s' used as commit in srclib '%s'" % (s, srclib) + for key in build.keys(): + if key not in supported_flags: + yield key + ' is not an accepted build field' def check_files_dir(app): From df99c85ca64a3feadaa4a9b5eb2a18ee03334efe Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 27 Jun 2017 23:55:38 +0200 Subject: [PATCH 2/6] support manually adding per-build antiFeatures in metadata For cases like the OpenVPN vuln that was recently announced, it is useful for fdroiddata maintainers to be able to mark builds that have known vulnerabilities. --- .gitlab-ci.yml | 1 + fdroidserver/metadata.py | 3 + fdroidserver/update.py | 35 ++++ tests/metadata.TestCase | 6 +- tests/metadata/com.politedroid.txt | 1 + tests/metadata/dump/com.politedroid.yaml | 178 ++++++++++++++++++ tests/metadata/dump/org.adaway.yaml | 28 +++ .../dump/org.smssecure.smssecure.yaml | 8 + tests/metadata/dump/org.videolan.vlc.yaml | 66 +++++++ tests/update.TestCase | 32 ++++ 10 files changed, 356 insertions(+), 2 deletions(-) create mode 100644 tests/metadata/dump/com.politedroid.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c14f4c24..b49592cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,7 @@ metadata_v0: - "sed -i -e '/AuthorWebSite/d' -e '/Description: No description available/d' -e \"/Description: ''/d\" + -e '/antifeatures:/d' -e '/buildozer:/d' metadata/dump_*/*.yaml" - diff -uw metadata/dump_* diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 254791f2..d832fd0c 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -230,6 +230,7 @@ build_flags_order = [ 'gradleprops', 'antcommands', 'novcheck', + 'antifeatures', ] # old .txt format has version name/code inline in the 'Build:' line @@ -271,6 +272,7 @@ class Build(dict): self.gradleprops = [] self.antcommands = [] self.novcheck = False + self.antifeatures = [] if copydict: super().__init__(copydict) return @@ -339,6 +341,7 @@ flagtypes = { 'forceversion': TYPE_BOOL, 'forcevercode': TYPE_BOOL, 'novcheck': TYPE_BOOL, + 'antifeatures': TYPE_LIST, } diff --git a/fdroidserver/update.py b/fdroidserver/update.py index a82645cf..267e66fe 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -593,6 +593,40 @@ def insert_obbs(repodir, apps, apks): break +def translate_per_build_anti_features(apps, apks): + """Grab the anti-features list from the build metadata + + For most Anti-Features, they are really most applicable per-APK, + not for an app. An app can fix a vulnerability, add/remove + tracking, etc. This reads the 'antifeatures' list from the Build + entries in the fdroiddata metadata file, then transforms it into + the 'antiFeatures' list of unique items for the index. + + The field key is all lower case in the metadata file to match the + rest of the Build fields. It is 'antiFeatures' camel case in the + implementation, index, and fdroidclient since it is translated + from the build 'antifeatures' field, not directly included. + + """ + + antiFeatures = dict() + for packageName, app in apps.items(): + d = dict() + for build in app['builds']: + afl = build.get('antifeatures') + if afl: + d[int(build.versionCode)] = afl + if len(d) > 0: + antiFeatures[packageName] = d + + for apk in apks: + d = antiFeatures.get(apk['packageName']) + if d: + afl = d.get(apk['versionCode']) + if afl: + apk['antiFeatures'].update(afl) + + def _get_localized_dict(app, locale): '''get the dict to add localized store metadata to''' if 'localized' not in app: @@ -1751,6 +1785,7 @@ def main(): copy_triple_t_store_metadata(apps) insert_obbs(repodirs[0], apps, apks) insert_localized_app_metadata(apps) + translate_per_build_anti_features(apps, apks) # Scan the archive repo for apks as well if len(repodirs) > 1: diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index ee113287..f3d58077 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -43,14 +43,16 @@ class MetadataTest(unittest.TestCase): fdroidserver.common.config = config apps = fdroidserver.metadata.read_metadata(xref=True) - for appid in ('org.smssecure.smssecure', 'org.adaway', 'org.videolan.vlc'): + for appid in ('org.smssecure.smssecure', 'org.adaway', + 'org.videolan.vlc', 'com.politedroid'): savepath = os.path.join('metadata', 'dump', appid + '.yaml') frommeta = dict(apps[appid]) self.assertTrue(appid in apps) with open(savepath, 'r') as f: frompickle = yaml.load(f) self.assertEqual(frommeta, frompickle) - # Uncomment to overwrite + # comment above assert and uncomment below to update test + # files when new metadata fields are added # with open(savepath, 'w') as f: # yaml.add_representer(fdroidserver.metadata.Build, _build_yaml_representer) # yaml.dump(frommeta, f, default_flow_style=False) diff --git a/tests/metadata/com.politedroid.txt b/tests/metadata/com.politedroid.txt index 526be787..404073c6 100644 --- a/tests/metadata/com.politedroid.txt +++ b/tests/metadata/com.politedroid.txt @@ -16,6 +16,7 @@ Repo:https://github.com/miguelvps/PoliteDroid.git Build:1.2,3 commit=6a548e4b19 target=android-10 + antifeatures=KnownVuln,UpstreamNonFree,NonFreeAssets Build:1.3,4 commit=ad865b57bf3ac59580f38485608a9b1dda4fa7dc diff --git a/tests/metadata/dump/com.politedroid.yaml b/tests/metadata/dump/com.politedroid.yaml new file mode 100644 index 00000000..84ee21e7 --- /dev/null +++ b/tests/metadata/dump/com.politedroid.yaml @@ -0,0 +1,178 @@ +AntiFeatures: [] +ArchivePolicy: 4 versions +AuthorEmail: null +AuthorName: null +AuthorWebSite: null +AutoName: Polite Droid +AutoUpdateMode: Version v%v +Binaries: null +Bitcoin: null +Categories: +- Time +Changelog: '' +CurrentVersion: '1.5' +CurrentVersionCode: '6' +Description: Activates silent mode during calendar events. +Disabled: null +Donate: null +FlattrID: null +IssueTracker: https://github.com/miguelvps/PoliteDroid/issues +License: GPL-3.0 +Litecoin: null +MaintainerNotes: '' +Name: null +NoSourceSince: '' +Provides: null +Repo: https://github.com/miguelvps/PoliteDroid.git +RepoType: git +RequiresRoot: false +SourceCode: https://github.com/miguelvps/PoliteDroid +Summary: Calendar tool +UpdateCheckData: null +UpdateCheckIgnore: null +UpdateCheckMode: Tags +UpdateCheckName: null +VercodeOperation: null +WebSite: '' +added: null +builds: +- androidupdate: [] + antcommands: [] + antifeatures: + - KnownVuln + - UpstreamNonFree + - NonFreeAssets + build: '' + buildjni: [] + buildozer: false + commit: 6a548e4b19 + disable: false + encoding: null + extlibs: [] + forcevercode: false + forceversion: false + gradle: [] + gradleprops: [] + init: '' + kivy: false + maven: false + ndk: null + novcheck: false + oldsdkloc: false + output: null + patch: [] + preassemble: [] + prebuild: '' + rm: [] + scandelete: [] + scanignore: [] + srclibs: [] + subdir: null + submodules: false + target: android-10 + versionCode: '3' + versionName: '1.2' +- androidupdate: [] + antcommands: [] + antifeatures: [] + build: '' + buildjni: [] + buildozer: false + commit: ad865b57bf3ac59580f38485608a9b1dda4fa7dc + disable: false + encoding: null + extlibs: [] + forcevercode: false + forceversion: false + gradle: [] + gradleprops: [] + init: '' + kivy: false + maven: false + ndk: null + novcheck: false + oldsdkloc: false + output: null + patch: [] + preassemble: [] + prebuild: '' + rm: [] + scandelete: [] + scanignore: [] + srclibs: [] + subdir: null + submodules: false + target: android-15 + versionCode: '4' + versionName: '1.3' +- androidupdate: [] + antcommands: [] + antifeatures: [] + build: '' + buildjni: [] + buildozer: false + commit: 456bd615f3fbe6dff06433928cf7ea20073601fb + disable: false + encoding: null + extlibs: [] + forcevercode: false + forceversion: false + gradle: [] + gradleprops: [] + init: '' + kivy: false + maven: false + ndk: null + novcheck: false + oldsdkloc: false + output: null + patch: [] + preassemble: [] + prebuild: '' + rm: [] + scandelete: [] + scanignore: [] + srclibs: [] + subdir: null + submodules: false + target: android-10 + versionCode: '5' + versionName: '1.4' +- androidupdate: [] + antcommands: [] + antifeatures: [] + build: '' + buildjni: [] + buildozer: false + commit: v1.5 + disable: false + encoding: null + extlibs: [] + forcevercode: false + forceversion: false + gradle: + - 'yes' + gradleprops: [] + init: '' + kivy: false + maven: false + ndk: null + novcheck: false + oldsdkloc: false + output: null + patch: [] + preassemble: [] + prebuild: '' + rm: [] + scandelete: [] + scanignore: [] + srclibs: [] + subdir: null + submodules: false + target: null + versionCode: '6' + versionName: '1.5' +comments: {} +id: com.politedroid +lastUpdated: null +metadatapath: metadata/com.politedroid.txt diff --git a/tests/metadata/dump/org.adaway.yaml b/tests/metadata/dump/org.adaway.yaml index e1f3a6f3..8ca9ded0 100644 --- a/tests/metadata/dump/org.adaway.yaml +++ b/tests/metadata/dump/org.adaway.yaml @@ -61,6 +61,7 @@ added: null builds: - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -94,6 +95,7 @@ builds: versionName: '1.12' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -128,6 +130,7 @@ builds: versionName: '1.15' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -162,6 +165,7 @@ builds: versionName: '1.18' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -196,6 +200,7 @@ builds: versionName: '1.19' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -230,6 +235,7 @@ builds: versionName: '1.20' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -264,6 +270,7 @@ builds: versionName: '1.21' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -296,6 +303,7 @@ builds: versionName: '1.23' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -329,6 +337,7 @@ builds: versionName: '1.24' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -362,6 +371,7 @@ builds: versionName: '1.25' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -395,6 +405,7 @@ builds: versionName: '1.26' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -428,6 +439,7 @@ builds: versionName: '1.27' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -461,6 +473,7 @@ builds: versionName: '1.29' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -494,6 +507,7 @@ builds: versionName: '1.32' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -526,6 +540,7 @@ builds: versionName: '1.33' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -559,6 +574,7 @@ builds: versionName: '1.34' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -592,6 +608,7 @@ builds: versionName: '1.35' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -625,6 +642,7 @@ builds: versionName: '1.36' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -662,6 +680,7 @@ builds: - android-libs/ActionBarSherlock - android-libs/HtmlSpanner/htmlspanner antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -707,6 +726,7 @@ builds: - android-libs/ActionBarSherlock - android-libs/HtmlSpanner/htmlspanner antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -748,6 +768,7 @@ builds: versionName: '2.3' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -783,6 +804,7 @@ builds: versionName: '2.6' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -818,6 +840,7 @@ builds: versionName: '2.7' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -853,6 +876,7 @@ builds: versionName: '2.8' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -888,6 +912,7 @@ builds: versionName: 2.8.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -923,6 +948,7 @@ builds: versionName: '2.9' - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -958,6 +984,7 @@ builds: versionName: 2.9.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' @@ -993,6 +1020,7 @@ builds: versionName: 2.9.2 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: - 'yes' diff --git a/tests/metadata/dump/org.smssecure.smssecure.yaml b/tests/metadata/dump/org.smssecure.smssecure.yaml index 407aa5d9..ce6dec68 100644 --- a/tests/metadata/dump/org.smssecure.smssecure.yaml +++ b/tests/metadata/dump/org.smssecure.smssecure.yaml @@ -58,6 +58,7 @@ added: null builds: - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -102,6 +103,7 @@ builds: versionName: 0.3.3 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -138,6 +140,7 @@ builds: versionName: 0.3.3 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -173,6 +176,7 @@ builds: versionName: 0.4.2 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -208,6 +212,7 @@ builds: versionName: 0.5.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -242,6 +247,7 @@ builds: versionName: 0.5.2 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -276,6 +282,7 @@ builds: versionName: 0.5.3 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false @@ -310,6 +317,7 @@ builds: versionName: 0.5.4 - androidupdate: [] antcommands: [] + antifeatures: [] build: '' buildjni: [] buildozer: false diff --git a/tests/metadata/dump/org.videolan.vlc.yaml b/tests/metadata/dump/org.videolan.vlc.yaml index 70b9d406..52681f92 100644 --- a/tests/metadata/dump/org.videolan.vlc.yaml +++ b/tests/metadata/dump/org.videolan.vlc.yaml @@ -63,6 +63,7 @@ builds: - ../java-libs/SlidingMenu - ../java-libs/ActionBarSherlock antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -98,6 +99,7 @@ builds: - ../java-libs/SlidingMenu - ../java-libs/ActionBarSherlock antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi ./compile.sh release buildjni: [] buildozer: false @@ -133,6 +135,7 @@ builds: - ../java-libs/SlidingMenu - ../java-libs/ActionBarSherlock antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -168,6 +171,7 @@ builds: - ../java-libs/SlidingMenu - ../java-libs/ActionBarSherlock antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=mips ./compile.sh release buildjni: [] buildozer: false @@ -200,6 +204,7 @@ builds: versionName: 0.0.11-mips - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=mips ./compile.sh release buildjni: [] buildozer: false @@ -234,6 +239,7 @@ builds: versionName: 0.1.3-MIPS - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -268,6 +274,7 @@ builds: versionName: 0.1.3-x86 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi ./compile.sh release buildjni: [] buildozer: false @@ -302,6 +309,7 @@ builds: versionName: 0.1.3-ARM - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -336,6 +344,7 @@ builds: versionName: 0.1.3-ARMv7 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -369,6 +378,7 @@ builds: versionName: 0.9.0 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -402,6 +412,7 @@ builds: versionName: 0.9.0 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -435,6 +446,7 @@ builds: versionName: 0.9.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -468,6 +480,7 @@ builds: versionName: 0.9.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -501,6 +514,7 @@ builds: versionName: 0.9.5 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -534,6 +548,7 @@ builds: versionName: 0.9.5 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -567,6 +582,7 @@ builds: versionName: 0.9.6 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -600,6 +616,7 @@ builds: versionName: 0.9.6 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -633,6 +650,7 @@ builds: versionName: 0.9.7 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -666,6 +684,7 @@ builds: versionName: 0.9.7 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=mips ./compile.sh release buildjni: [] buildozer: false @@ -699,6 +718,7 @@ builds: versionName: 0.9.7.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -732,6 +752,7 @@ builds: versionName: 0.9.7.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -765,6 +786,7 @@ builds: versionName: 0.9.7.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -798,6 +820,7 @@ builds: versionName: 0.9.8 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi ./compile.sh release buildjni: [] buildozer: false @@ -831,6 +854,7 @@ builds: versionName: 0.9.8 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -864,6 +888,7 @@ builds: versionName: 0.9.8 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -897,6 +922,7 @@ builds: versionName: 0.9.9 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi ./compile.sh release buildjni: [] buildozer: false @@ -930,6 +956,7 @@ builds: versionName: 0.9.9 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -963,6 +990,7 @@ builds: versionName: 0.9.9 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -996,6 +1024,7 @@ builds: versionName: 0.9.10 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi ./compile.sh release buildjni: [] buildozer: false @@ -1029,6 +1058,7 @@ builds: versionName: 0.9.10 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -1062,6 +1092,7 @@ builds: versionName: 0.9.10 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -1095,6 +1126,7 @@ builds: versionName: 1.0.0 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi ./compile.sh release buildjni: [] buildozer: false @@ -1128,6 +1160,7 @@ builds: versionName: 1.0.0 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -1161,6 +1194,7 @@ builds: versionName: 1.0.0 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=x86 ./compile.sh release buildjni: [] buildozer: false @@ -1194,6 +1228,7 @@ builds: versionName: 1.0.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi ./compile.sh release buildjni: [] buildozer: false @@ -1227,6 +1262,7 @@ builds: versionName: 1.0.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ANDROID_ABI=armeabi-v7a ./compile.sh release buildjni: [] buildozer: false @@ -1260,6 +1296,7 @@ builds: versionName: 1.0.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -1295,6 +1332,7 @@ builds: versionName: 1.1.3 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -1330,6 +1368,7 @@ builds: versionName: 1.1.3 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false @@ -1365,6 +1404,7 @@ builds: versionName: 1.1.3 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -1400,6 +1440,7 @@ builds: versionName: 1.1.5 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -1435,6 +1476,7 @@ builds: versionName: 1.1.5 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false @@ -1470,6 +1512,7 @@ builds: versionName: 1.1.5 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -1505,6 +1548,7 @@ builds: versionName: 1.1.6 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -1540,6 +1584,7 @@ builds: versionName: 1.1.6 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false @@ -1575,6 +1620,7 @@ builds: versionName: 1.1.6 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -1610,6 +1656,7 @@ builds: versionName: 1.2.0 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -1645,6 +1692,7 @@ builds: versionName: 1.2.0 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false @@ -1680,6 +1728,7 @@ builds: versionName: 1.2.0 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -1715,6 +1764,7 @@ builds: versionName: 1.2.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -1750,6 +1800,7 @@ builds: versionName: 1.2.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false @@ -1785,6 +1836,7 @@ builds: versionName: 1.2.1 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -1820,6 +1872,7 @@ builds: versionName: 1.2.2 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -1855,6 +1908,7 @@ builds: versionName: 1.2.2 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false @@ -1890,6 +1944,7 @@ builds: versionName: 1.2.2 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -1925,6 +1980,7 @@ builds: versionName: 1.2.3 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -1960,6 +2016,7 @@ builds: versionName: 1.2.3 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false @@ -1995,6 +2052,7 @@ builds: versionName: 1.2.3 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -2030,6 +2088,7 @@ builds: versionName: 1.2.4 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -2065,6 +2124,7 @@ builds: versionName: 1.2.4 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false @@ -2100,6 +2160,7 @@ builds: versionName: 1.2.4 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -2135,6 +2196,7 @@ builds: versionName: 1.2.5 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -2170,6 +2232,7 @@ builds: versionName: 1.2.5 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false @@ -2205,6 +2268,7 @@ builds: versionName: 1.2.5 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi" --release buildjni: [] buildozer: false @@ -2240,6 +2304,7 @@ builds: versionName: 1.2.6 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "armeabi-v7a" --release buildjni: [] buildozer: false @@ -2275,6 +2340,7 @@ builds: versionName: 1.2.6 - androidupdate: [] antcommands: [] + antifeatures: [] build: cd ../ && ./compile.sh -a "x86" --release buildjni: [] buildozer: false diff --git a/tests/update.TestCase b/tests/update.TestCase index bb79a35f..5bd81a8c 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -459,6 +459,38 @@ class UpdateTest(unittest.TestCase): self.assertIsNone(apk) self.assertFalse(cachechanged) + def test_translate_per_build_anti_features(self): + os.chdir(os.path.join(localmodule, 'tests')) + if os.path.basename(os.getcwd()) != 'tests': + raise Exception('This test must be run in the "tests/" subdir') + + config = dict() + fdroidserver.common.fill_config_defaults(config) + config['ndk_paths'] = dict() + config['accepted_formats'] = ['json', 'txt', 'yml'] + fdroidserver.common.config = config + fdroidserver.update.config = config + + fdroidserver.update.options = type('', (), {})() + fdroidserver.update.options.clean = True + fdroidserver.update.options.delete_unknown = True + fdroidserver.update.options.rename_apks = False + fdroidserver.update.options.allow_disabled_algorithms = False + + apps = fdroidserver.metadata.read_metadata(xref=True) + knownapks = fdroidserver.common.KnownApks() + apks, cachechanged = fdroidserver.update.process_apks({}, 'repo', knownapks, False) + fdroidserver.update.translate_per_build_anti_features(apps, apks) + self.assertEqual(len(apks), 11) + foundtest = False + for apk in apks: + if apk['packageName'] == 'com.politedroid' and apk['versionCode'] == 3: + antiFeatures = apk.get('antiFeatures') + self.assertTrue('KnownVuln' in antiFeatures) + self.assertEqual(3, len(antiFeatures)) + foundtest = True + self.assertTrue(foundtest) + if __name__ == "__main__": parser = optparse.OptionParser() From 9a90a5071da893141968d5837c4cc213f5d9995b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 28 Jun 2017 16:55:34 +0200 Subject: [PATCH 3/6] buildserver: support any recent NDK version, with stable filenames Now that the download file name and type seems to have stabilized, I think we no longer need to manually specify each new added release in this script to unpack. closes #331 --- buildserver/provision-android-ndk | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/buildserver/provision-android-ndk b/buildserver/provision-android-ndk index de6943b4..5566305e 100644 --- a/buildserver/provision-android-ndk +++ b/buildserver/provision-android-ndk @@ -20,30 +20,13 @@ if [ ! -e $NDK_BASE/r10e ]; then mv android-ndk-r10e r10e fi -if [ ! -e $NDK_BASE/r11c ]; then - unzip /vagrant/cache/android-ndk-r11c-linux-x86_64.zip > /dev/null - mv android-ndk-r11c r11c -fi - -if [ ! -e $NDK_BASE/r12b ]; then - unzip /vagrant/cache/android-ndk-r12b-linux-x86_64.zip > /dev/null - mv android-ndk-r12b r12b -fi - -if [ ! -e $NDK_BASE/r13b ]; then - unzip /vagrant/cache/android-ndk-r13b-linux-x86_64.zip > /dev/null - mv android-ndk-r13b r13b -fi - -if [ ! -e $NDK_BASE/r14b ]; then - unzip /vagrant/cache/android-ndk-r14b-linux-x86_64.zip > /dev/null - mv android-ndk-r14b r14b -fi - -if [ ! -e $NDK_BASE/r15b ]; then - unzip /vagrant/cache/android-ndk-r15b-linux-x86_64.zip > /dev/null - mv android-ndk-r15b r15b -fi +for f in /vagrant/cache/android-ndk-r[1-9][0-9]*-linux-x86_64.zip; do + version=`echo $f |sed 's,.*\(r[1-9][0-9][a-z]\?\).*,\1,'` + if [ ! -e ${NDK_BASE}/${version} ]; then + unzip /vagrant/cache/android-ndk-${version}-linux-x86_64.zip > /dev/null + mv android-ndk-${version} ${version} + fi +done chmod -R a+rX $NDK_BASE/ find $NDK_BASE/ -type f -executable -print0 | xargs -0 chmod a+x From 2548d71600b5c054fea2447003cbe89aff4d7274 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 29 Jun 2017 16:00:16 +0200 Subject: [PATCH 4/6] jenkins: document build/sign test flow --- jenkins-build-makebuildserver | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index 315a5366..b30b41c6 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -104,9 +104,12 @@ echo "build_server_always = True" >> config.py # publish process when building and signing are on separate machines test -d repo || mkdir repo test -d archive || mkdir archive +# copy everything over to run on SIGN machine ../fdroid publish --verbose ../fdroid gpgsign --verbose +# copy everything over to run on BUILD machine ../fdroid update --verbose --nosign +# copy everything over to run on SIGN machine ../fdroid signindex --verbose ../fdroid rewritemeta --verbose From c14cbeb0a52b3bb9d0751702dac01f95cf9acb67 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 4 Jul 2017 17:18:21 +0200 Subject: [PATCH 5/6] update: normalize var name to apkzip --- fdroidserver/update.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 267e66fe..1ccec78a 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1363,7 +1363,7 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False): return apks, cachechanged -def extract_apk_icons(icon_filename, apk, apk_zip, repo_dir): +def extract_apk_icons(icon_filename, apk, apkzip, repo_dir): """ Extracts icons from the given APK zip in various densities, saves them into given repo directory @@ -1372,7 +1372,7 @@ def extract_apk_icons(icon_filename, apk, apk_zip, repo_dir): :param icon_filename: A string representing the icon's file name :param apk: A populated dictionary containing APK metadata. Needs to have 'icons_src' key - :param apk_zip: An opened zipfile.ZipFile of the APK file + :param apkzip: An opened zipfile.ZipFile of the APK file :param repo_dir: The directory of the APK's repository :return: A list of icon densities that are missing """ @@ -1388,7 +1388,7 @@ def extract_apk_icons(icon_filename, apk, apk_zip, repo_dir): # Extract the icon files per density try: with open(icon_dest, 'wb') as f: - f.write(get_icon_bytes(apk_zip, icon_src)) + f.write(get_icon_bytes(apkzip, icon_src)) apk['icons'][density] = icon_filename except (zipfile.BadZipFile, ValueError, KeyError) as e: logging.warning("Error retrieving icon file: %s %s", icon_dest, e) @@ -1399,7 +1399,7 @@ def extract_apk_icons(icon_filename, apk, apk_zip, repo_dir): icon_src = apk['icons_src']['-1'] icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename) with open(icon_path, 'wb') as f: - f.write(get_icon_bytes(apk_zip, icon_src)) + f.write(get_icon_bytes(apkzip, icon_src)) try: im = Image.open(icon_path) dpi = px_to_dpi(im.size[0]) From 10a607ede3b61503cfd213b8b625b64ca06aaa25 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 4 Jul 2017 17:40:02 +0200 Subject: [PATCH 6/6] update: find PNG when recommended icon is an XML file APKs can now use XML files for vector graphics like the app icon. `aapt` returns the XML file by default, and perhaps also androguard. This checks if the icon is an XML file, and if so, it tries to find a PNG in the APK with the same name and density to use instead closes #322 This should also ultimately make the XML file available as an icon source as well fdroidclient#1091 --- fdroidserver/update.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 1ccec78a..286a5510 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1386,6 +1386,13 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir): icon_dest = os.path.join(icon_dir, icon_filename) # Extract the icon files per density + if icon_src.endswith('.xml'): + png = os.path.basename(icon_src)[:-4] + '.png' + for f in apkzip.namelist(): + if f.endswith(png): + m = re.match(r'res/drawable-(x*[hlm]dpi).*/', f) + if m and screen_resolutions[m.group(1)] == density: + icon_src = f try: with open(icon_dest, 'wb') as f: f.write(get_icon_bytes(apkzip, icon_src))