From a4177e5ec36f65557e22653655abb0e7a1915dcd Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Tue, 2 Jun 2020 13:32:11 +0200 Subject: [PATCH 1/9] add test for correct whatsnew handling without CVC --- tests/run-tests | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/run-tests b/tests/run-tests index 9c5d85d9..d685ea1f 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -607,6 +607,19 @@ grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index-v1.json ! grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index.xml +#------------------------------------------------------------------------------# +echo_header "test whatsnew from fastlane without CVC set" +REPOROOT=`create_test_dir` +cd $REPOROOT +fdroid_init_with_prebuilt_keystore +echo "accepted_formats = ['txt', 'yml']" >> config.py +mkdir -p metadata/com.politedroid/en-US/changelogs/ +cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo +cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata +echo "whatsnew test" > metadata/com.politedroid/en-US/changelogs/6.txt +sed -i -e '/Current Version/d' metadata/com.politedroid.txt +$fdroid update --pretty --nosign +grep -F 'whatsnew' repo/index-v1.json #------------------------------------------------------------------------------# echo_header "test metadata checks" From 03881154c65402e3b921e841a097aea6625efd73 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Wed, 3 Jun 2020 21:28:20 +0200 Subject: [PATCH 2/9] metadata: make linkresolver an actual object Previously this was magically capturing the apps dict when passing it around as a function. This also moved the code to the metadata module. Add a test doing read_metadata where the linkresolver is used. This happens when the apps we read have a [[app.id]] link to another app. --- fdroidserver/index.py | 12 +- fdroidserver/metadata.py | 27 +- tests/metadata.TestCase | 8 + tests/xref/metadata/aarddict.android.yml | 104 ++++ tests/xref/metadata/org.coolreader.yml | 551 +++++++++++++++++ .../org.geometerplus.zlibrary.ui.android.yml | 556 ++++++++++++++++++ 6 files changed, 1240 insertions(+), 18 deletions(-) create mode 100644 tests/xref/metadata/aarddict.android.yml create mode 100644 tests/xref/metadata/org.coolreader.yml create mode 100644 tests/xref/metadata/org.geometerplus.zlibrary.ui.android.yml diff --git a/fdroidserver/index.py b/fdroidserver/index.py index c19b6277..195b23e7 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -58,11 +58,6 @@ def make(apps, sortedids, apks, repodir, archive): """ from fdroidserver.update import METADATA_VERSION - def _resolve_description_link(appid): - if appid in apps: - return "fdroid.app:" + appid, apps[appid].Name - raise MetaDataException("Cannot resolve app id " + appid) - if not common.options.nosign: common.assert_config_keystore(common.config) @@ -117,7 +112,7 @@ def make(apps, sortedids, apks, repodir, archive): if apk['packageName'] == packageName: newapp = copy.copy(app) # update wiki needs unmodified description newapp['Description'] = metadata.description_html(app['Description'], - _resolve_description_link) + metadata.DescriptionResolver(apps)) appsWithPackages[packageName] = newapp break @@ -311,11 +306,6 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing value = str(apk[key]) addElement(name, value, doc, parent) - def addElementCDATA(name, value, doc, parent): - el = doc.createElement(name) - el.appendChild(doc.createCDATASection(value)) - parent.appendChild(el) - def addElementCheckLocalized(name, app, key, doc, parent, default=''): """Fill in field from metadata or localized block diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index a170699c..ee58b1b4 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -616,7 +616,7 @@ class DescriptionFormatter: warn_or_exception(_("Unterminated ]]")) url = txt[2:index] if self.linkResolver: - url, urltext = self.linkResolver(url) + url, urltext = self.linkResolver.resolve_description_link(url) else: urltext = url res_html += '' + html.escape(urltext, quote=False) + '' @@ -899,14 +899,9 @@ def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False): if xref: # Parse all descriptions at load time, just to ensure cross-referencing # errors are caught early rather than when they hit the build server. - def linkres(appid): - if appid in apps: - return ("fdroid.app:" + appid, "Dummy name - don't know yet") - warn_or_exception(_("Cannot resolve app id {appid}").format(appid=appid)) - for appid, app in apps.items(): try: - description_html(app.Description, linkres) + description_html(app.Description, DummyDescriptionResolver(apps)) except MetaDataException as e: warn_or_exception(_("Problem with description of {appid}: {error}") .format(appid=appid, error=str(e))) @@ -1679,3 +1674,21 @@ def add_metadata_arguments(parser): '''add common command line flags related to metadata processing''' parser.add_argument("-W", choices=['error', 'warn', 'ignore'], default='error', help=_("force metadata errors (default) to be warnings, or to be ignored.")) + + +class DescriptionResolver: + def __init__(self, apps): + self.apps = apps + + def resolve_description_link(self, appid): + if appid in self.apps: + if self.apps[appid].Name: + return "fdroid.app:" + appid, self.apps[appid].Name + raise MetaDataException("Cannot resolve app id " + appid) + + +class DummyDescriptionResolver(DescriptionResolver): + def resolve_description_link(self, appid): + if appid in self.apps: + return "fdroid.app:" + appid, "Dummy name - don't know yet" + warn_or_exception(_("Cannot resolve app id {appid}").format(appid=appid)) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 5a574702..49151083 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -905,6 +905,14 @@ class MetadataTest(unittest.TestCase): 'Subdir': None, 'Prepare': None}}) + def test_read_xref_metadata(self): + fdroidserver.common.config = {'accepted_formats': ['yml']} + os.chdir('xref') + fdroidserver.metadata.warnings_action = 'error' + apps = fdroidserver.metadata.read_metadata(xref=True) + self.assertListEqual(list(apps.keys()), + ['aarddict.android', 'org.coolreader', 'org.geometerplus.zlibrary.ui.android']) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__)) diff --git a/tests/xref/metadata/aarddict.android.yml b/tests/xref/metadata/aarddict.android.yml new file mode 100644 index 00000000..d679c6b0 --- /dev/null +++ b/tests/xref/metadata/aarddict.android.yml @@ -0,0 +1,104 @@ +Categories: + - Reading +License: GPL-3.0-only +WebSite: http://aarddict.org +SourceCode: https://github.com/aarddict/android +IssueTracker: https://github.com/aarddict/android/issues +Donate: http://aarddict.org/android +FlattrID: '80944' + +AutoName: Aard +Description: |- + '''Note:''' This app is no longer maintained. + + * looks up words fast even with huge dictionaries like English Wikipedia + * looks up words in multiple dictionaries in multiple languages without switching + * works great as an offline Wikipedia reader + * uses same the efficient, highly compressed dictionary data storage format as the desktop version + * it can integrate with both [[org.geometerplus.zlibrary.ui.android]] and [[org.coolreader]] + + Ready-made dictionaries can be found on the website, or you can roll your own + with the tools on the website. + +RepoType: git +Repo: https://github.com/aarddict/android.git + +Builds: + - versionName: 1.3.1 + versionCode: 10 + commit: 1.3.1 + prebuild: mv lib libs + + - versionName: 1.4.0 + versionCode: 12 + commit: 7df930161256324e31b2c720281629f58446b6d6 + prebuild: mv lib libs + + - versionName: 1.4.1 + versionCode: 13 + commit: b81c9c8c52de5f65b550e3c608a610962582e5cd + prebuild: mv lib libs + + - versionName: 1.6.1 + versionCode: 16 + commit: 1.6.1 + target: android-17 + + - versionName: 1.6.2 + versionCode: 17 + commit: 1.6.2 + target: android-17 + + - versionName: 1.6.3 + versionCode: 18 + commit: 1.6.3 + target: android-17 + + - versionName: 1.6.4 + versionCode: 19 + commit: 1.6.4 + target: android-17 + + - versionName: 1.6.5 + versionCode: 20 + commit: 1.6.5 + target: android-17 + + - versionName: 1.6.6 + versionCode: 21 + commit: 1.6.6 + target: android-17 + + - versionName: 1.6.7 + versionCode: 22 + commit: 1.6.7 + target: android-17 + + - versionName: 1.6.8 + versionCode: 23 + commit: 1.6.8 + target: android-17 + + - versionName: 1.6.9 + versionCode: 24 + commit: 1.6.9 + target: android-17 + + - versionName: 1.6.10 + versionCode: 25 + commit: 1.6.10 + target: android-17 + + - versionName: 1.6.11 + versionCode: 26 + commit: 1.6.11 + target: android-17 + +MaintainerNotes: |- + Github site points to https://github.com/itkach/aard2-android (itkach.aard2) as successor. + We cannot point to this app as all its builds are disabled (as of 2018-10-16). + +AutoUpdateMode: Version %v +UpdateCheckMode: Tags +CurrentVersion: 1.6.11 +CurrentVersionCode: 26 diff --git a/tests/xref/metadata/org.coolreader.yml b/tests/xref/metadata/org.coolreader.yml new file mode 100644 index 00000000..e1dc1df9 --- /dev/null +++ b/tests/xref/metadata/org.coolreader.yml @@ -0,0 +1,551 @@ +Categories: + - Reading +License: GPL-2.0-only +AuthorName: Vadim Lopatin +AuthorEmail: coolreader.org@gmail.com +WebSite: https://sourceforge.net/projects/crengine/ +SourceCode: https://github.com/buggins/coolreader +IssueTracker: https://github.com/buggins/coolreader/issues +Donate: https://sourceforge.net/p/crengine/donate + +AutoName: Cool Reader +Description: |- + An e-book reader. Supported formats: FB2, TXT, RTF, TCR, HTML, EPUB, CHM. Browse + free books online and add your own OPDS shares. + + The default dictionary app is non-free. However, you can choose + [[aarddict.android]] as a dictionary from the Dictionary section of the + Settings. + +RepoType: git +Repo: https://github.com/buggins/coolreader + +Builds: + - versionName: 3.0.39-35 + versionCode: 60 + commit: 68ad007ac1272ef322fd61cb6591618723422380 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.40-1 + versionCode: 64 + commit: cr3.0.40-1 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.40-6 + versionCode: 69 + commit: cr3.0.40-6 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.41-3 + versionCode: 75 + commit: cr3.0.41-3 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.41-9 + versionCode: 81 + commit: cr3.0.41-9 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.42-6 + versionCode: 86 + commit: cr3.0.42-6 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.43-4 + versionCode: 94 + commit: cr3.0.43-4 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.43-6 + versionCode: 96 + commit: cr3.0.43-6 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.44-03 + versionCode: 103 + commit: cr3.0.44-3 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.44-09 + versionCode: 109 + commit: cr3.0.44-9 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.45-04 + versionCode: 114 + disable: tag points to wrong version + commit: unknown - see disabled + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.45-09 + versionCode: 119 + disable: tag points to wrong version + commit: unknown - see disabled + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.45-11 + versionCode: 121 + commit: 6c42a9b65090da9640ccb6ee317bb32de24201fb + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.45-14 + versionCode: 124 + commit: cr3.0.45-14 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.46-1 + versionCode: 131 + commit: cr3.0.46-1 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.46-5 + versionCode: 135 + commit: cr3.0.46-5 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.46-11 + versionCode: 141 + commit: cr3.0.46-11 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.48-5 + versionCode: 155 + commit: cr3.0.48-5 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.49-2 + versionCode: 162 + commit: cr3.0.49-2 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.49-5 + versionCode: 165 + commit: cr3.0.49-5 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.49-9 + versionCode: 169 + commit: cr3.0.49-9 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.49-16 + versionCode: 176 + commit: cr3.0.49-16 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.51-7 + versionCode: 197 + commit: cr3.0.51-7 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.52-1 + versionCode: 241 + commit: cr3.0.52-1 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.52-4 + versionCode: 244 + commit: cr3.0.52-4 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.53-3 + versionCode: 247 + commit: cr3.0.53-3 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.53-10 + versionCode: 255 + commit: cr3.0.53-10 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.53-14 + versionCode: 259 + commit: cr3.0.53-14 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.53-18 + versionCode: 263 + commit: c555ecd66d18b218fb255733c8b33a0825992f76 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.53-19 + versionCode: 264 + commit: cr3.0.53-19 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.54-5 + versionCode: 275 + commit: cr3.0.54-5 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.54-9 + versionCode: 279 + commit: cr3.0.54-9 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.54-33 + versionCode: 303 + commit: cr3.0.54-33 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.54-38 + versionCode: 308 + commit: cr3.0.54-38 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.54-47 + versionCode: 447 + commit: cr3.0.54-47 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.55-5 + versionCode: 505 + commit: cr3.0.55-5 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.55-9 + versionCode: 509 + commit: cr3.0.55-9 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.55-14 + versionCode: 514 + commit: cr3.0.55-14 + subdir: android + rm: + - android/build.properties + buildjni: + - yes + + - versionName: 3.0.55-30 + versionCode: 530 + commit: cr3.0.55-30 + subdir: android + rm: + - android/ant.properties + buildjni: + - yes + + - versionName: 3.0.55-32 + versionCode: 532 + commit: cr3.0.55-32 + subdir: android + rm: + - android/ant.properties + buildjni: + - yes + + - versionName: 3.0.56-5 + versionCode: 565 + disable: builds ok but crashes at cr3.0.56-5 + commit: cr3.0.56-5 + subdir: android + rm: + - android/ant.properties + target: android-14 + buildjni: + - yes + + - versionName: 3.0.57-15 + versionCode: 625 + disable: builds ok but crashes at cr3.0.57-15 + commit: cr3.0.57-15 + subdir: android + rm: + - android/ant.properties + target: android-14 + buildjni: + - yes + + - versionName: 3.0.57-16 + versionCode: 626 + disable: builds ok but crashes at 2f3d5fb86df316d + commit: 2f3d5fb86df316d + subdir: android + rm: + - android/ant.properties + target: android-14 + buildjni: + - yes + + - versionName: 3.1.2-27 + versionCode: 847 + commit: cr3-3.1.2-27 + subdir: android + rm: + - android/ant.properties + target: android-16 + buildjni: + - yes + + - versionName: 3.1.2-38 + versionCode: 858 + disable: ndk build errors + commit: cr3-3.1.2-38 + subdir: android + buildjni: + - yes + + - versionName: 3.1.2-39 + versionCode: 859 + commit: cr3-3.1.2-39-market + subdir: android + rm: + - android/build.properties + target: android-19 + buildjni: + - yes + + - versionName: 3.1.2-48 + versionCode: 868 + commit: cr3.1.2-48 + subdir: android + rm: + - android/build.properties + target: android-19 + buildjni: + - yes + + - versionName: 3.1.2-50 + versionCode: 870 + commit: cr3.1.2-50 + subdir: android + rm: + - android/build.properties + target: android-19 + buildjni: + - yes + + - versionName: 3.1.2-51 + versionCode: 871 + commit: cr3-3.1.2-51 + subdir: android + rm: + - android/build.properties + target: android-19 + buildjni: + - yes + + - versionName: 3.1.2-55 + versionCode: 875 + commit: cr3-3.1.2-55 + subdir: android + rm: + - android/build.properties + target: android-19 + buildjni: + - yes + + - versionName: 3.1.2-68 + versionCode: 888 + commit: cr3.1.2-68 + subdir: android + rm: + - android/build.properties + target: android-22 + buildjni: + - yes + + - versionName: 3.1.2-72 + versionCode: 892 + commit: cr3.1.2-71 + subdir: android + rm: + - android/build.properties + prebuild: sed -i -e 's/^APP_ABI\s*:=.*/APP_ABI := all/' jni/Application.mk + target: android-22 + buildjni: + - yes + + - versionName: 3.1.2-87 + versionCode: 907 + commit: 1e07d15d4644c690 + subdir: android + rm: + - android/build.properties + prebuild: sed -i -e 's/^APP_ABI\s*:=.*/APP_ABI := all/' jni/Application.mk + target: android-22 + scanignore: + - cr3wx + buildjni: + - yes + + - versionName: 3.2.9-1 + versionCode: 2091 + commit: bf48e5b7a5e89e5fc8b1f971573b5046e6b27bd3 + subdir: android + gradle: + - yes + output: app/build/outputs/apk/release/app-universal-release-unsigned.apk + rm: + - android/build.properties + prebuild: sed -i -e 's/^APP_ABI\s*:=.*/APP_ABI := all/' jni/Application.mk + scanignore: + - cr3wx + ndk: r16b + + - versionName: 3.2.38-1 + versionCode: 32380 + commit: cr3.2.38 + subdir: android/app + gradle: + - yes + rm: + - cr3wx/src/resources/crrcconv.exe + prebuild: + - sed -i -e 's/\r//' ../gradle/wrapper/gradle-wrapper.properties + - sed -i -e 's/enable true/enable false/' build.gradle + ndk: r21 + +AutoUpdateMode: None +UpdateCheckMode: Tags +CurrentVersion: 3.2.39-1 +CurrentVersionCode: 32390 diff --git a/tests/xref/metadata/org.geometerplus.zlibrary.ui.android.yml b/tests/xref/metadata/org.geometerplus.zlibrary.ui.android.yml new file mode 100644 index 00000000..ff2897eb --- /dev/null +++ b/tests/xref/metadata/org.geometerplus.zlibrary.ui.android.yml @@ -0,0 +1,556 @@ +AntiFeatures: + - NonFreeAdd + - UpstreamNonFree +Categories: + - Reading +License: GPL-2.0-or-later +AuthorName: Nikolay Pultsin +AuthorEmail: geometer@fbreader.org +AuthorWebSite: https://fbreader.org/ +WebSite: https://fbreader.org/FBReaderJ +SourceCode: https://github.com/geometer/FBReaderJ +IssueTracker: https://github.com/geometer/FBReaderJ/issues +Changelog: https://raw.githubusercontent.com/geometer/FBReaderJ/HEAD/ChangeLog +Donate: https://fbreader.org/donation/make.php + +AutoName: FBReader +Description: |- + '''N.B'''There are three different apks to cover the different versions of + Android. Donut covers 1.5-1.6; Froyo covers 2.0-2.3 and Honeycomb covers 3.0+. + x86 and MIPS are supported natively in all apks. + + An e-book reader. Features include the ability to stock up on books from online + OPDS libraries like Project Gutenberg straight from the app. F-Droid.org has two + other addon apps that provide text-to-speech functionality and one to support + ''local'' OPDS shares. + + Anti-features: Addons. While there are some addons for this app that are free, + the dictionaries that are suggested are not. However, it does support + [[aarddict.android]], as long as that is installed beforehand '''and''' you + choose it via the Dictionary section of the settings. + +RepoType: git +Repo: https://github.com/geometer/FBReaderJ.git + +Builds: + - versionName: 0.99.11 + versionCode: 9911 + commit: 0.99.11 + antcommands: + - package + + - versionName: 0.99.12 + versionCode: 9912 + commit: 0.99.12 + antcommands: + - package + + - versionName: 0.99.15 + versionCode: 9915 + commit: 0.99.15 + antcommands: + - package + + - versionName: 0.99.18 + versionCode: 9918 + commit: 0.99.18 + antcommands: + - package + + - versionName: 1.0.9 + versionCode: 10011 + commit: 1.0.9 + antcommands: + - package + + - versionName: 1.0.11 + versionCode: 10013 + commit: 1.0.11 + antcommands: + - package + + - versionName: 1.0.12 + versionCode: 10014 + commit: fd349108eff9caa9152a + antcommands: + - package + + - versionName: 1.1.0 + versionCode: 10100 + commit: 5eb993e1fac2898d2361 + antcommands: + - package + + - versionName: 1.1.1 + versionCode: 10101 + commit: 1.1.1 + antcommands: + - package + + - versionName: 1.1.2 + versionCode: 10102 + commit: 1.1.2 + antcommands: + - package + + - versionName: 1.1.8 + versionCode: 101081 + commit: 1.1.8 + antcommands: + - package + + - versionName: 1.1.9 + versionCode: 101091 + commit: 1.1.9 + antcommands: + - package + + - versionName: 1.1.10 + versionCode: 101101 + commit: 13ee5d79431815dd694e + antcommands: + - package + + - versionName: 1.2.2 + versionCode: 102021 + commit: e63c553aeb032da828b270a735f0171d8d22c54c + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + target: android-10 + buildjni: + - yes + + - versionName: 1.2.3 + versionCode: 102031 + commit: 46d83bb4351c2f6ec51e0d9aa6202c86c1297e7f + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + target: android-10 + buildjni: + - yes + + - versionName: 1.2.4 + versionCode: 102041 + commit: 6426bcf131d4 + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + target: android-10 + buildjni: + - yes + + - versionName: 1.2.6 + versionCode: 102061 + commit: 1.2.6 + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + target: android-10 + buildjni: + - yes + + - versionName: 1.3.3 + versionCode: 103031 + commit: 1.3.3 + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + target: android-10 + buildjni: + - yes + + - versionName: 1.3.6 + versionCode: 103061 + commit: a16e3eb7ff731edea99248f8a7c1633148a26236 + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + target: android-10 + buildjni: + - yes + + - versionName: 1.5.5 + versionCode: 105051 + commit: 1.5.5 + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + target: android-10 + buildjni: + - yes + + - versionName: 1.6.1 + versionCode: 106011 + commit: 1.6.1 + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.6.4-Donut + versionCode: 106040 + commit: af881fe37 + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.6.4-Froyo + versionCode: 106041 + commit: 696ed7704 + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.6.4 + versionCode: 106042 + commit: b3b4667ccb + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.7.2-Donut + versionCode: 107019 + commit: a4a5e506b + forceversion: true + forcevercode: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.7.2-Froyo + versionCode: 107020 + commit: 33ffc7e5b + forceversion: true + forcevercode: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.7.2-Honeycomb + versionCode: 107021 + commit: 0520159677 + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.7.3-Donut + versionCode: 107040 + commit: 2c6253dd + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.7.3-Froyo + versionCode: 107041 + commit: 934bf7f5 + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.7.3-Honeycomb + versionCode: 107042 + commit: c4a3c7a9a + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.7.8-Donut + versionCode: 107080 + commit: c1470c9be1 + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.7.8-Froyo + versionCode: 107084 + commit: 1.7.8 + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.7.8-Honeycomb + versionCode: 107085 + commit: 1.7.8-ics + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + buildjni: + - yes + + - versionName: 1.8.2-Donut + versionCode: 108020 + commit: 9bec0ff445e66a + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + androidupdate: + - third-party/AmbilWarna + - . + buildjni: + - yes + + - versionName: 1.8.2-Froyo + versionCode: 108021 + commit: 0f02d4e9232227 + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + androidupdate: + - third-party/AmbilWarna + - . + buildjni: + - yes + + - versionName: 1.8.2-Honeycomb+ + versionCode: 108022 + commit: 1112df415d + forceversion: true + prebuild: + - mkdir res/drawable + - find icons -iname "*.*" -exec cp {} res/drawable \; + androidupdate: + - third-party/AmbilWarna + - . + buildjni: + - yes + + - versionName: 2.0.6-gb + versionCode: 2000610 + disable: missing res with android-9 + commit: 2.0.6 + patch: + - fbreader-2.0.6.patch + srclibs: + - PDFParseLib@36d7479ce85638eb4f0ff9c875ec77680177da5d + - ApacheHttpClient@4.2.5 + - NanoHttpd@Release-2.0.5 + - JsonSimple@tag_release_1_1_1 + forceversion: true + rm: + - libs/*jar + - obsolete/lib/*.jar + prebuild: + - echo -e "sdk.dir=$$SDK$$\nndk.dir=$$NDK$$\n" >> local.properties + - cp local.properties third-party/AmbilWarna/ + - cp local.properties third-party/android-filechooser/code/ + - cp local.properties third-party/drag-sort-listview/library/ + - pushd $$ApacheHttpClient$$/httpmime/ + - $$MVN3$$ package + - popd + - cp $$ApacheHttpClient$$/httpmime/target/httpmime-4.2.5.jar libs/ + - pushd $$NanoHttpd$$ + - $$MVN3$$ package + - popd + - cp $$NanoHttpd$$/core/target/nanohttpd-2.0.5.jar libs/ + - cp -fR $$JsonSimple$$/src/main/java/org src/ + - cp -fR $$PDFParseLib$$/pdfparse-lib/src/main/java/org src/ + - rm -fR src/com/paragon + androidupdate: + - third-party/AmbilWarna + - third-party/android-filechooser/code + - third-party/drag-sort-listview/library + - . + target: android-14 + buildjni: + - yes + + - versionName: 2.0.6-ics + versionCode: 2000620 + commit: b1dd70ff149560889e7548762046da4933e51e94 + patch: + - fbreader-2.0.6.patch + srclibs: + - FBReaderJ@2.0.6 + - PDFParseLib@36d7479ce85638eb4f0ff9c875ec77680177da5d + - ApacheHttpClient@4.2.5 + - NanoHttpd@Release-2.0.5 + - JsonSimple@tag_release_1_1_1 + forceversion: true + rm: + - libs/*jar + - obsolete/lib/*.jar + prebuild: + - echo -e "sdk.dir=$$SDK$$\nndk.dir=$$NDK$$\n" >> local.properties + - cp local.properties third-party/AmbilWarna/ + - cp local.properties third-party/android-filechooser/code/ + - cp local.properties third-party/drag-sort-listview/library/ + - pushd $$ApacheHttpClient$$/httpmime/ + - $$MVN3$$ package + - popd + - cp $$ApacheHttpClient$$/httpmime/target/httpmime-4.2.5.jar libs/ + - pushd $$NanoHttpd$$ + - $$MVN3$$ package + - popd + - cp $$NanoHttpd$$/core/target/nanohttpd-2.0.5.jar libs/ + - cp -fR $$JsonSimple$$/src/main/java/org src/ + - cp -fR $$PDFParseLib$$/pdfparse-lib/src/main/java/org src/ + - rm -fR src/com/paragon + - sed -i -e '/com.google.android.gms.version/d' -e '/google_play_services/d' AndroidManifest.xml + - sed -i -e '/google.services.lib.dir/d' project.properties + - rm -fR src/org/geometerplus/android/fbreader/network/auth + - cp -fR $$FBReaderJ$$/src/org/geometerplus/android/fbreader/network/auth src/org/geometerplus/android/fbreader/network/ + androidupdate: + - third-party/AmbilWarna + - third-party/android-filechooser/code + - third-party/drag-sort-listview/library + - . + target: android-14 + buildjni: + - yes + + - versionName: 2.1-ics + versionCode: 2010020 + commit: 33139e2b04ae36388956a57373ba74e8cc0ef23c + patch: + - fbreader-2.0.6.patch + srclibs: + - FBReaderJ@2.1 + - PDFParseLib@36d7479ce85638eb4f0ff9c875ec77680177da5d + - ApacheHttpClient@4.2.5 + - NanoHttpd@Release-2.0.5 + - JsonSimple@tag_release_1_1_1 + forceversion: true + rm: + - libs/*jar + - obsolete/lib/*.jar + prebuild: + - echo -e "sdk.dir=$$SDK$$\nndk.dir=$$NDK$$\n" >> local.properties + - cp local.properties third-party/AmbilWarna/ + - cp local.properties third-party/android-filechooser/code/ + - cp local.properties third-party/drag-sort-listview/library/ + - pushd $$ApacheHttpClient$$/httpmime/ + - $$MVN3$$ package + - popd + - cp $$ApacheHttpClient$$/httpmime/target/httpmime-4.2.5.jar libs/ + - pushd $$NanoHttpd$$ + - $$MVN3$$ package + - popd + - cp $$NanoHttpd$$/core/target/nanohttpd-2.0.5.jar libs/ + - cp -fR $$JsonSimple$$/src/main/java/org src/ + - cp -fR $$PDFParseLib$$/pdfparse-lib/src/main/java/org src/ + - rm -fR src/com/paragon + - sed -i -e '/com.google.android.gms.version/d' -e '/google_play_services/d' AndroidManifest.xml + - sed -i -e '/google.services.lib.dir/d' project.properties + - rm -fR src/org/geometerplus/android/fbreader/network/auth + - cp -fR $$FBReaderJ$$/src/org/geometerplus/android/fbreader/network/auth src/org/geometerplus/android/fbreader/network/ + androidupdate: + - third-party/AmbilWarna + - third-party/android-filechooser/code + - third-party/drag-sort-listview/library + - . + target: android-14 + buildjni: + - yes + + - versionName: 2.5.9-ics + versionCode: 2050920 + commit: 43e14feedf10ad53ec68bf42b1644f488889381c + srclibs: + - FBReaderJ@2.5.9 + - PDFParseLib@36d7479ce85638eb4f0ff9c875ec77680177da5d + - ApacheHttpClient@4.2.5 + - NanoHttpd@Release-2.0.5 + - JsonSimple@tag_release_1_1_1 + forceversion: true + rm: + - libs/*jar + - obsolete/lib/*.jar + - src/org/geometerplus/android/fbreader/dict/OpenDictionary.java + - src/org/geometerplus/android/fbreader/dict/Lingvo.java + extlibs: + - android/android-support-v4.jar + prebuild: + - echo -e "sdk.dir=$$SDK$$\nndk.dir=$$NDK$$\n" >> local.properties + - cp local.properties third-party/AmbilWarna/ + - cp local.properties third-party/android-filechooser/code/ + - cp local.properties third-party/drag-sort-listview/library/ + - echo 'APP_PLATFORM := android-11' >> jni/Application.mk + - pushd $$ApacheHttpClient$$/httpmime/ + - $$MVN3$$ package -Dmaven.javadoc.skip=true + - popd + - cp $$ApacheHttpClient$$/httpmime/target/httpmime-4.2.5.jar libs/ + - pushd $$NanoHttpd$$ + - $$MVN3$$ package + - popd + - cp $$NanoHttpd$$/core/target/nanohttpd-2.0.5.jar libs/ + - cp -fR $$JsonSimple$$/src/main/java/org src/ + - cp -fR $$PDFParseLib$$/pdfparse-lib/src/main/java/org src/ + - rm -fR src/com/paragon + - sed -i -e '/com.google.android.gms.version/d' -e '/google_play_services/d' AndroidManifest.xml + - sed -i -e '/google.services.lib.dir/d' project.properties + - mkdir third-party/drag-sort-listview/library/libs + - cp libs/android-support-v4.jar third-party/drag-sort-listview/library/libs/&& + sed -i -e '/Lingvo/d' src/org/geometerplus/android/fbreader/dict/DictionaryUtil.java + - rm -fR src/org/geometerplus/android/fbreader/network/auth + - cp -fR $$FBReaderJ$$/src/org/geometerplus/android/fbreader/network/auth src/org/geometerplus/android/fbreader/network/ + - sed -i -e '/^\s*OpenDictionary.collect/d' src/org/geometerplus/android/fbreader/dict/DictionaryUtil.java + androidupdate: + - third-party/AmbilWarna + - third-party/android-filechooser/code + - third-party/drag-sort-listview/library + - . + target: android-21 + buildjni: + - yes + +MaintainerNotes: |- + * LingvoIntegration and OpenDictionary APIs are non-free. Remove jars and patch + depending code. Currently done with rm and sed in prebuilds. + * %v tags are currently targeting gingerbread, but have ressource conflicts + with target=android-9; they build with target=android-14 + * %v-ics tags are actually based on the yotaphone branch, so we have to + use raw commits for the ice-cream-sandwich branch (look for "Merge + branch 'master' into ice-cream-sandwich" after a commit with "version + => ...") + * ics branch uses google play, so we have to sed AM.xml and ant files. + /fbreader/network/ depends on Google Play Services, so these are + removed and replaced with the master branch which does not depend on + these. + * UCM is set to master branch, we don't care for target or device specific + releases. + + TODO: + * make gingerbread/master available for android-9! + +ArchivePolicy: 6 versions +AutoUpdateMode: None +UpdateCheckMode: Tags ^[0-9.]*$ +VercodeOperation: '%c + 10' +CurrentVersion: 2.5.9 +CurrentVersionCode: 2050920 From ee162d9e623f672db0e4a2c74bb6daf1f1c19233 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Wed, 3 Jun 2020 21:30:11 +0200 Subject: [PATCH 3/9] gitignore tests/repo/status --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 57da9fb8..d0955dac 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ README.rst tmp/ /tests/repo/icons* /tests/repo/latestapps.dat +/tests/repo/status # files used in manual testing /config.py From ee4ee85cbd8adf108f5685cf42cc91ee80eb4d75 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Wed, 3 Jun 2020 21:41:28 +0200 Subject: [PATCH 4/9] update:archive_old_apks: handle apps with no CVC If an app doesn't have a CVC, we can just skip any special archive handling. Also rename weirdly named `res` to `apkList`. --- fdroidserver/update.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 06ed01de..2963071d 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -2009,19 +2009,19 @@ def make_categories_txt(repodir, categories): def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversions): - def filter_apk_list_sorted(apk_list): - res = [] + apkList = [] currentVersionApk = None for apk in apk_list: if apk['packageName'] == appid: - if apk['versionCode'] == common.version_code_string_to_int(app.CurrentVersionCode): - currentVersionApk = apk - continue - res.append(apk) + if app.CurrentVersionCode is not None: + if apk['versionCode'] == common.version_code_string_to_int(app.CurrentVersionCode): + currentVersionApk = apk + continue + apkList.append(apk) # Sort the apk list by version code. First is highest/newest. - sorted_list = sorted(res, key=lambda apk: apk['versionCode'], reverse=True) + sorted_list = sorted(apkList, key=lambda apk: apk['versionCode'], reverse=True) if currentVersionApk: # Insert apk which corresponds to currentVersion at the front sorted_list.insert(0, currentVersionApk) From 8c71637d43346ffb06c556d175d16dfb5c7c0df6 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Wed, 3 Jun 2020 22:26:03 +0200 Subject: [PATCH 5/9] update: make copy_triple_t_store_metadata and insert_localized_app_metadata not assume /repo This will enable copying the localized metadata to the archive as well. --- fdroidserver/update.py | 20 ++++++++++---------- tests/update.TestCase | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 2963071d..99595e86 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -977,7 +977,7 @@ def insert_funding_yml_donation_links(apps): break -def copy_triple_t_store_metadata(apps): +def copy_triple_t_store_metadata(apps, repodir): """Include store metadata from the app's source repo The Triple-T Gradle Play Publisher is a plugin that has a standard @@ -1091,14 +1091,14 @@ def copy_triple_t_store_metadata(apps): dirname = SCREENSHOT_DIRS[tt_screenshot_dirs.index(dirname)] else: locale = segments[-2] - destdir = os.path.join('repo', packageName, locale, dirname) + destdir = os.path.join(repodir, packageName, locale, dirname) os.makedirs(destdir, mode=0o755, exist_ok=True) sourcefile = os.path.join(root, f) destfile = os.path.join(destdir, repofilename) _strip_and_copy_image(sourcefile, destfile) -def insert_localized_app_metadata(apps): +def insert_localized_app_metadata(apps, repodir): """scans standard locations for graphics and localized text Scans for localized description files, changelogs, store graphics, and @@ -1113,7 +1113,7 @@ def insert_localized_app_metadata(apps): ...as well as the /metadata// directory. If it finds them, they will be added to the dict of all packages, with the - versions in the /metadata/ folder taking precendence over the what + versions in the /metadata/ folder taking precedence over the what is in the app's source repo. The is the locale of the files supplied in that directory, using @@ -1142,7 +1142,7 @@ def insert_localized_app_metadata(apps): logging.debug(packageName + ' does not have app metadata, skipping l18n scan.') continue locale = segments[-1] - destdir = os.path.join('repo', packageName, locale) + destdir = os.path.join(repodir, packageName, locale) # flavours specified in build receipt build_flavours = "" @@ -1180,7 +1180,7 @@ def insert_localized_app_metadata(apps): base, extension = common.get_extension(f) if locale == 'images': locale = segments[-2] - destdir = os.path.join('repo', packageName, locale) + destdir = os.path.join(repodir, packageName, locale) if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS: os.makedirs(destdir, mode=0o755, exist_ok=True) _strip_and_copy_image(os.path.join(root, f), destdir) @@ -1188,7 +1188,7 @@ def insert_localized_app_metadata(apps): if d in SCREENSHOT_DIRS: if locale == 'images': locale = segments[-2] - destdir = os.path.join('repo', packageName, locale) + destdir = os.path.join(repodir, packageName, locale) for f in glob.glob(os.path.join(root, d, '*.*')): _ignored, extension = common.get_extension(f) if extension in ALLOWED_EXTENSIONS: @@ -1196,7 +1196,7 @@ def insert_localized_app_metadata(apps): os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True) _strip_and_copy_image(f, screenshotdestdir) - repodirs = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z]*'))) + repodirs = sorted(glob.glob(os.path.join(repodir, '[A-Za-z]*', '[a-z][a-z]*'))) for d in repodirs: if not os.path.isdir(d): continue @@ -2303,9 +2303,9 @@ def main(): logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.')) insert_funding_yml_donation_links(apps) - copy_triple_t_store_metadata(apps) + copy_triple_t_store_metadata(apps, repodirs[0]) insert_obbs(repodirs[0], apps, apks) - insert_localized_app_metadata(apps) + insert_localized_app_metadata(apps, repodirs[0]) translate_per_build_anti_features(apps, apks) # Scan the archive repo for apks as well diff --git a/tests/update.TestCase b/tests/update.TestCase index eee01734..11f544cd 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -103,7 +103,7 @@ class UpdateTest(unittest.TestCase): build_conversations.gradle = ['free'] apps['eu.siacs.conversations']['builds'] = [build_conversations] - fdroidserver.update.insert_localized_app_metadata(apps) + fdroidserver.update.insert_localized_app_metadata(apps, 'repo') appdir = os.path.join('repo', 'info.guardianproject.urzip', 'en-US') self.assertTrue(os.path.isfile(os.path.join( @@ -183,7 +183,7 @@ class UpdateTest(unittest.TestCase): os.chdir(tmptestsdir) apps = fdroidserver.metadata.read_metadata(xref=True) - fdroidserver.update.copy_triple_t_store_metadata(apps) + fdroidserver.update.copy_triple_t_store_metadata(apps, 'repo') # TODO ideally, this would compare the whole dict like in metadata.TestCase's test_read_metadata() correctlocales = [ @@ -211,7 +211,7 @@ class UpdateTest(unittest.TestCase): apps = fdroidserver.metadata.read_metadata(xref=True) self.assertTrue(packageName in apps) - fdroidserver.update.copy_triple_t_store_metadata(apps) + fdroidserver.update.copy_triple_t_store_metadata(apps, 'repo') correctlocales = ['de-DE', 'en-US', 'fr-FR', 'kn-IN'] app = apps[packageName] self.assertEqual('android@piwigo.org', app['authorEmail']) From e66683720b6f6e02c08d2d06f5ba55bfd33ed5a7 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Wed, 3 Jun 2020 22:27:09 +0200 Subject: [PATCH 6/9] move index sorting to index module This is a historic detail of the index format, so move it there. For wiki update and status json there's really no reason why this should be done in alphabetic app name order. Use the default sort order by appid. --- fdroidserver/index.py | 6 ++++-- fdroidserver/update.py | 23 +++++++++-------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 195b23e7..00d3f3f8 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -44,13 +44,12 @@ from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid from fdroidserver.exception import FDroidException, VerificationException, MetaDataException -def make(apps, sortedids, apks, repodir, archive): +def make(apps, apks, repodir, archive): """Generate the repo index files. This requires properly initialized options and config objects. :param apps: fully populated apps list - :param sortedids: app package IDs, sorted :param apks: full populated apks list :param repodir: the repo directory :param archive: True if this is the archive repo, False if it's the @@ -101,6 +100,9 @@ def make(apps, sortedids, apks, repodir, archive): if mirrors: repodict['mirrors'] = mirrors + # Historically the index has been sorted by App Name, so we enforce this ordering here + sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper()) + appsWithPackages = collections.OrderedDict() for packageName in sortedids: app = apps[packageName] diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 99595e86..f2575f07 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -126,7 +126,7 @@ def disabled_algorithms_allowed(): return options.allow_disabled_algorithms or config['allow_disabled_algorithms'] -def status_update_json(apps, sortedids, apks): +def status_update_json(apps, apks): """Output a JSON file with metadata about this `fdroid update` run :param apps: fully populated list of all applications @@ -141,7 +141,7 @@ def status_update_json(apps, sortedids, apks): output['failedBuilds'] = dict() output['noPackages'] = [] - for appid in sortedids: + for appid in apps: app = apps[appid] for af in app.get('AntiFeatures', []): antiFeatures = output['antiFeatures'] # JSON camelCase @@ -177,7 +177,7 @@ def status_update_json(apps, sortedids, apks): common.write_status_json(output, options.pretty) -def update_wiki(apps, sortedids, apks): +def update_wiki(apps, apks): """Update the wiki :param apps: fully populated list of all applications @@ -193,7 +193,7 @@ def update_wiki(apps, sortedids, apks): generated_pages = {} generated_redirects = {} - for appid in sortedids: + for appid in apps: app = metadata.App(apps[appid]) wikidata = '' @@ -2319,11 +2319,6 @@ def main(): # Apply information from latest apks to the application and update dates apply_info_from_latest_apk(apps, apks + archapks) - # Sort the app list by name, then the web site doesn't have to by default. - # (we had to wait until we'd scanned the apks to do this, because mostly the - # name comes from there!) - sortedids = sorted(apps.keys(), key=lambda appid: apps[appid].Name.upper()) - # APKs are placed into multiple repos based on the app package, providing # per-app subscription feeds for nightly builds and things like it if config['per_app_repos']: @@ -2333,7 +2328,7 @@ def main(): appdict = dict() appdict[appid] = app if os.path.isdir(repodir): - index.make(appdict, [appid], apks, repodir, False) + index.make(appdict, apks, repodir, False) else: logging.info(_('Skipping index generation for {appid}').format(appid=appid)) return @@ -2342,7 +2337,7 @@ def main(): archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older']) # Make the index for the main repo... - index.make(apps, sortedids, apks, repodirs[0], False) + index.make(apps, apks, repodirs[0], False) make_categories_txt(repodirs[0], categories) # If there's an archive repo, make the index for it. We already scanned it @@ -2350,7 +2345,7 @@ def main(): if len(repodirs) > 1: archived_apps = copy.deepcopy(apps) apply_info_from_latest_apk(archived_apps, archapks) - index.make(archived_apps, sortedids, archapks, repodirs[1], True) + index.make(archived_apps, archapks, repodirs[1], True) git_remote = config.get('binary_transparency_remote') if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): @@ -2381,8 +2376,8 @@ def main(): # Update the wiki... if options.wiki: - update_wiki(apps, sortedids, apks + archapks) - status_update_json(apps, sortedids, apks + archapks) + update_wiki(apps, apks + archapks) + status_update_json(apps, apks + archapks) logging.info(_("Finished")) From d720c99ae517384b7f1d445108a40067b9097b52 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Wed, 3 Jun 2020 22:28:18 +0200 Subject: [PATCH 7/9] refactor update.py:main This makes update.py:main a lot saner by removing a lot of the implicit assumptions between the different stages of generating the repository index. * mostly unify repo and archive processing, that means the archive is now actually getting the same treatment regarding i.e. fastlane data. Previously the archive didn't get considered at all here. * already filter the list of apps to include in a repo in update.py and give that prefiltered list to index. This makes sure we actually only copy fastlane/triple-t/etc. stuff for apps ending up in the index. This both, can save a lot of time if there are a lot of old /build dirs lying around and doesn't clutter /repo with things that aren't referenced from the index. Closes fdroid/fdroidserver#524 --- fdroidserver/index.py | 36 ++++++---------- fdroidserver/update.py | 97 +++++++++++++++++++++++++++++++++--------- 2 files changed, 90 insertions(+), 43 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 00d3f3f8..af5c3522 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -21,7 +21,6 @@ # along with this program. If not, see . import collections -import copy import json import logging import os @@ -41,7 +40,7 @@ from . import metadata from . import net from . import signindex from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints -from fdroidserver.exception import FDroidException, VerificationException, MetaDataException +from fdroidserver.exception import FDroidException, VerificationException def make(apps, apks, repodir, archive): @@ -49,8 +48,9 @@ def make(apps, apks, repodir, archive): This requires properly initialized options and config objects. - :param apps: fully populated apps list - :param apks: full populated apks list + :param apps: OrderedDict of apps to go into the index, each app should have + at least one associated apk + :param apks: list of apks to go into the index :param repodir: the repo directory :param archive: True if this is the archive repo, False if it's the main one. @@ -60,6 +60,12 @@ def make(apps, apks, repodir, archive): if not common.options.nosign: common.assert_config_keystore(common.config) + # Historically the index has been sorted by App Name, so we enforce this ordering here + sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper()) + sortedapps = collections.OrderedDict() + for appid in sortedids: + sortedapps[appid] = apps[appid] + repodict = collections.OrderedDict() repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc) repodict['version'] = METADATA_VERSION @@ -100,24 +106,6 @@ def make(apps, apks, repodir, archive): if mirrors: repodict['mirrors'] = mirrors - # Historically the index has been sorted by App Name, so we enforce this ordering here - sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper()) - - appsWithPackages = collections.OrderedDict() - for packageName in sortedids: - app = apps[packageName] - if app['Disabled']: - continue - - # only include apps with packages - for apk in apks: - if apk['packageName'] == packageName: - newapp = copy.copy(app) # update wiki needs unmodified description - newapp['Description'] = metadata.description_html(app['Description'], - metadata.DescriptionResolver(apps)) - appsWithPackages[packageName] = newapp - break - requestsdict = collections.OrderedDict() for command in ('install', 'uninstall'): packageNames = [] @@ -133,9 +121,9 @@ def make(apps, apks, repodir, archive): fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints() - make_v0(appsWithPackages, apks, repodir, repodict, requestsdict, + make_v0(sortedapps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints) - make_v1(appsWithPackages, apks, repodir, repodict, requestsdict, + make_v1(sortedapps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fingerprints) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index f2575f07..64c5c6b3 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -2166,6 +2166,71 @@ def create_metadata_from_template(apk): logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName'])) +def read_names_from_apks(apps, apks): + """This is a stripped down copy of apply_info_from_latest_apk that only parses app names""" + for appid, app in apps.items(): + bestver = UNSET_VERSION_CODE + for apk in apks: + if apk['packageName'] == appid: + if apk['versionCode'] > bestver: + bestver = apk['versionCode'] + bestapk = apk + + if bestver == UNSET_VERSION_CODE: + if app.Name is None: + app.Name = app.AutoName or appid + app.icon = None + else: + if app.Name is None: + app.Name = bestapk['name'] + + +def render_app_descriptions(apps, all_apps): + """ + Renders the app html description. + For resolving inter-app links it needs the full list of apps, even if they end up in + separate repos (i.e. archive or per app repos). + """ + for app in apps.values(): + app['Description'] = metadata.description_html(app['Description'], metadata.DescriptionResolver(all_apps)) + + +def get_apps_with_packages(apps, apks): + """Returns a deepcopy of that subset apps that actually has any associated packages. Skips disabled apps.""" + appsWithPackages = collections.OrderedDict() + for packageName in apps: + app = apps[packageName] + if app['Disabled']: + continue + + # only include apps with packages + for apk in apks: + if apk['packageName'] == packageName: + newapp = copy.copy(app) + appsWithPackages[packageName] = newapp + break + return appsWithPackages + + +def prepare_apps(apps, apks, repodir): + """Encapsulates all necessary preparation steps before we can build an index out of apps and apks. + + :param apps: All apps as read from metadata + :param apks: list of apks that belong into repo, this gets modified in place + :param repodir: the target repository directory, metadata files will be copied here + :return: the relevant subset of apps (as a deepcopy) + """ + apps_with_packages = get_apps_with_packages(apps, apks) + apply_info_from_latest_apk(apps_with_packages, apks) + render_app_descriptions(apps_with_packages, apps) + insert_funding_yml_donation_links(apps) + copy_triple_t_store_metadata(apps_with_packages, repodir) + insert_obbs(repodir, apps_with_packages, apks) + translate_per_build_anti_features(apps_with_packages, apks) + insert_localized_app_metadata(apps_with_packages, repodir) + return apps_with_packages + + config = None options = None start_timestamp = time.gmtime() @@ -2302,12 +2367,6 @@ def main(): else: logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.')) - insert_funding_yml_donation_links(apps) - copy_triple_t_store_metadata(apps, repodirs[0]) - insert_obbs(repodirs[0], apps, apks) - insert_localized_app_metadata(apps, repodirs[0]) - translate_per_build_anti_features(apps, apks) - # Scan the archive repo for apks as well if len(repodirs) > 1: archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk) @@ -2316,8 +2375,18 @@ def main(): else: archapks = [] - # Apply information from latest apks to the application and update dates - apply_info_from_latest_apk(apps, apks + archapks) + # We need app.Name populated for all apps regardless of which repo they end up in + # for the old-style inter-app links, so let's do it before we do anything else. + # This will be done again (as part of apply_info_from_latest_apk) for repo and archive + # separately later on, but it's fairly cheap anyway. + read_names_from_apks(apps, apks + archapks) + + if len(repodirs) > 1: + archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older']) + archived_apps = prepare_apps(apps, archapks, repodirs[1]) + index.make(archived_apps, archapks, repodirs[1], True) + + repoapps = prepare_apps(apps, apks, repodirs[0]) # APKs are placed into multiple repos based on the app package, providing # per-app subscription feeds for nightly builds and things like it @@ -2333,20 +2402,10 @@ def main(): logging.info(_('Skipping index generation for {appid}').format(appid=appid)) return - if len(repodirs) > 1: - archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older']) - # Make the index for the main repo... - index.make(apps, apks, repodirs[0], False) + index.make(repoapps, apks, repodirs[0], False) make_categories_txt(repodirs[0], categories) - # If there's an archive repo, make the index for it. We already scanned it - # earlier on. - if len(repodirs) > 1: - archived_apps = copy.deepcopy(apps) - apply_info_from_latest_apk(archived_apps, archapks) - index.make(archived_apps, archapks, repodirs[1], True) - git_remote = config.get('binary_transparency_remote') if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')): from . import btlog From 07caa889205edb94955c6c26946a77bfc426edac Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Wed, 24 Jun 2020 17:14:01 +0200 Subject: [PATCH 8/9] don't include the localized metadata things for /archive We haven't done this so far and it's a potential big change in archive size and update performance. --- fdroidserver/update.py | 22 +++++++++++++--------- tests/update.TestCase | 6 +++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 64c5c6b3..babc3254 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -977,7 +977,7 @@ def insert_funding_yml_donation_links(apps): break -def copy_triple_t_store_metadata(apps, repodir): +def copy_triple_t_store_metadata(apps): """Include store metadata from the app's source repo The Triple-T Gradle Play Publisher is a plugin that has a standard @@ -1091,14 +1091,14 @@ def copy_triple_t_store_metadata(apps, repodir): dirname = SCREENSHOT_DIRS[tt_screenshot_dirs.index(dirname)] else: locale = segments[-2] - destdir = os.path.join(repodir, packageName, locale, dirname) + destdir = os.path.join('repo', packageName, locale, dirname) os.makedirs(destdir, mode=0o755, exist_ok=True) sourcefile = os.path.join(root, f) destfile = os.path.join(destdir, repofilename) _strip_and_copy_image(sourcefile, destfile) -def insert_localized_app_metadata(apps, repodir): +def insert_localized_app_metadata(apps): """scans standard locations for graphics and localized text Scans for localized description files, changelogs, store graphics, and @@ -1142,7 +1142,7 @@ def insert_localized_app_metadata(apps, repodir): logging.debug(packageName + ' does not have app metadata, skipping l18n scan.') continue locale = segments[-1] - destdir = os.path.join(repodir, packageName, locale) + destdir = os.path.join('repo', packageName, locale) # flavours specified in build receipt build_flavours = "" @@ -1180,7 +1180,7 @@ def insert_localized_app_metadata(apps, repodir): base, extension = common.get_extension(f) if locale == 'images': locale = segments[-2] - destdir = os.path.join(repodir, packageName, locale) + destdir = os.path.join('repo', packageName, locale) if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS: os.makedirs(destdir, mode=0o755, exist_ok=True) _strip_and_copy_image(os.path.join(root, f), destdir) @@ -1188,7 +1188,7 @@ def insert_localized_app_metadata(apps, repodir): if d in SCREENSHOT_DIRS: if locale == 'images': locale = segments[-2] - destdir = os.path.join(repodir, packageName, locale) + destdir = os.path.join('repo', packageName, locale) for f in glob.glob(os.path.join(root, d, '*.*')): _ignored, extension = common.get_extension(f) if extension in ALLOWED_EXTENSIONS: @@ -1196,7 +1196,7 @@ def insert_localized_app_metadata(apps, repodir): os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True) _strip_and_copy_image(f, screenshotdestdir) - repodirs = sorted(glob.glob(os.path.join(repodir, '[A-Za-z]*', '[a-z][a-z]*'))) + repodirs = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z]*'))) for d in repodirs: if not os.path.isdir(d): continue @@ -2224,10 +2224,14 @@ def prepare_apps(apps, apks, repodir): apply_info_from_latest_apk(apps_with_packages, apks) render_app_descriptions(apps_with_packages, apps) insert_funding_yml_donation_links(apps) - copy_triple_t_store_metadata(apps_with_packages, repodir) + # This is only currently done for /repo because doing it for the archive + # will take a lot of time and bloat the archive mirrors and index + if repodir == 'repo': + copy_triple_t_store_metadata(apps_with_packages) insert_obbs(repodir, apps_with_packages, apks) translate_per_build_anti_features(apps_with_packages, apks) - insert_localized_app_metadata(apps_with_packages, repodir) + if repodir == 'repo': + insert_localized_app_metadata(apps_with_packages) return apps_with_packages diff --git a/tests/update.TestCase b/tests/update.TestCase index 11f544cd..eee01734 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -103,7 +103,7 @@ class UpdateTest(unittest.TestCase): build_conversations.gradle = ['free'] apps['eu.siacs.conversations']['builds'] = [build_conversations] - fdroidserver.update.insert_localized_app_metadata(apps, 'repo') + fdroidserver.update.insert_localized_app_metadata(apps) appdir = os.path.join('repo', 'info.guardianproject.urzip', 'en-US') self.assertTrue(os.path.isfile(os.path.join( @@ -183,7 +183,7 @@ class UpdateTest(unittest.TestCase): os.chdir(tmptestsdir) apps = fdroidserver.metadata.read_metadata(xref=True) - fdroidserver.update.copy_triple_t_store_metadata(apps, 'repo') + fdroidserver.update.copy_triple_t_store_metadata(apps) # TODO ideally, this would compare the whole dict like in metadata.TestCase's test_read_metadata() correctlocales = [ @@ -211,7 +211,7 @@ class UpdateTest(unittest.TestCase): apps = fdroidserver.metadata.read_metadata(xref=True) self.assertTrue(packageName in apps) - fdroidserver.update.copy_triple_t_store_metadata(apps, 'repo') + fdroidserver.update.copy_triple_t_store_metadata(apps) correctlocales = ['de-DE', 'en-US', 'fr-FR', 'kn-IN'] app = apps[packageName] self.assertEqual('android@piwigo.org', app['authorEmail']) From 3ebc44c54ff47337c1a666df91d9f4245cfd97a5 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Wed, 24 Jun 2020 20:30:26 +0200 Subject: [PATCH 9/9] fix tests after they switched to yaml --- tests/metadata.TestCase | 1 - tests/run-tests | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 49151083..f9ba731d 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -906,7 +906,6 @@ class MetadataTest(unittest.TestCase): 'Prepare': None}}) def test_read_xref_metadata(self): - fdroidserver.common.config = {'accepted_formats': ['yml']} os.chdir('xref') fdroidserver.metadata.warnings_action = 'error' apps = fdroidserver.metadata.read_metadata(xref=True) diff --git a/tests/run-tests b/tests/run-tests index d685ea1f..a5435351 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -612,12 +612,11 @@ echo_header "test whatsnew from fastlane without CVC set" REPOROOT=`create_test_dir` cd $REPOROOT fdroid_init_with_prebuilt_keystore -echo "accepted_formats = ['txt', 'yml']" >> config.py mkdir -p metadata/com.politedroid/en-US/changelogs/ cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo -cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata +cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata echo "whatsnew test" > metadata/com.politedroid/en-US/changelogs/6.txt -sed -i -e '/Current Version/d' metadata/com.politedroid.txt +sed -i -e '/CurrentVersion/d' metadata/com.politedroid.yml $fdroid update --pretty --nosign grep -F 'whatsnew' repo/index-v1.json