From 7f6efa74f59a69962184a5a3cba2981223803990 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Mon, 23 Dec 2019 20:07:56 +0100 Subject: [PATCH 1/3] tests: add test for correctly handling CVC when archiving --- tests/keystore.jks | Bin 3760 -> 5967 bytes tests/run-tests | 15 +++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/tests/keystore.jks b/tests/keystore.jks index 1eb91fe78cd6b773f05764a10a25b2809392c3d7..ca13f91754e1bb641da876784ff0aa59e77e1a56 100644 GIT binary patch delta 2252 zcmb`Ic{J1u8^?dYnK78LFLkd%*+rRQWK@G+Cl#P=rgl)DW^q z2-(+c7g@?0YD!tkb?MbP_dTa`-hbaeoO%`}jFg&@ZvPq~Y?#_g*gQ!mc)D6|`Zm={y@jK(|1o>Hgr zH~AxX4PisZ7L)YOr6a{Fh&x`Qi8uMIlQI;BsNNzt?TY+su@QhUe|klb4*}M>PRsc; zu-q!P8pl;rCIE?o&-5O(l4&fe=>;veUax!hs^%B{;~>1lxSnrNwKn|?$*%Boy-?E! z`O!VmZawhZF@mVD%#H&+3?d_?1^vz9lbT#vf^SqryIr}rIxFo*xh9C0IDO5L!nmQKf0S*dO%Rh*qZ5kn*ZjL1xTG zRLt_(7TXYC%Xuen9_-SgmPn0fmD6z95d2Q!H+h{QMrqeW;F1*7-t^*S&JcCK%Ftd}o*AOX z`8l*pjXdG5PTB3FYn^H{A|gTw&s_$$X)B0lTJTMZ?q};q@;|VqECs}%O9^_m$2k@UV#QbtbhX zLqbnK>FJScG5Dl>%lm~sr>imEx!wOY-cEe1cj`o^{@ON zFZ5r{{GYwVfKtCkaqu#rF(5yHiUARj7!U;VPwx2BE4!=8xSJ)f`|%#CZH|*^yjRh| zr=z9HmSzV$_9Gib`7h;II4Xolt}n;{JT|0?b)wYiwo&O6`~pOE?#G#j!OJA=Npnhs z(R#xMq_d)+OkgPA$Bp6G=~&PF3;s+sVtuGYh%42uuQHmgS1R8JS%0}Nlziv=R{?U3 zn%y`0q0Z9K^X!-9b4|q!k-r=Ar_?CMt5})|x;8n9ip#K83aMY{%X+~dou?YLz7~jk zX_kx0sIrm6Vk0aZ!UXLh=c6K#91uSk| z*YxX8z_ZG;o$~XQv3;K#G{`&3=|$?Vhak{Y7ak$H|NO}QxN}kPDOGjCZc}uIXh1^E zugM)8(Lr(`057)>+`hjRLaR)nE?J(%<-a(zN83=kT{*v_Vdfj2Nt3i~%C*0Nh(i^( z+=?`Qn>n`QLMa=YRU(q>W(C`N20};W2#s0#OGIke#ldsd{UUbI#&JG6b7Qhc@;(yA zy_IXpB!}~*>P+lY@V>@|+_B>@f5#9vFvoV*U@pf(SNuLp+FiKDSxDIwAaj=hmpM} zbqgz)V4+pz@IedSAadfz{O3F=^9p>MPQdV4eVho}cjDu2-|JyXT%|SsNzT%?*Mm*M jfXAQT7t49>@`=oPJ*A|Dru|c1s7Tm)lR{xSDv$mLLEhVz delta 43 zcmX@Fw?UTY-`jt085kItfS7UfYK{qfB3mxxe0skzDK}yJ+rT@' archive/index.xml | wc -l` -eq 3 + test `grep '' repo/index.xml | wc -l` -eq 1 + grep -F com.politedroid_3.apk archive/index.xml + grep -F com.politedroid_4.apk archive/index.xml + grep -F com.politedroid_5.apk repo/index.xml + grep -F com.politedroid_6.apk archive/index.xml + test -e archive/com.politedroid_3.apk + test -e archive/com.politedroid_4.apk + test -e repo/com.politedroid_5.apk + ! test -e repo/com.politedroid_6.apk + test -e archive/com.politedroid_6.apk fi From a7a83e1ee35cf38ab102261d514c0933468ac94b Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Mon, 23 Dec 2019 16:32:14 +0100 Subject: [PATCH 2/3] update: keep CVC of an app out of the archive We want to keep the stable version (CV) of an app in /repo with highest priority and only move it to /archive when ArchivePolicy is set to 0. For this to work we need two changes here: 1) when sorting apks by version code we insert the apk corresponding to the CVC of the app afterwards with highest priority. So when walking the list of apks afterwards the CVC apk is always kept first. 2) Instead of the two pass algorithm of moving things back and forth to the archive we instead figure out where each apk of an app goes in first before actually moving them into the right place. Fixes: fdroid/fdroidserver#385 --- fdroidserver/update.py | 50 +++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 463a1c7d..9225c241 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1823,12 +1823,20 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi def filter_apk_list_sorted(apk_list): res = [] + currentVersionApk = None for apk in apk_list: if apk['packageName'] == appid: + if apk['versionCode'] == common.version_code_string_to_int(app.CurrentVersionCode): + currentVersionApk = apk + continue res.append(apk) # Sort the apk list by version code. First is highest/newest. - return sorted(res, key=lambda apk: apk['versionCode'], reverse=True) + sorted_list = sorted(res, key=lambda apk: apk['versionCode'], reverse=True) + if currentVersionApk: + # Insert apk which corresponds to currentVersion at the front + sorted_list.insert(0, currentVersionApk) + return sorted_list for appid, app in apps.items(): @@ -1840,26 +1848,28 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi logging.debug(_("Checking archiving for {appid} - apks:{integer}, keepversions:{keep}, archapks:{arch}") .format(appid=appid, integer=len(apks), keep=keepversions, arch=len(archapks))) - current_app_apks = filter_apk_list_sorted(apks) - if len(current_app_apks) > keepversions: - # Move back the ones we don't want. - for apk in current_app_apks[keepversions:]: - move_apk_between_sections(repodir, archivedir, apk) - archapks.append(apk) - apks.remove(apk) + all_app_apks = filter_apk_list_sorted(apks + archapks) - current_app_archapks = filter_apk_list_sorted(archapks) - if len(current_app_apks) < keepversions and len(current_app_archapks) > 0: - kept = 0 - # Move forward the ones we want again, except DisableAlgorithm - for apk in current_app_archapks: - if 'DisabledAlgorithm' not in apk['antiFeatures']: - move_apk_between_sections(archivedir, repodir, apk) - archapks.remove(apk) - apks.append(apk) - kept += 1 - if kept == keepversions: - break + # determine which apks to keep in repo + keep = [] + for apk in all_app_apks: + if len(keep) == keepversions: + break + if 'antiFeatures' not in apk: + keep.append(apk) + elif 'DisabledAlgorithm' not in apk['antiFeatures']: + keep.append(apk) + + # actually move apks to the target section + for apk in all_app_apks: + if apk in apks and apk not in keep: + apks.remove(apk) + archapks.append(apk) + move_apk_between_sections(repodir, archivedir, apk) + elif apk in archapks and apk in keep: + archapks.remove(apk) + apks.append(apk) + move_apk_between_sections(archivedir, repodir, apk) def move_apk_between_sections(from_dir, to_dir, apk): From 83526e09a35aafcf1316fae844fa7604e9a659e1 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Tue, 24 Dec 2019 17:18:42 +0100 Subject: [PATCH 3/3] update: fix unarchiving of allowed disabled algorithm 6d0b1bbe6fae0909683f2c6a154515bc4bfcb674 didn't handle the allow_disabled_algorithm case at all, so we add it back. This additionally fixes a (previously existing) bug where setting allow_disabled_algorithms to True didn't move apks back from archive to repo. Introduce a new test for this. The disabled_algorithm archiving logic is still all over the place so ideally that needs a future refactor. --- fdroidserver/update.py | 10 +++++++--- tests/run-tests | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 9225c241..bc2583fc 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -117,6 +117,10 @@ def get_all_icon_dirs(repodir): yield get_icon_dir(repodir, density) +def disabled_algorithms_allowed(): + return options.allow_disabled_algorithms or config['allow_disabled_algorithms'] + + def update_wiki(apps, sortedids, apks): """Update the wiki @@ -460,7 +464,7 @@ def get_cache(): """ apkcachefile = get_cache_file() - ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms'] + ada = disabled_algorithms_allowed() if not options.clean and os.path.exists(apkcachefile): with open(apkcachefile) as fp: apkcache = json.load(fp, object_pairs_hook=collections.OrderedDict) @@ -1598,7 +1602,7 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False): apks = [] for apkfile in sorted(glob.glob(os.path.join(repodir, '*.apk'))): apkfilename = apkfile[len(repodir) + 1:] - ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms'] + ada = disabled_algorithms_allowed() (skip, apk, cachethis) = process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk, ada, True) if skip: @@ -1857,7 +1861,7 @@ def archive_old_apks(apps, apks, archapks, repodir, archivedir, defaultkeepversi break if 'antiFeatures' not in apk: keep.append(apk) - elif 'DisabledAlgorithm' not in apk['antiFeatures']: + elif 'DisabledAlgorithm' not in apk['antiFeatures'] or disabled_algorithms_allowed(): keep.append(apk) # actually move apks to the target section diff --git a/tests/run-tests b/tests/run-tests index 16ec8bb3..7ae22fe9 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -555,6 +555,30 @@ if ! which apksigner; then test -e repo/com.politedroid_6.apk fi +# test unarchiving when disabled_algorithms are allowed again +echo 'allow_disabled_algorithms = True' >> config.py +$fdroid update --pretty --nosign +test `grep '' archive/index.xml | wc -l` -eq 2 +test `grep '' repo/index.xml | wc -l` -eq 6 +grep -F com.politedroid_3.apk archive/index.xml +grep -F com.politedroid_4.apk repo/index.xml +grep -F com.politedroid_5.apk repo/index.xml +grep -F com.politedroid_6.apk repo/index.xml +grep -F org.bitbucket.tickytacky.mirrormirror_1.apk archive/index.xml +grep -F org.bitbucket.tickytacky.mirrormirror_2.apk repo/index.xml +grep -F org.bitbucket.tickytacky.mirrormirror_3.apk repo/index.xml +grep -F org.bitbucket.tickytacky.mirrormirror_4.apk repo/index.xml +! grep -F urzip-badsig.apk repo/index.xml +! grep -F urzip-badsig.apk archive/index.xml +test -e archive/com.politedroid_3.apk +test -e repo/com.politedroid_4.apk +test -e repo/com.politedroid_5.apk +test -e repo/com.politedroid_6.apk +test -e archive/org.bitbucket.tickytacky.mirrormirror_1.apk +test -e repo/org.bitbucket.tickytacky.mirrormirror_2.apk +test -e repo/org.bitbucket.tickytacky.mirrormirror_3.apk +test -e repo/org.bitbucket.tickytacky.mirrormirror_4.apk +test -e archive/urzip-badsig.apk #------------------------------------------------------------------------------# echo_header 'rename apks with `fdroid update --rename-apks`, --nosign for speed'