From afac1b2efd2152ffbb73091eb1a01a95ff1e51f5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 25 May 2017 20:39:15 +0200 Subject: [PATCH 1/9] add basic test for `fdroid scanner` There was no test coverage at all for this command, this is a very basic test that should prevent things like 2626858450953ac65124765d2cd73d1602846372 --- tests/run-tests | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/run-tests b/tests/run-tests index f640322e..931d6e06 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -178,6 +178,28 @@ else echo 'WARNING: Skipping `fdroid build` test since android-23 is missing!' fi +#------------------------------------------------------------------------------# +echo_header 'copy git import and run `fdroid scanner` on it' + +REPOROOT=`create_test_dir` +cd $REPOROOT +touch config.py +cp $WORKSPACE/examples/fdroid-icon.png $REPOROOT/ +mkdir metadata +echo "Auto Name:Just A Test" > metadata/org.fdroid.ci.test.app.txt +echo "Web Site:" >> metadata/org.fdroid.ci.test.app.txt +echo "Build:0.3,300" >> metadata/org.fdroid.ci.test.app.txt +echo " commit=0.3" >> metadata/org.fdroid.ci.test.app.txt +echo " subdir=app" >> metadata/org.fdroid.ci.test.app.txt +echo " gradle=yes" >> metadata/org.fdroid.ci.test.app.txt +echo "" >> metadata/org.fdroid.ci.test.app.txt +echo "Repo:https://gitlab.com/fdroid/ci-test-app.git" >> metadata/org.fdroid.ci.test.app.txt +echo "Repo Type:git" >> metadata/org.fdroid.ci.test.app.txt +mkdir build +cp -a $WORKSPACE/tests/tmp/importer build/org.fdroid.ci.test.app +ls -l build/org.fdroid.ci.test.app +$fdroid scanner org.fdroid.ci.test.app --verbose + #------------------------------------------------------------------------------# echo_header "copy tests/repo, generate java/gpg keys, update, and gpgsign" From 2c6945dac70edd565438dc6711872e3504fc3e80 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sat, 27 May 2017 21:13:36 +0200 Subject: [PATCH 2/9] gitlab-ci: add index v0 metadata parsing test This test is very handy for making sure the old index.xml v0 format does not inadvertantly change. --- .gitlab-ci.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbda46b7..a04c539c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,3 +8,24 @@ test: - pip3 install -e . - cd tests - ./complete-ci-tests + +# Test that the parsing of the .txt format didn't change. The metadata +# field 'Author Web Site' was added after 0.7.0, so that can't be part +# of the test. +metadata_v0: + script: + - cd tests + - cp dump_internal_metadata_format.py dump.py # since this isn't in old commits + - git checkout 0.7.0 # or any old commit of your choosing + - cd .. + - sed -i "s/'Author Email',/'Author Email',\n'Author Web Site',/" fdroidserver/metadata.py + - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata + - cd fdroiddata + - ../tests/dump.py + - cd .. + - git reset --hard + - git checkout master + - cd fdroiddata + - ../tests/dump.py + - sed -i "/AuthorWebSite/d" metadata/dump_*/*.yaml + - diff -uw metadata/dump_* From 0f4cbc7224409c8741dec9a2cc19f23c4d0aaacc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 30 May 2017 14:52:33 +0200 Subject: [PATCH 3/9] allow APKs with same packageName/versionCode but different signer There are many APKs out in the wild that claim to be the same app and version and each other, but they are signed by different keys. fdroid should be able to index these, and work with them. This supports having the developer's signature via reproducible builds, random collections of APKs like repomaker, etc. --- fdroidserver/index.py | 13 ++++++++--- tests/repo/index.xml | 2 +- ...patch.current_1619_another-release-key.apk | Bin 0 -> 10541 bytes tests/run-tests | 22 ++++++++++++++++++ tests/stats/known_apks.txt | 1 + tests/update.TestCase | 2 +- 6 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 tests/repo/obb.mainpatch.current_1619_another-release-key.apk diff --git a/fdroidserver/index.py b/fdroidserver/index.py index f7162092..d3c1a0b7 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -361,9 +361,16 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): # Check for duplicates - they will make the client unhappy... for i in range(len(apklist) - 1): - if apklist[i]['versionCode'] == apklist[i + 1]['versionCode']: - raise FDroidException("duplicate versions: '%s' - '%s'" % ( - apklist[i]['apkName'], apklist[i + 1]['apkName'])) + first = apklist[i] + second = apklist[i + 1] + if first['versionCode'] == second['versionCode'] \ + and first['sig'] == second['sig']: + if first['hash'] == second['hash']: + raise FDroidException('"{0}/{1}" and "{0}/{2}" are exact duplicates!'.format( + repodir, first['apkName'], second['apkName'])) + else: + raise FDroidException('duplicates: "{0}/{1}" - "{0}/{2}"'.format( + repodir, first['apkName'], second['apkName'])) current_version_code = 0 current_version_file = None diff --git a/tests/repo/index.xml b/tests/repo/index.xml index 88507d52..ff958b7b 100644 --- a/tests/repo/index.xml +++ b/tests/repo/index.xml @@ -129,7 +129,7 @@ obb.mainpatch.current 2016-04-23 - 2016-04-23 + 2017-06-01 OBB Main/Patch Current obb.mainpatch.current.1619.png diff --git a/tests/repo/obb.mainpatch.current_1619_another-release-key.apk b/tests/repo/obb.mainpatch.current_1619_another-release-key.apk new file mode 100644 index 0000000000000000000000000000000000000000..1a494fe23bafc25c4915f95a9349575d4e4746d7 GIT binary patch literal 10541 zcmdsdWmsHIljz{??he7--3jgq?hZ4!1Pg)SM1kNQ2sS~3O&|~?!6CRy2noSm2G}#% z&G+Vg?|%2kz58oxo;uxKr%SrJp01wmqpyX6N(#cn!~}Iav}@9{7gmg+fezwZoSppq`GUPXZ~8%?>U155g$ZHo=G+qpjiL;Zp}2K1-G%N1dFW0z z7E-O-93$7`2KRC1Ld})ls8$`ifq?!#<2wmHXjrE)YhZ&BrD$Qqpxy`R~o@v!T3GA zuQQJzbS#0W*4twxLFMs5appaZ9_MXb^Ktu?%xg%!z9~rWo%@w-jWIZ5HBsfWNb3^Iw za4$iKivv2Q+Vo_yyrR?8B-7VrFM7lP+JE9PDC%=vXzX`E>Z39{Z)k+%xtkS4eU5q| z87#@ByVFR&lehoPg#U}Ee~!n2yR^a-OdjT2fiaNm?M4H$Sg|T3aMfD$xEyeyFp%~q z^Z6{?vojXZT{=Q~h1KeVL$+PWA}#=q5}6OefHLofPol9O@VAzRRcwhkAJ7bNRW>DI zga>^WtD7%+N+>=Z1kRp${xg12H6q)vu=tRw+cSN4<%{&wu&wYUN#O_Xy~h$RCjzn@ zp%1c%grisnKh3tY=qOzdqP|SM7-4;xakU)Jo_Txgu(c#f;+w-Nf-73cp&w7%7u7Y& zoEnv0AX6*(oMm#8uas9$3ce~sWM9D^h~@@<7c|EQZDSuPw=99 zJi;m>^yscqtB9ti;sb;42p;alKS&?-Vsc9I_jHiC6is%A|KJn#Unl7IV>)ADYO(H; zD6Msl+31*YQNG-{V(sZ0n{qfsv$Gk!bm+Z+9c5h?Jr?JNnWeb&UYueTqj$Xf3~JOt zd&A4tQqmw=#?z>Su`+}O(OuWE2u=XOLRN=7U@?zh;f`K4?2l8ABp;2esSZVy5166G zk;gPKirp;oBzhTGusGF(UFA4MZlY{Mq!ez2UTCRV2z&il?xMPC8hsr}GahFRV_W5Q zgQ?;`$_FC8!^8`nR(U;SgoihI&sJtneP_D{PFFHFdp4bTVU`3-&$@;9D-V3ht>r-c z9I%|;0gK?rAQ)`_kov9dY70^@+A5hyp2sQ~%D&}l3-w#q)imVc`k*o(*a@IP>4 zy)8NDCtuP8S(^nrr8L>ER->;}V?tEdtIdPcKroZlYIj)T#!`#(0!#Q+>jCcS{IXb{ zRoInu?@_kH%=PB`NF;qN49q{*K7C+S#0G&dK>#p;(qwxh$k0F_1kwT&Unf6)M_;=j zJ9|$jem4gjPrCrHgR7G-pEuYgL0?xL51Sfb2I1Y+P%#8bN(5q}0MF_2Az<}HZq(N{ zRt1`zOcW$!lzbYtDzf5Y64Fk-5i0UhsuElTxR{tY_(JOD91K)g0N~)FV*o2O1_2%p z4i5J19F&KRo}7e;o`H#mQ%r)7g#ZUrQ%g%xK_2LYPeQ>$ONNh&Lr+IXM9C<`K}$nT zg-1jttYXYaP0Yu~hf6@r$S2J}NyNa&M9nV9NJY%g&re24!zHGyEXsk4g^o=?Y89C; zC@4fpLoX zI8c({LA>07u>l5aNb`{p;1E(U&{L5R;9=|N>WPYokPs09J#nxw(b3U(cz6LD2r1|o zsL5!lDd=gbhzW4I&wg-p^>q;xELWdi@GEaF#!$+ z77in?6f+$a1sxj~BLyBl0i&S8T@^)H6-{MjWg=n{e0%~E6VvLqwV}ZQ1%<`6wKWeT zBg92T0Bc)e?SQX#wtt3(1ikO*+}qnbJ3oiR;Rgo?hX)7Yp&=ceAFtt8j~_iWx3Ki{ z^|7$B{t!;TF^yK(RTvAF@M99+C;qnTe^(s|MNALLL z^!VsdQbOYOo65#IsFjT!{QBnT@SuNizjl9J#tFw@yY3T^)*C<1oCon z4Na|`9q)nIaPSNs z)B64pHQubGWL~XB2MW~MhnSX|Wu!#bm`4F{d{rO+IL;@Jlf1 zTd-!+^}@7w@9OcQN--(DV#{rj+eFJb{L~ z!}Mq<`FyhPK_H|cV0w)2j5ydTfYjKX89#URqqELF@0OHkX^tW#;AiM~SU_lb|GtWU zY=*=$CMHQEO>;GJJ|+%%k9Uul<6oD^xIh!Rv@LbQm*8kPq9Tf()v9w+0zOvnd$iJd z%v!aVL7xOggv6o}viWT4>^CYVJ*KiUg$3^4H&=|Q_VS$OHS4fjFCDUY2d=Mo@%Hvd zPLyhEZf${JWYs%tLKn5tw-5G@?Lk_>SKEtCw;+A{&z;I~QD2E{ccC{P2(7%3$~8v)QRb?9#o3=iz;~2{8dbQ&D4!*BHC8px|W97 zr>rg!$?Q>K>o-3AwWaGhDRg73*vnW90tQ*Ir@g`}TUT(%;jx(qO{q$#y34&F0+doz z6lI9bEh?tR8Y?MQ>}R7=CCV?Q!U7-=vv*_4Ym+;`6o5c?RdrRMO16&>%jsX6%=vSp z7HlLCNE`XD<rB zP5>Ks1mc5G5Ge{0;OpSz$7kp3=KxFvG6?tAOTh0SBp{1@2TCBF{R@DZ`xnRnwO{~a z0n7rB7UTz#ZU+hgfq}X!K=lRjfxLm1%inkr zeL_GGpr1d$X9x842e<Sv6Sq}iJH_*Za83D5We{msX z`vEjukOn~W1bz?@BVZQ<=Nf}`CS1@TmTsXM8pQ_FMs(1UP8>oKjQ!d zLI))MHNS|qAK**hUo(n`1B9+-p!}^HsQ!|RC=oynpag*00D=L;0XYA=$O5)ifS!LB zI|$qCfN1ds;t34Yyns0cfe8M^0RgBsfMx%TBb;Ax`=4=S^G6&3A%si_A_QarZuSahJ!$&_`j)qsmY(;qw7^&bVA7hF zhN&udS)bN9sf$sWF{?@@-_c6roGpCuNo(na<&WmjxwhgvlQoW$PjY%SzB2ka&3NZX z@uh-vzrY_xe^5-Ga7(!{VDuX8RmhrgjbhhWfGeyt8(^PF%kMT9y=EkqyIO)>Tz?xH z+^t=y>W+iFPl>oOJ=tg)_VkIInr(-jjo(_ggzfBKcZB+E%7*_;&{S#g^>leL(eZo21`WWh0ai;ZWkZ%3b+)wVTQf@``ZypM-&^BYrp+g^1bPlZS zb!M-;S`lCI+7I356PSyqRuFbYp|d4iz;piKw(qmgbIx@xzAyI47QO9*hy(%j(ey=? zp~~V4pEqU>GDWA+%H9Y0hqs^(^y-X0wHg{)G@KRWX5@BUYZS;MSBx>|eMNoLd{i4$ z{-{XgTjU!$I3*kxh60TWimJtrKw;O25}^czb}9;>=tlXi5C@(qn8N-gdb$UoZ4 zmU0`}!Iqd74UO<2MyJDy&^b>tvK$8VQ3sk03V#%;5(v!E93$ZQD4~uKgK#T`SAj-B zyhefGI~&zexGl*H?z@K(HDsPB^PM0{<11Ez*iL+JoEa=IN>lV%l+_BpHwPtB@yfNW z_zVI5Lr!nP1(dAG2)HwMSkdvlf(*?_ya!3+ksvyYhazd;TA|0J%O6+NE3}0iHzO|i zXXeu&5o26CmbcYh`B-TqF>{|eLk)jK>IbyY3%(w{&c^()uv7}yabL1Xi)b33A2NB5 zb6GNerF0tgtX;U|Y3y-0lPu9VOQBecyvL>So56dww*}*6cDE1583Z<@yo?-lZQnU` zPZy@iDTn1cW(~xp%JAE}(TG;uH#a@-SLvC^=aj5=L=y<+VRn?3<(ir-cxkVb%}Mx^ z^QicakEKS6XBw32qKYz~=T!kEFBTP(jm9h(nI%l=6W;^PPk6`48G3PVrXN|cIAuA< zB!9tTBGFVUXLL;G!^$)oFX|vZvf=LQ4NtPWbpvBO$L<(5@cOl|=hvt9 zto2G_&p6gp()WyfU%tprTPSjj47UnN)(9TlHSoCz!G(wE<}!WTqW^Xl7`Y=*IC;cE z+`s%74yhbkteD%y-(ZM6*!qRP|MAxX0nyp=``$E?49iyhE)|d#X3Z zk}Y~u-)t0a2oL3-zr1qtx^eotoB-$WJ%m|j-_kD6S42iw3#EE|`h4`_=iSk?mq)|g zE!-RfyN}@t8uG;B^0ecdChuQ7)BgU}u_Rxs^f(Onbx8j5z$fXbKBdodN(B0EmmV87 zF;5Gx&s-$LyX=(sXnvoKSIY~?9+iEWuO#!9>Sx!6R|PIkS_@q@>;1P=nrxP(^@H`k z5+O2Ql=gVn7FHZ0SXlNHm+d9uWa_b4bQBI@sPjE*qXC1=g=tmfRa~cSffP)4Av(GN+49u zqm1y0XewVD``$VC%Ro7cz&Uq?wmeBj^Oop{LomNNRuhqh7m7!NxN(ysUhyuu&o|#? zT*r;sy-XHYZt%8{dHJ*kb2!I%f~#9|emfDb+1*wKhn>f^dgZx~54+E|sH~42+IV+}_&9OJ)5UOguJ=qjmf0MQoFc@Q>`v$XU-Azju`IVS$3r zZAp751UI7s&T2?iR_H9#%H5iR z>_>4Nvs8Vi@21BwRB{=)Smjil5ve^auQnUooN>#h*?kUou(ZFi%%guYUKcn=+trdE z|4y|9J+$FA3*P(VtAkT+cxz%=C8bAle)JsUoc6(q>E3-IwT_&$lTsbEvZ1PR8NX;L z2hqu4Ew1!7GwPix62VHGk(EJ{s{+R*rXpFTK2iF{#-nuxld4eXCRdYF?c))%kU$-W zxc(pox7+DfS?299y5P$=vx}vcKyewXr(b{6YFLff&r}BHF10D+4i-_Y->n<>rrELk zoR{JiojOt4$KtQ=VlE^p3&k0#m+DTwH`0AuwPPd9|IN7m_L%*&32c8mq_f@8mT*nr zu??RWg+mlxXKDr=25HTlR&$CsvWMSh`P=EVHn3l>WQiCpGd4zu9_&6R`PsYiAg#&j zGZI^9>O$#P+wT?+%g-oFcNN*~75hz1o^-?sp74!K1R)tMYioxSxUR`EvhAFR2-CEOr~>bzX>l z&SVbvbx&jt)+SPRPs}d#FcNM6_n%@%?U5e_mZvT!^$&nIwlM>ycKq7!`M@+I4C9I1 zD>s96*&;=})!!6@4+k#ZZ`3a>G+4=zxl4P}7aGsVnVw*)y2QK)aA;g3KQg-M!aUI> z+g_^+<${Fn6D+KTUSL6>-(Ly#928mcJspjuNzFW#;n2U375^mvooA>589%YAL-Bd& zBH9Sl7$un}GE=su6?&d?}0YCFgf`>R9(AqZWHb^Q$jbbh36ji1SZ^t z@bm{a5lwd%h;?mj-WBJGhq&514RiXGS-Ie!!Jy#w_O#Rn3NHwg{I@IBK5H*!N@X1m zwNn2bBFkuwlo8s zz=HDk@Q~a6lsWKizR^%fjl10ALR$6{AK*uQm(>vPMM(9k%BC7#OT5uq2{E*fAzn>U z_F7Gq5iC|OnJ7D3(us}tcCUh9x@9*rfH*EviAYg_t(T<1Up)j{8Q#g=!%j+kPnaXo;lTi8|G2WyxR_W4d?T zNhRxv1?i9Ef27R%m60PesB?)C>p*_LzG#z^EQ=MTB#S!7SAxfg4{{nB3{Llt3GzV6X(@_ z1IwyOGPR)CIaHMQW=|X@78;mn_)eO1lOP?+|g)0H>ci ze0vN7KOA~>B(}%AI?W_S^oC@7X0R+fyl420jMaEc)_Du=`MK?EQ@`5sOXACY)V3

D6 zID!}(z0JC~qu0{}_1t;xxtN%uqCB!VuZLt%bG0uZIU38-N{~`0<)}^Y9CJqMN+c08 zhvW-$qt7rJ&cs(}Cg{B?nly|e(0q%|SL-B{zGF|8#D{9yCqdHuY04DXBPJFW#SfDw zu5P5}CN~a#Z+K>Jp_41y)9UQMt$K&W*CCo9jYhFufYp@W;RD(O%>4XdI&H$ z(=?Nh(L4VLH>WkT8uNEC*CQx{=n$0PRVQTMcWIltlSXdVVKdQRFcluK6V-ZZYt_c{ zQ&py!UR2E^)FOZlMmxv)3JtEvCZe4n?it}9EF!ofNizoz| zJH|gJda8|4s}W=MZZL_0nLnroXGAq(rnA8&K&4TAyj*?8MQGdPC$P^6VZqs`rTrpe zHLykfMaXprNAfh|#2uP+eWz~*;wKdfONzXp%xrrW;v^ecxs0;I{j)_hYZm8Pw6IUe zVRZTaNXK0oA>7-(=Mm}?GZ;IEv0|+^^YH_7%Q%gr%eVpN*KJIgFt(N<)!_CD055H*lBvRf4t=@6~q^?(QW1lII31cg5il}%H= zd{;96GKYV&S>ZDg&g_CMA7(L1b#;@mQ@5Z7tytWo>}VbjwXa&2i-`dlfuYPOeZm#! zSV+g%>BO!v+ktrT5peFxv4DAa(%pZp9MtfmBV!%My~@ht>}K7YCmEG1{VvKgR>|j+ za%f7oNPRSM8^#gnZ=YC}jm^G)CcMs!-@e_!I3Urbgu$nua)#p_4-&mYr!dq*eUIFD!DBS67Q4ym)mF;n_d^_jk{_H>ur70$!a05OFN= zFV8Bg8XEH%sd)_#0bV8j?NuaHUSnd4g8QOx^wUVioHM8bQsrGhg?cs)a}6(c!C>EZ zzp1a!9u<|`KNfUi)xuMJ-YV0Smz5~NjxmVsgJf9OK^IHN^G$bJ#kwwN!Y8eBR;%IA z`di%1Cz`{Bczba7F83J-z#eL|O=CfPL4$!2QH2 zdHAWVsbPF3>YZp_rNs}hXtOeLoafaLxwq3Y&l=&R0hiS}@sGIU?RiL=lN2+%SV|4^ z=;v%_X!T^9e3`y5_Zj_^sQDBqB)f6SK-l}RcAh!rr)gflA2%22eJ>YJh)sI?9{W;1 z?pQ?V$ba6j{^{JmyM6d^shkmTq~i`84-)+!ZZ|Yi6n3@Oaaf;lKXSbq9{4oeP>>+H z0ANPK^RZ|14_~I3WIxTko0!)5kft!+N0G#@3BWc*5~CYrXLdDqcJj0~@($eKRd>pk zD09sZ>;iFhHCV!us#J%R-cb^?U4yz$3}PRl!x`z@1mY|r+Mhf!vM16;q9CY$TUk_? z97P)6$ID~+!rbyj<|`D7Xo)B`uMpdqgty(Twp3qZ{?z;wy{f1uJ ziyo_{=B&nUum)pik429y)!zlRR3sQ+Nd{AzQDtOPUQj(;v((l}PR6x>WOfLrRV7~1 zk9KInnO-JrKK`;x;XLU&L$`eaAFdlc|C+Zsj0+94T5j#>PmRWL)|Fg*aVC4{A3qzA z6p(&*K}YJ$o}(4&H&m6;@72hzAn6gXrtF;Cs(Q<-`}RWcqf3zHSd_(Bsqj-9)3q=# zX#l^(%e`T{Gs=`qJ-AZ`bj9a%&PRE5sqJG`eY4md4%Ow)jiz977!9dchKFeA5L)lq z7w9{(a96oz-*TT9j(f|Wrdl1#$sV+|qx)IQcZ_=d$fd3gz@EEng+cXg*x&m*BoP+m zXzuAgjuls~*C@;JKC^$e@m4=#vrsvT+g>Z?#uXxT6quj8NY*4R*f7CYITVrPK4VIx z&hD&+-rJLieu!T8fiQK$fyjaAB2+ebDO-#9ZGIk>nNfh;-iINli!E2TWGd3Ygj>1W z22BEKj`t3II~4iFp@)3SwU-U;;PLLTybdm#2jC^2uNMc)L3!cKksni+Bp4P0Qp_=y zLhs3kZhiG4@__mD*evJzo_&j~^V#JJb0}L`Ru#$ z!8TTB>h`QYdjGfHor{OEr{YgF!(=%m_vSh?E(T+Bkx6>MpYOfRpm|oO>A0hKl0uC^ z>bzKOtS1p3I$MiA|DtRmboj^X&A|cbGdcO2MKg%i(Xc?}%yD@CY-q~Myk;$ri22)t z)oF99yzrbk-&mdGq$n&mIe`)C30&>VSaOiY2AaFQi<`aU%FMg-q3w$#-wv5WuW~=D z(`(`AX;|6qBu8E3wfua$j5g1@SIf#NXVo>I{kOw!U4)1_57xAkM1)n@>mnl0XhDb^o^yN&achpLc(V zgZaO}2~dZS^zZIa|7p|T?+X8t0|K=G=jMOEIsB&;f9^>ThwOg=3`m!M9J&9K@z0ct z*s}ZuWek6$**`Wg|CI1&_CTcOzW|Bp4+;O3sQ=0R=S(9q%wG`9{7-Hm-{@> config.py +echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py +mkdir $REPOROOT/metadata +cp -a $WORKSPACE/tests/metadata/obb.mainpatch.current.txt $REPOROOT/metadata +echo "accepted_formats = ['txt']" >> config.py +cp $WORKSPACE/tests/repo/obb.mainpatch.current_1619.apk $REPOROOT/repo/ +cp $WORKSPACE/tests/repo/obb.mainpatch.current_1619_another-release-key.apk $REPOROOT/repo/ +$fdroid update --pretty +grep -F 'obb.mainpatch.current_1619.apk' repo/index.xml +grep -F 'obb.mainpatch.current_1619_another-release-key.apk' repo/index.xml +# die if there are exact duplicates +cp $WORKSPACE/tests/repo/obb.mainpatch.current_1619.apk $REPOROOT/repo/duplicate.apk +! $fdroid update + + #------------------------------------------------------------------------------# echo_header "setup new repo from scratch using ANDROID_HOME, putting APKs in repo first" diff --git a/tests/stats/known_apks.txt b/tests/stats/known_apks.txt index 94a40a74..ec777242 100644 --- a/tests/stats/known_apks.txt +++ b/tests/stats/known_apks.txt @@ -4,4 +4,5 @@ obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12 obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01 obb.main.twoversions_1101617.apk obb.main.twoversions 2016-06-20 obb.mainpatch.current_1619.apk obb.mainpatch.current 2016-04-23 +obb.mainpatch.current_1619_another-release-key.apk obb.mainpatch.current 2017-06-01 urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk info.guardianproject.urzip 2016-06-23 diff --git a/tests/update.TestCase b/tests/update.TestCase index be1a7266..3742f965 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -204,7 +204,7 @@ class UpdateTest(unittest.TestCase): apps = fdroidserver.metadata.read_metadata(xref=True) knownapks = fdroidserver.common.KnownApks() apks, cachechanged = fdroidserver.update.scan_apks({}, 'repo', knownapks, False) - self.assertEqual(len(apks), 6) + self.assertEqual(len(apks), 7) apk = apks[0] self.assertEqual(apk['minSdkVersion'], '4') self.assertEqual(apk['targetSdkVersion'], '18') From 4053f03d77e99b24bccf6f43ce63287cef410e37 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 31 May 2017 21:20:35 +0200 Subject: [PATCH 4/9] update: add --rename-apks to force APK filenames to fdroid standard uses the standard package.name_123.apk. If that exists, it appends the shasum. If that exists, then its a duplicate, so its deleted. This should help @SergeWinters with his 12,000 APKs. --- completion/bash-completion | 2 +- fdroidserver/update.py | 49 ++++++++++++++++++++++++++++++++------ tests/run-tests | 29 ++++++++++++++++++++++ tests/update.TestCase | 1 + 4 files changed, 73 insertions(+), 8 deletions(-) diff --git a/completion/bash-completion b/completion/bash-completion index 48352447..3965791d 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -125,7 +125,7 @@ __complete_update() { opts="-c -v -q -b -i -I -e -w" lopts="--create-metadata --verbose --quiet --buildreport --interactive --icons --editor --wiki --pretty --clean --delete-unknown - --nosign --use-date-from-apk" + --nosign --rename-apks --use-date-from-apk" case "${prev}" in -e|--editor) _filedir diff --git a/fdroidserver/update.py b/fdroidserver/update.py index b0ff8d0a..caea795b 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1089,8 +1089,14 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): """ if ' ' in apkfilename: - logging.critical("Spaces in filenames are not allowed.") - return True, None, False + if options.rename_apks: + newfilename = apkfilename.replace(' ', '_') + os.rename(os.path.join(repodir, apkfilename), + os.path.join(repodir, newfilename)) + apkfilename = newfilename + else: + logging.critical("Spaces in filenames are not allowed.") + return True, None, False apkfile = os.path.join(repodir, apkfilename) shasum = sha256sum(apkfile) @@ -1108,13 +1114,8 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): if not usecache: logging.debug("Processing " + apkfilename) apk = {} - apk['apkName'] = apkfilename apk['hash'] = shasum apk['hashType'] = 'sha256' - srcfilename = apkfilename[:-4] + "_src.tar.gz" - if os.path.exists(os.path.join(repodir, srcfilename)): - apk['srcname'] = srcfilename - apk['size'] = os.path.getsize(apkfile) apk['uses-permission'] = [] apk['uses-permission-sdk-23'] = [] apk['features'] = [] @@ -1147,6 +1148,35 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): logging.critical("Failed to get apk signature") return True, None, False + if options.rename_apks: + n = apk['packageName'] + '_' + str(apk['versionCode']) + '.apk' + std_short_name = os.path.join(repodir, n) + if apkfile != std_short_name: + if os.path.exists(std_short_name): + std_long_name = std_short_name.replace('.apk', '_' + apk['sig'][:7] + '.apk') + if apkfile != std_long_name: + if os.path.exists(std_long_name): + dupdir = os.path.join('duplicates', repodir) + if not os.path.isdir(dupdir): + os.makedirs(dupdir, exist_ok=True) + dupfile = os.path.join('duplicates', std_long_name) + logging.warning('Moving duplicate ' + std_long_name + ' to ' + dupfile) + os.rename(apkfile, dupfile) + return True, None, False + else: + os.rename(apkfile, std_long_name) + apkfile = std_long_name + else: + os.rename(apkfile, std_short_name) + apkfile = std_short_name + apkfilename = apkfile[len(repodir) + 1:] + + apk['apkName'] = apkfilename + srcfilename = apkfilename[:-4] + "_src.tar.gz" + if os.path.exists(os.path.join(repodir, srcfilename)): + apk['srcname'] = srcfilename + apk['size'] = os.path.getsize(apkfile) + apkzip = zipfile.ZipFile(apkfile, 'r') # if an APK has files newer than the system time, suggest updating @@ -1498,6 +1528,8 @@ def main(): help="When configured for signed indexes, create only unsigned indexes at this stage") parser.add_argument("--use-date-from-apk", action="store_true", default=False, help="Use date from apk instead of current time for newly added apks") + parser.add_argument("--rename-apks", action="store_true", default=False, + help="Rename APK files that do not match package.name_123.apk") metadata.add_metadata_arguments(parser) options = parser.parse_args() metadata.warnings_action = options.W @@ -1517,6 +1549,9 @@ def main(): resize_all_icons(repodirs) sys.exit(0) + if options.rename_apks: + options.clean = True + # check that icons exist now, rather than fail at the end of `fdroid update` for k in ['repo_icon', 'archive_icon']: if k in config: diff --git a/tests/run-tests b/tests/run-tests index 84f7f766..484ed2a8 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -239,6 +239,35 @@ test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc sed -i --expression='s,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xml diff $WORKSPACE/tests/repo/index.xml repo/index.xml + +#------------------------------------------------------------------------------# +echo_header 'rename apks with `fdroid update --rename-apks`' + +REPOROOT=`create_test_dir` +cd $REPOROOT +cp $WORKSPACE/tests/keystore.jks $REPOROOT/ +$fdroid init --keystore keystore.jks --repo-keyalias=sova +echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py +echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py +echo "accepted_formats = ['txt', 'yml']" >> config.py +echo 'keydname = "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"' >> config.py +test -d metadata || mkdir metadata +cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/ +test -d repo || mkdir repo +cp $WORKSPACE/tests/urzip.apk "repo/asdfiuhk urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234 ö.apk" +$fdroid update --rename-apks +test -e repo/info.guardianproject.urzip_100.apk +cp $WORKSPACE/tests/urzip-release.apk repo/ +$fdroid update --rename-apks +test -e repo/info.guardianproject.urzip_100.apk +test -e repo/info.guardianproject.urzip_100_b4964fd.apk +cp $WORKSPACE/tests/urzip-release.apk repo/ +$fdroid update --rename-apks +test -e repo/info.guardianproject.urzip_100.apk +test -e repo/info.guardianproject.urzip_100_b4964fd.apk +test -e duplicates/repo/info.guardianproject.urzip_100_b4964fd.apk + + #------------------------------------------------------------------------------# echo_header "test metadata checks" diff --git a/tests/update.TestCase b/tests/update.TestCase index 3742f965..b5020877 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -200,6 +200,7 @@ class UpdateTest(unittest.TestCase): fdroidserver.update.options = type('', (), {})() fdroidserver.update.options.clean = True fdroidserver.update.options.delete_unknown = True + fdroidserver.update.options.rename_apks = False apps = fdroidserver.metadata.read_metadata(xref=True) knownapks = fdroidserver.common.KnownApks() From ceac6d25cba7991a717cb7f1f287b46f369bd331 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 31 May 2017 21:43:40 +0200 Subject: [PATCH 5/9] index.xml cannot handle APKs with the same packageName/versionCode Really, it is the fdroidclient parser of index.xml that fails, due to the hardcoded expectation that there will only ever be a single APK for any given versionCode. We keep index.xml backwards compatible for old clients, and use index-v1.json to support new things. Having multiple APKs that have the same packageName and versionCode will break the client v0.103.* since that version uses index-v1.json, but still has the hard- coded database parsing stuff. #153 --- fdroidserver/index.py | 5 ++++- tests/run-tests | 20 ++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index d3c1a0b7..32727115 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -294,9 +294,12 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): # Get a list of the apks for this app... apklist = [] + versionCodes = [] for apk in apks: if apk['packageName'] == appid: - apklist.append(apk) + if apk['versionCode'] not in versionCodes: + apklist.append(apk) + versionCodes.append(apk['versionCode']) if len(apklist) == 0: continue diff --git a/tests/run-tests b/tests/run-tests index 484ed2a8..a3d62d27 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -241,7 +241,7 @@ diff $WORKSPACE/tests/repo/index.xml repo/index.xml #------------------------------------------------------------------------------# -echo_header 'rename apks with `fdroid update --rename-apks`' +echo_header 'rename apks with `fdroid update --rename-apks`, --nosign for speed' REPOROOT=`create_test_dir` cd $REPOROOT @@ -255,17 +255,24 @@ test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/ test -d repo || mkdir repo cp $WORKSPACE/tests/urzip.apk "repo/asdfiuhk urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234 ö.apk" -$fdroid update --rename-apks +$fdroid update --rename-apks --pretty --nosign test -e repo/info.guardianproject.urzip_100.apk +grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml cp $WORKSPACE/tests/urzip-release.apk repo/ -$fdroid update --rename-apks +$fdroid update --rename-apks --pretty --nosign test -e repo/info.guardianproject.urzip_100.apk test -e repo/info.guardianproject.urzip_100_b4964fd.apk +grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml +grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index-v1.json +! grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index.xml cp $WORKSPACE/tests/urzip-release.apk repo/ -$fdroid update --rename-apks +$fdroid update --rename-apks --pretty --nosign test -e repo/info.guardianproject.urzip_100.apk test -e repo/info.guardianproject.urzip_100_b4964fd.apk test -e duplicates/repo/info.guardianproject.urzip_100_b4964fd.apk +grep -F 'info.guardianproject.urzip_100.apk' repo/index-v1.json repo/index.xml +grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index-v1.json +! grep -F 'info.guardianproject.urzip_100_b4964fd.apk' repo/index.xml #------------------------------------------------------------------------------# @@ -528,8 +535,9 @@ echo "accepted_formats = ['txt']" >> config.py cp $WORKSPACE/tests/repo/obb.mainpatch.current_1619.apk $REPOROOT/repo/ cp $WORKSPACE/tests/repo/obb.mainpatch.current_1619_another-release-key.apk $REPOROOT/repo/ $fdroid update --pretty -grep -F 'obb.mainpatch.current_1619.apk' repo/index.xml -grep -F 'obb.mainpatch.current_1619_another-release-key.apk' repo/index.xml +grep -F 'obb.mainpatch.current_1619.apk' repo/index.xml repo/index-v1.json +grep -F 'obb.mainpatch.current_1619_another-release-key.apk' repo/index-v1.json +! grep -F 'obb.mainpatch.current_1619_another-release-key.apk' repo/index.xml # die if there are exact duplicates cp $WORKSPACE/tests/repo/obb.mainpatch.current_1619.apk $REPOROOT/repo/duplicate.apk ! $fdroid update From 9471bf273117694bf671ae627d0e7dcc956562d3 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 31 May 2017 23:02:28 +0200 Subject: [PATCH 6/9] regexs for getting packageName and versionCode from filenames This is useful for parsing APK files, which can include packageName, versionCode, and optionally 7 char signing key ID (i.e. ). This also can set the packageName and versionCoe for non APK files, so that it is easy to assign them to metadata files, and to allow for upgrades by setting the versionCode in the filename. --- fdroidserver/common.py | 2 ++ fdroidserver/update.py | 12 +++---- tests/common.TestCase | 64 ++++++++++++++++++++++++++++++++++++++ tests/stats/known_apks.txt | 2 +- 4 files changed, 71 insertions(+), 9 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index ca02219f..75eac97c 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -55,6 +55,8 @@ from .asynchronousfilereader import AsynchronousFileReader # A signature block file with a .DSA, .RSA, or .EC extension CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$') +APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk') +STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+') XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android') diff --git a/fdroidserver/update.py b/fdroidserver/update.py index caea795b..c36fd86e 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -853,14 +853,10 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): # the static ID is the SHA256 unless it is set in the metadata repo_file['packageName'] = shasum - n = name_utf8.rsplit('_', maxsplit=1) - if len(n) == 2: - packageName = n[0] - versionCode = n[1].split('.')[0] - if re.match('^-?[0-9]+$', versionCode) \ - and common.is_valid_package_name(n[0]): - repo_file['packageName'] = packageName - repo_file['versionCode'] = int(versionCode) + m = common.STANDARD_FILE_NAME_REGEX.match(name_utf8) + if m: + repo_file['packageName'] = m.group(1) + repo_file['versionCode'] = int(m.group(2)) srcfilename = name + b'_src.tar.gz' if os.path.exists(os.path.join(repodir, srcfilename)): repo_file['srcname'] = srcfilename.decode('utf-8') diff --git a/tests/common.TestCase b/tests/common.TestCase index 74364098..f0dcacce 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -281,6 +281,70 @@ class CommonTest(unittest.TestCase): key = "val" """)) + def test_apk_name_regex(self): + good = [ + 'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_-123456.apk', + 'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_123456_abcdef0.apk', + 'urzip_-123456.apk', + 'a0_0.apk', + 'Z0_0.apk', + 'a0_0_abcdef0.apk', + 'a_a_a_a_0_abcdef0.apk', + 'a_____0.apk', + 'a_____123456_abcdef0.apk', + 'org.fdroid.fdroid_123456.apk', + # valid, but "_99999" is part of packageName rather than versionCode + 'org.fdroid.fdroid_99999_123456.apk', + # should be valid, but I can't figure out the regex since \w includes digits + # 'πÇÇπÇÇ现代汉语通用字българскиعربي1234ö_0_123bafd.apk', + ] + for name in good: + m = fdroidserver.common.APK_NAME_REGEX.match(name) + self.assertIsNotNone(m) + self.assertIn(m.group(2), ('-123456', '0', '123456')) + self.assertIn(m.group(3), ('abcdef0', None)) + + bad = [ + 'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_123456_abcdefg.apk', + 'urzip-_-198274.apk', + 'urzip-_0_123bafd.apk', + 'no spaces allowed_123.apk', + '0_0.apk', + '0_0_abcdef0.apk', + ] + for name in bad: + self.assertIsNone(fdroidserver.common.APK_NAME_REGEX.match(name)) + + def test_standard_file_name_regex(self): + good = [ + 'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_-123456.mp3', + 'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_123456.mov', + 'Document_-123456.pdf', + 'WTF_0.MOV', + 'Z0_0.ebk', + 'a_a_a_a_0.txt', + 'org.fdroid.fdroid.privileged.ota_123456.zip', + 'πÇÇπÇÇ现代汉语通用字българскиعربي1234ö_0.jpeg', + 'a_____0.PNG', + # valid, but "_99999" is part of packageName rather than versionCode + 'a_____99999_123456.zip', + 'org.fdroid.fdroid_99999_123456.zip', + ] + for name in good: + m = fdroidserver.common.STANDARD_FILE_NAME_REGEX.match(name) + self.assertIsNotNone(m) + self.assertIn(m.group(2), ('-123456', '0', '123456')) + + bad = [ + 'urzipπÇÇπÇÇ现代汉语通用字българскиعربي1234ö_abcdefg.JPEG', + 'urzip-_-198274.zip', + 'urzip-_123bafd.pdf', + 'no spaces allowed_123.foobar', + 'a_____0.', + ] + for name in bad: + self.assertIsNone(fdroidserver.common.STANDARD_FILE_NAME_REGEX.match(name)) + if __name__ == "__main__": parser = optparse.OptionParser() diff --git a/tests/stats/known_apks.txt b/tests/stats/known_apks.txt index ec777242..329213b7 100644 --- a/tests/stats/known_apks.txt +++ b/tests/stats/known_apks.txt @@ -1,4 +1,4 @@ -fake.ota.update_1234.zip 897a92a4ccff4f415f6ba275b2af16d4ecaee60a983b215bddcb9f8964e7a24c 2016-03-10 +fake.ota.update_1234.zip fake.ota.update 2016-03-10 obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31 obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12 obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01 From 6105f8a184228889fffdfc63ead11a264a6043ea Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 1 Jun 2017 10:27:35 +0200 Subject: [PATCH 7/9] use var naming scheme in KnownApks (apk --> apkName) Everywhere else, the file name of the APK is called apkName. --- fdroidserver/common.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 75eac97c..acca01dd 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1582,6 +1582,11 @@ def natural_key(s): class KnownApks: + """permanent store of existing APKs with the date they were added + + This is currently the only way to permanently store the "updated" + date of APKs. + """ def __init__(self): self.path = os.path.join('stats', 'known_apks.txt') @@ -1615,17 +1620,17 @@ class KnownApks: for line in sorted(lst, key=natural_key): f.write(line + '\n') - def recordapk(self, apk, app, default_date=None): + def recordapk(self, apkName, app, default_date=None): ''' Record an apk (if it's new, otherwise does nothing) Returns the date it was added as a datetime instance ''' - if apk not in self.apks: + if apkName not in self.apks: if default_date is None: default_date = datetime.utcnow() - self.apks[apk] = (app, default_date) + self.apks[apkName] = (app, default_date) self.changed = True - _, added = self.apks[apk] + _, added = self.apks[apkName] return added # Look up information - given the 'apkname', returns (app id, date added/None). From 372c8b418db7e3c661debbd24df2f405baa95569 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 1 Jun 2017 10:29:30 +0200 Subject: [PATCH 8/9] strip file extension from generated name for non-APKs With a generic file, the file name is the only guaranteed name metadata field. So if the name is not specified in the metadata, then the name is set to the filename. This changes that so that the file extension is stripped from that generated name. --- fdroidserver/index.py | 2 +- fdroidserver/update.py | 2 +- tests/repo/index.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 32727115..942c5861 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -448,7 +448,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict): and repodir == 'repo': # only create these namefield = common.config['current_version_name_source'] sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8')) - apklinkname = sanitized_name + b'.apk' + apklinkname = sanitized_name + os.path.splitext(current_version_file)[1].encode('utf-8') current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape') if os.path.islink(apklinkname): os.remove(apklinkname) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index c36fd86e..614ae07f 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -843,8 +843,8 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): if not usecache: logging.debug("Processing " + name_utf8) repo_file = collections.OrderedDict() + repo_file['name'] = os.path.splitext(name_utf8)[0] # TODO rename apkname globally to something more generic - repo_file['name'] = name_utf8 repo_file['apkName'] = name_utf8 repo_file['hash'] = shasum repo_file['hashType'] = 'sha256' diff --git a/tests/repo/index.xml b/tests/repo/index.xml index ff958b7b..4b85a960 100644 --- a/tests/repo/index.xml +++ b/tests/repo/index.xml @@ -12,7 +12,7 @@ fake.ota.update 2016-03-10 2016-03-10 - fake.ota.update_1234.zip + fake.ota.update_1234

Tests whether OTA ZIP files are being include <p>F-Droid can make use of system privileges or permissions to install, update and remove applications on its own. The only way to obtain those privileges is to become a system app.</p><p>This is where the Privileged Extension comes in - being a separate app and much smaller, it can be installed as a system app and communicate with the main app via AIDL IPC.</p><p>This has several advantages:</p><ul><li> Reduced disk usage in the system partition</li><li> System updates don't remove F-Droid</li><li> The process of installing into system via root is safer</li></ul><p>This is packaged as an OTA (Over-The-Air) update ZIP file. It must be installed using TWRP or other Android recovery that can flash updates to the system from the /data/data/org.fdroid.fdroid folder on the /data partition. The standalone APK is called F-Droid Privileged Extension.</p> Apache-2.0 From 8776221988803ac27a42febe475503546a04752a Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 1 Jun 2017 16:24:31 +0200 Subject: [PATCH 9/9] check signature and OpenSSL after APK has proven valid If working with a random grabbag of APKs, there can be all sorts of issues like corrupt entries in the ZIP, bad signatures, signatures that are invalid since they use MD5, etc. Moving these two checks later means that the APKs can be renamed still. This does change how common.getsig() works. For years, it returned None if the signature check failed. Now that I've started working with giant APK collections gathered from the wild, I can see that `fdroid update` needs to be able to first index what's there, then make decisions based on that information. So that means separating the getsig() fingerprint fetching from the APK signature verification. This is not hugely security sensitive, since the APKs still have to get past the Android checks, e.g. update signature checks. Plus the APK hash is already included in the signed index. --- fdroidserver/update.py | 13 +++++++------ tests/update.TestCase | 16 ++++++++-------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 614ae07f..faec9144 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -402,10 +402,6 @@ def getsig(apkpath): if an error occurred. """ - # verify the jar signature is correct - if not common.verify_apk_signature(apkpath): - return None - with zipfile.ZipFile(apkpath, 'r') as apk: certs = [n for n in apk.namelist() if common.CERT_PATH_REGEX.match(n)] @@ -1118,8 +1114,6 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): apk['icons_src'] = {} apk['icons'] = {} apk['antiFeatures'] = set() - if has_old_openssl(apkfile): - apk['antiFeatures'].add('KnownVuln') try: if SdkToolsPopen(['aapt', 'version'], output=False): @@ -1173,6 +1167,13 @@ def scan_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk): apk['srcname'] = srcfilename apk['size'] = os.path.getsize(apkfile) + # verify the jar signature is correct + if not common.verify_apk_signature(apkfile): + return True, None, False + + if has_old_openssl(apkfile): + apk['antiFeatures'].add('KnownVuln') + apkzip = zipfile.ZipFile(apkfile, 'r') # if an APK has files newer than the system time, suggest updating diff --git a/tests/update.TestCase b/tests/update.TestCase index b5020877..7e1626c5 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -169,21 +169,21 @@ class UpdateTest(unittest.TestCase): self.assertTrue(False, 'TypeError!') def testBadGetsig(self): + """getsig() should still be able to fetch the fingerprint of bad signatures""" # config needed to use jarsigner and keytool config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.update.config = config + apkfile = os.path.join(os.path.dirname(__file__), 'urzip-badsig.apk') - sig = self.javagetsig(apkfile) - self.assertIsNone(sig, "sig should be None: " + str(sig)) - pysig = fdroidserver.update.getsig(apkfile) - self.assertIsNone(pysig, "python sig should be None: " + str(sig)) + sig = fdroidserver.update.getsig(apkfile) + self.assertEqual(sig, 'e0ecb5fc2d63088e4a07ae410a127722', + "python sig should be: " + str(sig)) apkfile = os.path.join(os.path.dirname(__file__), 'urzip-badcert.apk') - sig = self.javagetsig(apkfile) - self.assertIsNone(sig, "sig should be None: " + str(sig)) - pysig = fdroidserver.update.getsig(apkfile) - self.assertIsNone(pysig, "python sig should be None: " + str(sig)) + sig = fdroidserver.update.getsig(apkfile) + self.assertEqual(sig, 'e0ecb5fc2d63088e4a07ae410a127722', + "python sig should be: " + str(sig)) def testScanApksAndObbs(self): os.chdir(os.path.join(localmodule, 'tests'))