From 1d4e3a254daed9f975ec94f6b140250ae7f50dbe Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sun, 17 Sep 2017 21:54:21 +0200 Subject: [PATCH 01/14] create blank config.py using cross-platform technique os.mknod() fails on OSX with: Traceback (most recent call last): File "/Users/travis/build/fdroidtravis/fdroidserver/tests/../fdroid", line 154, in main() File "/Users/travis/build/fdroidtravis/fdroidserver/tests/../fdroid", line 130, in main mod.main() File "/Users/travis/build/fdroidtravis/fdroidserver/fdroidserver/update.py", line 1768, in main common.write_to_config(config, 'repo_keyalias', config['repo_keyalias']) File "/Users/travis/build/fdroidtravis/fdroidserver/fdroidserver/common.py", line 2328, in write_to_config os.mknod(cfg) PermissionError: [Errno 1] Operation not permitted --- fdroidserver/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 98aab209..4d897653 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -2325,7 +2325,7 @@ def write_to_config(thisconfig, key, value=None, config_file=None): # load config file, create one if it doesn't exist if not os.path.exists(cfg): - os.mknod(cfg) + open(cfg, 'a').close() logging.info("Creating empty " + cfg) with open(cfg, 'r', encoding="utf-8") as f: lines = f.readlines() From 535f2afe907219143489d411836e3987c453f27d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 19 Sep 2017 10:55:16 +0200 Subject: [PATCH 02/14] init: prompt user for Android SDK path using platform-specific default --- fdroidserver/init.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 0067ffcd..3aa0b61a 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -100,11 +100,21 @@ def main(): # make sure at least aapt is found, since this can't do anything without it test_config['aapt'] = common.find_sdk_tools_cmd('aapt') else: - # if neither --android-home nor the default sdk_path exist, prompt the user + # if neither --android-home nor the default sdk_path + # exist, prompt the user using platform-specific default default_sdk_path = '/opt/android-sdk' if sys.platform == 'win32' or sys.platform == 'cygwin': - default_sdk_path = os.path.join(os.getenv('USERPROFILE'), - 'AppData', 'Local', 'Android', 'android-sdk') + p = os.path.join(os.getenv('USERPROFILE'), + 'AppData', 'Local', 'Android', 'android-sdk') + elif sys.platform == 'darwin': + # on OSX, Homebrew is common and has an easy path to detect + p = '/usr/local/opt/android-sdk' + else: + # if the Debian packages are installed, suggest them + p = '/usr/lib/android-sdk' + if os.path.exists(p): + default_sdk_path = p + while not options.no_prompt: try: s = input(_('Enter the path to the Android SDK (%s) here:\n> ') % default_sdk_path) From 5dcb48831feee272a9da4a564c52383d3c952d04 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 19 Sep 2017 15:07:19 +0200 Subject: [PATCH 03/14] index: always use jarsigner for verifying JAR signatures apksigner v0.7 (build-tools 26.0.1), Google made it require that the AndroidManifest.xml was present in the archive before it verifies the signature. So this needs to stick with the jarsigner hack for JARs. --- fdroidserver/common.py | 37 +++++++++++++++++++++++++++---------- fdroidserver/index.py | 12 +----------- tests/index.TestCase | 4 ++-- tests/run-tests | 1 + 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 4d897653..807974d6 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -50,7 +50,7 @@ from distutils.util import strtobool import fdroidserver.metadata from fdroidserver import _ -from fdroidserver.exception import FDroidException, VCSException, BuildException +from fdroidserver.exception import FDroidException, VCSException, BuildException, VerificationException from .asynchronousfilereader import AsynchronousFileReader @@ -2077,24 +2077,41 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir): return None -def verify_apk_signature(apk, jar=False): +def verify_jar_signature(jar): + """Verifies the signature of a given JAR file. + + jarsigner is very shitty: unsigned JARs pass as "verified"! So + this has to turn on -strict then check for result 4, since this + does not expect the signature to be from a CA-signed certificate. + + :raises: VerificationException() if the JAR's signature could not be verified + + """ + + if subprocess.call([config['jarsigner'], '-strict', '-verify', jar]) != 4: + raise VerificationException(_("The repository's index could not be verified.")) + + +def verify_apk_signature(apk, min_sdk_version=None): """verify the signature on an APK Try to use apksigner whenever possible since jarsigner is very - shitty: unsigned APKs pass as "verified"! So this has to turn on - -strict then check for result 4. - - You can set :param: jar to True if you want to use this method - to verify jar signatures. + shitty: unsigned APKs pass as "verified"! Warning, this does + not work on JARs with apksigner >= 0.7 (build-tools 26.0.1) """ if set_command_in_config('apksigner'): args = [config['apksigner'], 'verify'] - if jar: - args += ['--min-sdk-version=1'] + if min_sdk_version: + args += ['--min-sdk-version=' + min_sdk_version] return subprocess.call(args + [apk]) == 0 else: logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner") - return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4 + try: + verify_jar_signature(apk) + return True + except: + pass + return False def verify_old_apk_signature(apk): diff --git a/fdroidserver/index.py b/fdroidserver/index.py index 59139e17..79322d55 100644 --- a/fdroidserver/index.py +++ b/fdroidserver/index.py @@ -632,7 +632,7 @@ def download_repo_index(url_str, etag=None, verify_fingerprint=True): jar = zipfile.ZipFile(fp) # verify that the JAR signature is valid - verify_jar_signature(fp.name) + common.verify_jar_signature(fp.name) # get public key and its fingerprint from JAR public_key, public_key_fingerprint = get_public_key_from_jar(jar) @@ -652,16 +652,6 @@ def download_repo_index(url_str, etag=None, verify_fingerprint=True): return index, new_etag -def verify_jar_signature(file): - """ - Verifies the signature of a given JAR file. - - :raises: VerificationException() if the JAR's signature could not be verified - """ - if not common.verify_apk_signature(file, jar=True): - raise VerificationException(_("The repository's index could not be verified.")) - - def get_public_key_from_jar(jar): """ Get the public key and its fingerprint from a JAR file. diff --git a/tests/index.TestCase b/tests/index.TestCase index 2798e781..31593523 100755 --- a/tests/index.TestCase +++ b/tests/index.TestCase @@ -39,14 +39,14 @@ class IndexTest(unittest.TestCase): source_dir = os.path.join(basedir, 'signindex') for f in ('testy.jar', 'guardianproject.jar'): testfile = os.path.join(source_dir, f) - fdroidserver.index.verify_jar_signature(testfile) + fdroidserver.common.verify_jar_signature(testfile) def test_verify_jar_signature_fails(self): basedir = os.path.dirname(__file__) source_dir = os.path.join(basedir, 'signindex') testfile = os.path.join(source_dir, 'unsigned.jar') with self.assertRaises(fdroidserver.index.VerificationException): - fdroidserver.index.verify_jar_signature(testfile) + fdroidserver.common.verify_jar_signature(testfile) def test_get_public_key_from_jar_succeeds(self): basedir = os.path.dirname(__file__) diff --git a/tests/run-tests b/tests/run-tests index 608c1e2e..c58942bd 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -266,6 +266,7 @@ cp $WORKSPACE/tests/urzip.apk \ sed -i 's,archive_older = [0-9],archive_older = 3,' config.py $fdroid update --pretty --nosign +echo "This will fail when jarsigner allows MD5 for APK signatures" test `grep '' archive/index.xml | wc -l` -eq 5 test `grep '' repo/index.xml | wc -l` -eq 7 From 6adf309bef5ee7dbc6785d44d30f28c41ba2866b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 19 Sep 2017 16:48:20 +0200 Subject: [PATCH 04/14] tests: move test APK with bad unicode filename to separate test This filename has some messed up bytes related to bi-directional script that is included (Left-to-Right and Right-to-Left). GNU/Linux always interprets filenames as pure byte sequences. Windows and OSX store filenames as Unicode strings. So on OSX, the invalid filename gets converted to a valid name. That works fine, but the test fails because it is compared to a file generated on Ubuntu, where it preserves the byte sequence. This includes an APK with a valid Unicode filename that includes bi-directional script. --- ...用字-български-عربي1234.apk} | Bin tests/repo/index.xml | 2 +- ...西里耶维奇·拉赫玛.apk尼诺夫 .apk | Bin 0 -> 11471 bytes tests/run-tests | 4 ++-- tests/stats/known_apks.txt | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename tests/{repo/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk => bad-unicode-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk} (100%) create mode 100644 tests/repo/urzip-Sergey Vasilyevich Rakhmaninov; Серге́й Васи́льевич Рахма́нинов, IPA: [sʲɪrˈɡʲej rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢尔盖·瓦西里耶维奇·拉赫玛.apk尼诺夫 .apk diff --git a/tests/repo/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk b/tests/bad-unicode-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk similarity index 100% rename from tests/repo/urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk rename to tests/bad-unicode-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk diff --git a/tests/repo/index.xml b/tests/repo/index.xml index 9836fc3e..a2eb17e3 100644 --- a/tests/repo/index.xml +++ b/tests/repo/index.xml @@ -244,7 +244,7 @@ 0.1 100 - urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk + urzip-Sergey Vasilyevich Rakhmaninov; Серге́й Васи́льевич Рахма́нинов, IPA: [sʲɪrˈɡʲej rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢尔盖·瓦西里耶维奇·拉赫玛.apk尼诺夫 .apk 15c0ec72c74a3791f42cdb43c57df0fb11a4dbb656851bbb8cf05b26a8372789 11471 4 diff --git a/tests/repo/urzip-Sergey Vasilyevich Rakhmaninov; Серге́й Васи́льевич Рахма́нинов, IPA: [sʲɪrˈɡʲej rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢尔盖·瓦西里耶维奇·拉赫玛.apk尼诺夫 .apk b/tests/repo/urzip-Sergey Vasilyevich Rakhmaninov; Серге́й Васи́льевич Рахма́нинов, IPA: [sʲɪrˈɡʲej rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢尔盖·瓦西里耶维奇·拉赫玛.apk尼诺夫 .apk new file mode 100644 index 0000000000000000000000000000000000000000..0851e066f73a2ba72dad3900caa61225bc441787 GIT binary patch literal 11471 zcmdsdWmsIxvgpD+xI4jv6WoHkTY?4#4;I`B4nY&#h5$hWge15-!QI_GxZ9oC_w2K? z_kHL6xbNSq`D(hly1S~ny1wpOtCyN096TO?jEoG3=g!GV&3is;fd_yR900%t-~n-a zGbaaYGi4KdYYTH{7d8(&+lO8NC{0py(AUQ5-kMc0mCGfn9%z4i>I7o&Gw6;@Xn`H6 zD0;zcEj?OZ1mGe$rVA$RC3J^|YbS^Eydl6>Ou%fxaqWRUhEXF7=Rx~?A=d4@7mrJM zJ-fVp=qzHn>29%^;q4rRm!Fj*^`x_igq~S*>onPoJojS$cKGSG&^FRGiVN&L>OIVT z&)^evygH&>DYp4ZW#F5sBX{j8%w=8i4&Nx{D^aaxRcHCq$vB&5#bQ#L>;lEAP|%~fmf+AfSUc?M z@@ZknJ)HtZLXqNOYtYlLVConqp)9zE!GMALH(6++ibhWkGE6Pq3}f9?Jx_~ucMQ+K zZ>*=PVoN*=JT|T=Sh(Br(w7%t4oMm%(|3!;j78_dXLI(#BQf!+C9AR0i5M@Qx~x56 zo)(w22hGD&ePz>vJ6ho5 z6K;lR<5RAe!s)N4QNA#2DYd*^9Tq5rA(w=4B11xzkY_|-m3(FD3_s-WcY-6*y_MmW zw8FO^%Om7}#4Jg8#7Fz++hw*c(6A^~Z0ejqDOSqpA->r?5%FejFp@R!*mRB{;{5?n z6z(b?wdcH+WpjM@=XLDi6J#sT;0}Uu+kQ-i(yv4;Mr$BfKG%KC?=ouopqh5ch-`1V`C%LH+n}v>44{*nDitx)(7>3p9Y-cUwMGcQ$t#5gD6X+P$yX z*7W(jJL=Sfkmt4}gpZ*?AlkCa^8zIo@oSYlWu@|Up`DRa>eOV$_i^5XplYfHglImy zl}?`{f{^OMV^ad7dD%nUHbpC8?m?l*A8Gea$v=>vu%bA~+xjfgb|#HartrN4`?h{> zjugG%Tgmlj>Y>XkO)Wojux@!^b=_I2JIf|?c;(SY!fw67>OO;+dC6Qs(|`{(q9Kt& zc+RNorN<71@M(ibLq$mPPAh5R%gsC6K&6~mP=6`wAb)i8gej=#61wAbt{e(L_MTe z@Og}S+MeAmCkE?|dYJe9a31iG@w#}{B@}Gwzu@71yzfAC=n!Laxp-*=s3{^L|Gw3! zK{i2D06+%**aSuJ2UtT8004?)01V({?#yoHWa4gOYHQAJ4KlVhakU3onLDvL+FJq< zYAUiAs3efl0EWDrlm?_Ef+8{^Z97rp{&Ep%&k+=rO4LjjCKNn!Gp<-$t0^TtjB_U9#B_ zM2=Cl23L!sNWnX}_b4zlNmj?lzthmr`KVA-foi_145*#;c=~wz@FkQDA|E?rYSi@1 zsZ36)kC1wr+_Xa%nC ze}+X)@NAJ)2?_A{89W066R@6rvG0635h}0c7q8s38c}PIKt@J3@130;0oHlE_tw&S z`2r_3XV%}XL@67kjS)Mif}|t#+Vl9JCi4RJt)W+sxV?nT;`pYXoty*684Vj7108*M zx;@N{H{IIGDjZ!>^2|b=x(Bb==HRZAOZ8E$K{M_>=|rAwYUHQE8SPi7fNay47$*gf z3(c@>h<0C&Cvz&J@|TGV-f9C{8k)!%sro(zYgZeR4Ks(_@iU(Bu{l{#uFR}aoGeYm zUCj$(DjkpOpm3r@10O9~P0Lu-gKU>bER;+_LBVPB*%>EQMFYdIA@HqfTh;vNp4Hfw z)C&6pz2Xv49wWXObWOiO-Y>a7Y9tDu;J~$T1K);=LOM-}+n;V~OqXCO=-;A>@TTq@ z&OWXnRXjhYBjJouDU3xYWn|dndA2l@XX>WaAVo*0D~$2_){|Lh*M4D9fk2TreV|gm z!3vw-c9J{nt6GgXElxo9nsTAg5xKfvRV1fAOHo*?Hn$I39|CY+YoGhE{w&c}erCq? zG#ReBva($^H}!0n3mpy-;O-$=8bL9yUo_>?VfaeI5!He7ni%P>c&{V1ixQsxhj#1A zY@lU#yZ?vy7Gjj+V-~is0csLV-~z6F`(q6Fp;Vm{Z4Dy`ZhLoE=F?-;7NXC+Hm-tO zlj~ThiBbm#?;6StQgkOMR_TG<>pn$gD(2?K>t*TL{N}5f+MA1CO-fDVTe5&ZOt3_& zQ|cG~?*;WwM>wuN-6`2At4{Q2j`2CzOX;|kJ#-C#ql0`GFr+$^W>Y?5!Q`-YO-Yih zZSCE9o1gcfOdX&8sH~s|k-uV@Jl0dxgtM;H6Pc1HL3%wJ#!HcRb4Ry7XQ*PA={vwO zcGN}Q8FHY_Q>m%4I65D0SBtPveMnj@%zMVO?|XQ#BfVJ7fCO-~OV+haO%;@W-y9Ey zw}=c$BVilvPvsVpwg)Q$%Nuth+cj4+>&BgC%3A* z{@SL!k}r+cxeo4B-sW5wgJU6NiHTg1>S8qsk(Po59su$F&n0uPPVJEG6_A%!kt&mT z9SGgO|4L%#t~6^=VE{l07P@}}f9&73CY}ziF6<^C7i%|b7f)k56Ki{DHtQPKsM7ud zE4W!ipH>1UPfq;9$M-Zl*gT1Jw82>9ROQI44C=NI%pSC;6d|$!1Svw9=FWja(e4fw z?v3@tzNkoDDaw*xFZNyTzLm4?qsq2@IihbsgDVr*h!AieCp&GS+d8oroy4(|VLF$n zN=n<)KOwpUui7V8Yn2Rzm11fC7|trF!?&bo>$TDm~BIV@;;Ci_f-yEXEhxX17H1=D^iF|Qe1v)I*SPn*_q2O;@l7XI~9 zLqhMLx#wvYqG!B^>(4X3y5jJTPwsZ=&4^VtJrqtjmTmNZ5VL$6I`P@B9rGhG%Sw8x z{~u@K@1FbnnfPgX@DUku9zyX600;G{gR2wB+?mb9$r%K3GAw}pMjD|=20<4H-x#oi@T?)D*dV&?AQAz99@5td z@Pcq10al;|t{dq7*MGV5nu@&bK_p2WqpPkyE5He_051g34bK7p0$v@Ar39-)Q54}+9oFP1PKn}vQh5Q@~z70eQbVh8y*$fr&{|it-;sx^H2}J!C zH_$kOt`n61C$RiUKuUB7Qb7=Tdj*xh?hfsEqRRsXde~)m0 zaE&3B{WFTt{)qekjw0jVq6iW~uvh>VL@ z0Z5Mg2S7?lsQs6J@rPaFQ!_b2cFRUcF?R3Eg2&JVh&Lu29(10dy}0M#1|sr=cd z1)=}9^Mv}83WCu2LVXF<|F7~7z0f*TKeQdH4_b$oe?N+B*+D<@p8$XyIW%Gb0Ay?8 z>KBa5vl@lR2HeHl1Xy|BbA^YL)hc z+j4Y$ecPEN6Y|+gNgWr%ktaL6=-w`S zI`I1MT%>2wGb8sOMdS>h#pt7mflo6}MTRcxl3Dl_zm3dQYpmW3JsuZ1_>PHs9!Z^S zgAAu0_lG-3+5JyX?~!zH6CR9N}Nzi{t!x zC>xHbi~UMHhnv1PfSDhYEwHvNgGNstX z8Z*CD;E2`(<__jc=twiJ6}c*a>4?z---omTX|f`Bs0Ahq))+<%9`R-Yvnnw5ou?y; z58@rgmEe{45iZsN)h?W;BXR~}Y@lTeVT*CVVL)|DY>R4(E7lI}EyP?-+-e zGT)O9$1C{b3RVHu%mc25I|n|PLH-(^qcB3Tt^O@d3Y{wl1lC(O_00aNRQ{@;z((c{ znm4Z6Nqf|qyTUQY?*5tAXS_AE}2^kvSd%MH& zucKZjed~x8N>av3 z{3ljoaKbsm@aYoP_jJ)`>m+pwKE6FptO{ugRm?N_w3M+4nF{$D`n!k1XEY&oTa!%Ok7q?ceCE{~njK?M4 z%rx0>wuM!D8)B=?B|h}tH!k;m+`w54{~NOhn287Aj zUGpTJxkL#yit$)|!sFag^EBO1?Ca}=2K)6}cI~DO`Ug9XCbUOIL#NEIXOvbSkoLY9 zj#iI8Qr*41g7%F`eDJ)guTLH#Ta;TE$`VT24WsUHjvT+%xfFwWv@7-rBk}fcd8E11 zJo3*|1Sb=$zj(2PH+%}?+n_k<`f`RM)TM4+pzj=~=F^FA-ue#-uIxuenTsE`_V_|y z>>tJ3RoW=T4;lumxlYP{nw%$=QVU88h=du;t(^B;WFQ~MSuRL}@TPuHM+-k=cNto* z#=4~AG-5kt#zb463D8?*DR5QutO&D6u{>YxIO(4o_R`2;@QL!$Swx_VT)8{x=l3EB zTW{i@>oOg7jiM<_EF~yer8TegX8htGIQR;CJ}~I9ubaL*zNhzmd@yt*R*18jbG6ZG zVlIWR4r8ZA^om;;t)**k>D`_3YMD&*esSRldWW6j#pdz+1NB8Fr77ht)@=a38wIv~ zGyAj485Dv5c3Pjh-BYc`s0pSUn?im+JU72V?@i=8PGO{NVV#snvUulStXg_bvBq$g zEJ-OneU-yyb&IVM`tiNgbZOSP9pj~;m3@>+>{n%QZ$R#!Y-j4=%#p%}sIzeSo*gm| z%}3XncJbzhILVOg7|M0GZ;Y7Sg~O4W}P5dAhe131seEpvp5%Ggf1}`Gzu2zx&&{gyXTs*G_W>+eViH2Y8G=b z)jX$2((EbkvVxN^KzG;z56I@}rlxmUpFM(O#oFk)hBj(PI9P4A`dMpqS63!Ere0%W zK4kf9DV#Qa=kcx6Q+=X=6>k|< zCV5ZWjE}KSmax$TU2;}F_HK%pR(5a0(=;}SNfD$;tXE*YMej zqHjdxqp6dVygUT>XbCj<8G^&($eJg`sC9AUz z+BM7ttKtsfG}5RTlxa6%DA;Q`z34etc!IEqkmAK=vI{n+xzKPc^tro^C~}9(Uqnms z5;NWPeSv#v6}#O>rMItrNJP4hVXLi3VBL;j`n03`i+c@Nzq4#QmBCqcfR&(2VbR*% z{wk``O7k}5;FVF4&A?5Jp=(Lto_+$+8p>39ygcae3VeI1d0BIq?6Y^*pzp$BS+qO% zgJ#GYoPXuf?vyNT8(4udNV|g@ACK48P0RV2zLvO-S3o&4zDoB<(cYwkEm_tr_b5ld zF|NR+T~z<1I7XK|6^ll-6eD&^OZM00g0SmzrM%vszj_fI+yuny7HYX^)%)Rb8m+p`Iz&GCJ%`RQ3 zr7`1ok@lXn9Aln(?q4#(Y^U^N?q~JQMP@B^zxaU>?K6v(=e7=rDW)IvEm>v>6)Jbx z`*n(L!UZrdOTXU-+Aka`*Kd4ZyR_>a`^lcD&ge98_Q-{BWp<*>vzmnUFjhdm#H8f0 zp?Cfg|JFL-(m(Ikv6SS9f%L)GUR}wL!xyv|6Y8*fbTwL{xv98@=v=-r7<%2MH=EaJ z5_zoF5krFRSa2mdeado(`YPr^KvW*EtQ zk|w&7i~A=RoSAe{HN?6wX{`K;VXB&e3^Dvo`pjfZ+Keo!a&q12l3zC4dPk%QNkC}q z8PYY_oY?UenC=z3*S1+eb`(?wlZHnvm$i_X*dIOa{*gJqpg!4srBpL68lNV@Kk(ThwV2T6NWPR9 zgw9=bb=sb04!Mt=gHj*I1P9Eg1vM?R%xkR;Z^`!NzrlT-%d+^eJ{TluIE=Zwk3c|j z^OCT14Z$Fj!X!6z@a4;bxysldu`(YCj~Cye;@-8dQy$zs4UKJl-ccj^xv|4YCebA+ zGm2J`cyeKN_3aJGvpbTKvZVVoH;Z#;ukEQ6-i&S2l1~Usgtl$bd00i~+)6 z5h*;2%jWEAG?8?DLwSprmIc=7VY7Q!k#p)E#&RuB_HekZtxWbWX-BFr4RNiCBQVLb z+~F;i7sBZk?$W?~UZII`@FDb~fms%ZK^sG9h%yQOqN993> zu$SZR>ne}S2@d(N(H<=Usx-Jtkg}QVMu(yP&UCv}CErjbq|$OU_1XRg%dO&`kfGkT ztf3nC&>}6L&pel8%*$iP{v!C?!ab&OVPHCOLa@G0&$ockXKCg8LwXLum7#jU#uCAG zz1jVxbWoB%X#KF@e2LgKzT@)X&NuqiiSGvLSvlR3_O*L*e?s&5(V-BJVpX4i^0>C& zu}B78v(U!U9&wUwtUqP1Kk$CCS&YZL`T8*X5fge-_WMKZ-yd&(u+zEVL7vL9Aqag5 zpe(H=&ML1W!>%l@A}=GYsl}!&V>hVi5cq;9WJLt!F~Gb&PO-x7`3@|5qudBVDGaa0 z>!wvA3&c&D)vsSU>FwmJ9h|@!Sez#a8t1%Jv@5n|o>gy1aAD+NenrnImfQP;A~0M{ zK34(-Fh-TULHYX5_j8_giILA&KDthUsFDaFi=7&4js-%8<%T-U_5cqtHLoe*2Y%Sk zK?pW#I)RcWb`j6}+^P28#JnKJkuef*nFDG6By zGRjhHe$yH}B%vQv2= z9-V|B^fko4y(%fKp~a>N@u)Jyqgd}&gdiRTc?075h4dotUMCZjF1;uG(Gp0SEbT@p z?e80HjZ4?4On)kxN<7GjhcCH{!x@7%tBJ0hoF^)&_DRZhusY<#Y%Ca`-!mzXxEy1m zlXz8vT;l!Mo}-AKJD=cotP%qq6)_EnhU=tKINIcjP4WBlK!IJR=3e9c2?0;Q`@Q+r zqz|Fsvm-s-)u3z@g}bQKun4~`jt+g4AO<4r@wVfnmiJvk9Tji`B%i*xzu3v0=v2BK zQLKR@G-fvhIaIeJ$}QIxN}+YZQhwuia1P%QqZG_0ilbY6IUTt_e0)UYGG}2}*YkkQ z!c6eAOuTCi@xwPRl`tMMa>7#@cR|gxHBWMnk=f>_rmzCduqef(zQ;2IOURC;%tT}2 z2g=P1sUbmJ!ff{Ue+2UHj{UpKjr!5?U6BCb9pp6*_Wy9XhNgI@_O;k}BnkL>(1uW< zP)Q?V^ioKo$gGdxyE3D9N0d2NLLRLoZSp>~i5X(DV+@^25)1e9w+G5Z3HM0?Gnh6t zu7|akVDHC()3wuvpqP3JIsbwOntoP?|MK5W9FiD9|07)Lr zd%#ScLv6d6#zPc85kW(JhYBAX91AQ1?}YPbv(Vn($-)udfStCo@Uh@&^>C=B??_+M z!Jr{o5i(xDy{7%2Tspd2@AP7^j!e+wq9NEN$l2Ei6 zClUxCh+}!&xLb}bd;uRa8I2$X358bJC1=df(o8++s zds$vaIqA+Du_ujOg}Yn$|Jr$5>xmM}s9vAH=6L8BM{nmk<^5p41eC)*$nDjVXxST z{7NB7hf&(#QpuKt7txuLP4ilO`K4yYDiBmUz288sQ%q&eb2Z90Sl$8fIVILFf4Yp( z3MgZCo%cImS;rH}*BrFlGHYxs^Wrtx)pTDCPPNfZYEwJGBH%%T zDD;EYd6rhRZL|1&?37V)yU6}zy3V|Dtn)*cLFLr>AUI;-;N7#W+(I{9Qx+P5m0MJf z?V5vjq^CZ%ercZWcwGN58Qw z;Fd~S4Jv#r|J7@a9)?($lhxdIp3Y}7fo5#_&@yR+`HEtI>S4`(s#`mx`Kv zOd7XfB?g8T%WP88oNc`pMlYEIz*9-abx0uBocZu}Q>HKyw@4FRh^6@N@4FwNQ zS_TV^YW25dj#yj*A8FT=e`0*u$U1K>V)f&gNmMsFwR*$zw!33Buum@W+w7pQ&CGh? zM%ml*(3pd*_y$r|^VuE^4Ly^NwbAbV%97+RTf?VE&1xEpqRyZkB}%|%IuN>UD9z83UI#oKd|6$GHG4a|e=_~lhcjR`Et+L| zRFvbzHpP$0Zms4Mw}a2wI+Hm0y~T&9$0R~^w_zEKk%axWgx~cB^`4H^>5g^tk$$&* z_Dd9maZ^6(aF_V-nMQc%S1Hn=4m{C;mwA+zLcv`^RU6~mT62f1p91WtG$}us5V{T> zJg_@#J-|VAD`)3!;iw6Id>?ZTpFEL~xi1pEyw$&IC%iTulhlAo2|QXt!y~ce!MS0i zw^9lfq_hmkIpDUoTtl+Pc6l+Ik50N!u>-DBm!B(lEweG%Q{w-A#dKq-61s~MrnPRy zotYi8e7djGI%}0Gd-2B3>Bp~ux6$I;?|kN(;K6vAE^0^Hp`)M}YJ-QL6AA^XyUa?J zU7eiRMP<6jA;yOw3exVpf!B>>iyyA2!$MJwo5B@N6*k50T}|If&OE(E9m;&tBE+_& zqCy|HID|#n(I=z!eC)vQyGBeuHO9BuHlTCcIiX7Q>F$9g2cNl^$)sIxU59CzFG3ib zo=FIePF^r)^GXA}vb*KVTwX=b=TzhU*tHr_S49fU9lfm)ldzj}ReR+*am$*7J`(xJ z(%VR$HIGDtuJQGbD&zP4zoLYd+QzCI%X8X47L63M3!-*zW`cKqz2R8zm^>NGjt!%V z-Oa2|9Xl61=JOs~Q7jx8%#!}9YS8f@=Ae>%i*y;&I(|vvhrI< zHQi#-fG+L^(Lga;h1#h~qIr`2c_nO@iv9G4cxQw%y85Ds3w)YnFRN>kPmd^L^<`gD z_k4S7mAPxZyZy0QMet={tdXD_W3`qS;n1~ZU*TykI-zg@gS2_5(AE`vWwT5829x1a z%iZ>I_o#jz_E5ubq_I3cd)wABmT7(^`evH#H=mOEc5@$KplQ*MpfBFjHpEmvd zw*4IsCi8y*JJoMV=x?e0KPCL0 zUqKVszaWJAHwphrWB)n&_mzg`V1L0C%|Ay&GBPzq1n4Jrkc&5D00VL}SHcXf{TIgD BZBqaM literal 0 HcmV?d00001 diff --git a/tests/run-tests b/tests/run-tests index c58942bd..8558794e 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -8,7 +8,7 @@ echo_header() { copy_apks_into_repo() { set +x - for f in `find $APKDIR -name '*.apk' | grep -F -v -e unaligned -e unsigned -e badsig -e badcert`; do + for f in `find $APKDIR -name '*.apk' | grep -F -v -e unaligned -e unsigned -e badsig -e badcert -e bad-unicode`; do name=$(basename $(dirname `dirname $f`)) apk=`$aapt dump badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"` test $f -nt repo/$apk && rm -f repo/$apk # delete existing if $f is newer @@ -157,7 +157,7 @@ $fdroid init sed -i.tmp 's,^ *repo_description.*,repo_description = """获取已安装在您的设备上的应用的,' config.py echo "mirrors = ('https://foo.bar/fdroid', 'http://secret.onion/fdroid')" >> config.py mkdir metadata -cp $WORKSPACE/tests/urzip.apk repo/ +cp $WORKSPACE/tests/urzip.apk $WORKSPACE/tests/bad-unicode*.apk repo/ cp $WORKSPACE/tests/metadata/info.guardianproject.urzip.yml metadata/ $fdroid readmeta diff --git a/tests/stats/known_apks.txt b/tests/stats/known_apks.txt index 6d457a6f..b9fd7f4b 100644 --- a/tests/stats/known_apks.txt +++ b/tests/stats/known_apks.txt @@ -9,4 +9,4 @@ obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01 obb.main.twoversions_1101617.apk obb.main.twoversions 2016-06-20 obb.mainpatch.current_1619.apk obb.mainpatch.current 2016-04-23 obb.mainpatch.current_1619_another-release-key.apk obb.mainpatch.current 2017-06-01 -urzip-πÇÇπÇÇ现代汉语通用字-български-عربي1234.apk info.guardianproject.urzip 2016-06-23 +urzip-Sergey Vasilyevich Rakhmaninov; Серге́й Васи́льевич Рахма́нинов, IPA: [sʲɪrˈɡʲej rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢尔盖·瓦西里耶维奇·拉赫玛.apk尼诺夫 .apk info.guardianproject.urzip 2016-06-23 From 176f539647a6c343ebc41284d0a9608a012f85eb Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 19 Sep 2017 10:57:29 +0200 Subject: [PATCH 05/14] allow spaces in filenames This fixes all the bugs I could find that prevented fdroid from handling files with spaces in them. This is more important now that fdroid supports random media files, and Repomaker --- fdroidserver/common.py | 12 +++++++++++- fdroidserver/update.py | 10 ---------- tests/run-tests | 10 +++++----- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 807974d6..c8442d5e 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -1599,6 +1599,13 @@ class KnownApks: """ def __init__(self): + '''Load filename/date info about previously seen APKs + + Since the appid and date strings both will never have spaces, + this is parsed as a list from the end to allow the filename to + have any combo of spaces. + ''' + self.path = os.path.join('stats', 'known_apks.txt') self.apks = {} if os.path.isfile(self.path): @@ -1608,7 +1615,10 @@ class KnownApks: if len(t) == 2: self.apks[t[0]] = (t[1], None) else: - self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d')) + appid = t[-2] + date = datetime.strptime(t[-1], '%Y-%m-%d') + filename = line[0:line.rfind(appid) - 1] + self.apks[filename] = (appid, date) self.changed = False def writeifchanged(self): diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 65379276..99fb80e5 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -1197,16 +1197,6 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal apk is the scanned apk information, and cachechanged is True if the apkcache got changed. """ - if ' ' in apkfilename: - if options.rename_apks: - newfilename = apkfilename.replace(' ', '_') - os.rename(os.path.join(repodir, apkfilename), - os.path.join(repodir, newfilename)) - apkfilename = newfilename - else: - logging.critical("Spaces in filenames are not allowed.") - return True, None, False - apk = {} apkfile = os.path.join(repodir, apkfilename) diff --git a/tests/run-tests b/tests/run-tests index 8558794e..e0986b97 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -8,14 +8,14 @@ echo_header() { copy_apks_into_repo() { set +x - for f in `find $APKDIR -name '*.apk' | grep -F -v -e unaligned -e unsigned -e badsig -e badcert -e bad-unicode`; do - name=$(basename $(dirname `dirname $f`)) + find $APKDIR -type f -name '*.apk' -print0 | while IFS= read -r -d '' f; do + echo $f | grep -F -v -e unaligned -e unsigned -e badsig -e badcert -e bad-unicode || continue apk=`$aapt dump badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"` - test $f -nt repo/$apk && rm -f repo/$apk # delete existing if $f is newer + test "$f" -nt repo/$apk && rm -f repo/$apk # delete existing if $f is newer if [ ! -e repo/$apk ] && [ ! -e archive/$apk ]; then echo "$f --> repo/$apk" - ln $f $1/repo/$apk || \ - rsync -axv $f $1/repo/$apk # rsync if hard link is not possible + ln "$f" $1/repo/$apk || \ + rsync -axv "$f" $1/repo/$apk # rsync if hard link is not possible fi done set -x From bc192b6be0d070b5abb2cf274fe362a80ae86f18 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sat, 16 Sep 2017 10:03:17 +0200 Subject: [PATCH 06/14] tests: prefer GNU sed on BSD/OSX, BSD's sed has lame syntax https://blog.remibergsma.com/2012/09/18/sed-inline-editing-different-on-mac-osx/ --- tests/run-tests | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index e0986b97..0ce5e69e 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -91,6 +91,13 @@ if [ -z $python ]; then python=python3 fi +# try to use GNU sed on OSX/BSD cuz BSD sed sucks +if which gsed; then + sed=gsed +else + sed=sed +fi + set -x # show each command as it is executed #------------------------------------------------------------------------------# @@ -154,7 +161,7 @@ REPOROOT=`create_test_dir` cd $REPOROOT $fdroid init -sed -i.tmp 's,^ *repo_description.*,repo_description = """获取已安装在您的设备上的应用的,' config.py +$sed -i.tmp 's,^ *repo_description.*,repo_description = """获取已安装在您的设备上的应用的,' config.py echo "mirrors = ('https://foo.bar/fdroid', 'http://secret.onion/fdroid')" >> config.py mkdir metadata cp $WORKSPACE/tests/urzip.apk $WORKSPACE/tests/bad-unicode*.apk repo/ @@ -237,7 +244,7 @@ test -e repo/obb.main.twoversions_1101617_src.tar.gz.asc ! test -e repo/index.xml.asc # we can't easily reproduce the timestamps for things, so just hardcode them -sed -i --expression='s,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xml +$sed -i.tmp -e 's,timestamp="[0-9]*",timestamp="1480431575",' repo/index.xml diff -uw $WORKSPACE/tests/repo/index.xml repo/index.xml @@ -251,19 +258,19 @@ $fdroid init --keystore keystore.jks --repo-keyalias=sova echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py echo "accepted_formats = ['txt']" >> config.py -sed -i '/allow_disabled_algorithms/d' config.py +$sed -i.tmp '/allow_disabled_algorithms/d' config.py test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/*.txt metadata/ echo 'Summary:good test version of urzip' > metadata/info.guardianproject.urzip.txt echo 'Summary:good MD5 sig, which is disabled algorithm' > metadata/org.bitbucket.tickytacky.mirrormirror.txt -sed -i '/Archive Policy:/d' metadata/*.txt +$sed -i.tmp '/Archive Policy:/d' metadata/*.txt test -d repo || mkdir repo cp $WORKSPACE/tests/urzip.apk \ $WORKSPACE/tests/org.bitbucket.tickytacky.mirrormirror_[0-9].apk \ $WORKSPACE/tests/repo/com.politedroid_[0-9].apk \ $WORKSPACE/tests/repo/obb.main.twoversions_110161[357].apk \ repo/ -sed -i 's,archive_older = [0-9],archive_older = 3,' config.py +$sed -i.tmp 's,archive_older = [0-9],archive_older = 3,' config.py $fdroid update --pretty --nosign echo "This will fail when jarsigner allows MD5 for APK signatures" @@ -285,7 +292,7 @@ test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/ test -d repo || mkdir repo cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/ -sed -i 's,archive_older = [0-9],archive_older = 3,' config.py +$sed -i.tmp 's,archive_older = [0-9],archive_older = 3,' config.py $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 0 @@ -300,7 +307,7 @@ test -e repo/com.politedroid_5.apk test -e repo/com.politedroid_6.apk echo "enable one app in the repo" -sed -i 's,^Archive Policy:4,Archive Policy:1,' metadata/com.politedroid.txt +$sed -i.tmp 's,^Archive Policy:4,Archive Policy:1,' metadata/com.politedroid.txt $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 3 test `grep '' repo/index.xml | wc -l` -eq 1 @@ -314,7 +321,7 @@ test -e archive/com.politedroid_5.apk test -e repo/com.politedroid_6.apk echo "remove all apps from the repo" -sed -i 's,^Archive Policy:1,Archive Policy:0,' metadata/com.politedroid.txt +$sed -i.tmp 's,^Archive Policy:1,Archive Policy:0,' metadata/com.politedroid.txt $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 4 test `grep '' repo/index.xml | wc -l` -eq 0 @@ -329,7 +336,7 @@ test -e archive/com.politedroid_6.apk ! test -e repo/com.politedroid_6.apk echo "move back one from archive to the repo" -sed -i 's,^Archive Policy:0,Archive Policy:1,' metadata/com.politedroid.txt +$sed -i.tmp 's,^Archive Policy:0,Archive Policy:1,' metadata/com.politedroid.txt $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 3 test `grep '' repo/index.xml | wc -l` -eq 1 @@ -357,10 +364,10 @@ echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py echo "accepted_formats = ['txt']" >> config.py test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/ -sed -i '/Archive Policy:/d' metadata/com.politedroid.txt +$sed -i.tmp '/Archive Policy:/d' metadata/com.politedroid.txt test -d repo || mkdir repo cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk repo/ -sed -i 's,archive_older = [0-9],archive_older = 3,' config.py +$sed -i.tmp 's,archive_older = [0-9],archive_older = 3,' config.py $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 1 @@ -374,7 +381,7 @@ test -e repo/com.politedroid_4.apk test -e repo/com.politedroid_5.apk test -e repo/com.politedroid_6.apk -sed -i 's,archive_older = 3,archive_older = 1,' config.py +$sed -i.tmp 's,archive_older = 3,archive_older = 1,' config.py $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 3 test `grep '' repo/index.xml | wc -l` -eq 1 @@ -388,7 +395,7 @@ test -e archive/com.politedroid_5.apk test -e repo/com.politedroid_6.apk # disabling deletes from the archive -sed -i 's/Build:1.3,4/Build:1.3,4\n disable=testing deletion/' metadata/com.politedroid.txt +$sed -i.tmp 's/Build:1.3,4/Build:1.3,4\n disable=testing deletion/' metadata/com.politedroid.txt $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 2 test `grep '' repo/index.xml | wc -l` -eq 1 @@ -402,7 +409,7 @@ test -e archive/com.politedroid_5.apk test -e repo/com.politedroid_6.apk # disabling deletes from the repo, and promotes one from the archive -sed -i 's/Build:1.5,6/Build:1.5,6\n disable=testing deletion/' metadata/com.politedroid.txt +$sed -i.tmp 's/Build:1.5,6/Build:1.5,6\n disable=testing deletion/' metadata/com.politedroid.txt $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 1 test `grep '' repo/index.xml | wc -l` -eq 1 @@ -425,12 +432,12 @@ echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.p echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py echo "accepted_formats = ['txt']" >> config.py echo 'allow_disabled_algorithms = True' >> config.py -sed -i 's,archive_older = [0-9],archive_older = 3,' config.py +$sed -i.tmp 's,archive_older = [0-9],archive_older = 3,' config.py test -d metadata || mkdir metadata cp $WORKSPACE/tests/metadata/com.politedroid.txt metadata/ echo 'Summary:good test version of urzip' > metadata/info.guardianproject.urzip.txt echo 'Summary:good MD5 sig, disabled algorithm' > metadata/org.bitbucket.tickytacky.mirrormirror.txt -sed -i '/Archive Policy:/d' metadata/*.txt +$sed -i.tmp '/Archive Policy:/d' metadata/*.txt test -d repo || mkdir repo cp $WORKSPACE/tests/repo/com.politedroid_[0-9].apk \ $WORKSPACE/tests/org.bitbucket.tickytacky.mirrormirror_[0-9].apk \ @@ -460,7 +467,7 @@ test -e repo/org.bitbucket.tickytacky.mirrormirror_3.apk test -e repo/org.bitbucket.tickytacky.mirrormirror_4.apk test -e archive/urzip-badsig.apk -sed -i '/allow_disabled_algorithms/d' config.py +$sed -i.tmp '/allow_disabled_algorithms/d' config.py $fdroid update --pretty --nosign test `grep '' archive/index.xml | wc -l` -eq 5 test `grep '' repo/index.xml | wc -l` -eq 3 @@ -972,7 +979,7 @@ test -e tmp/apkcache grep -F ' Date: Fri, 15 Sep 2017 23:25:07 +0200 Subject: [PATCH 07/14] tests: support Ubuntu/trusty's python3.4-venv --- tests/complete-ci-tests | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/complete-ci-tests b/tests/complete-ci-tests index b09e111d..9792b5bf 100755 --- a/tests/complete-ci-tests +++ b/tests/complete-ci-tests @@ -47,6 +47,18 @@ export PATH=/usr/lib/jvm/java-8-openjdk-amd64/bin:$PATH cd $WORKSPACE/tests ./run-tests $apksource +#------------------------------------------------------------------------------# +# find pyvenv, to support Ubuntu/trusty's python3.4-venv + +if which pyvenv; then + pyvenv=pyvenv +elif which pyvenv-3.4; then + pyvenv=pyvenv-3.4 +else + echo "pyvenv required to run this test suite!" + exit 1 +fi + #------------------------------------------------------------------------------# # test building the source tarball, then installing it @@ -54,7 +66,7 @@ cd $WORKSPACE python3 setup.py sdist rm -rf $WORKSPACE/env -pyvenv $WORKSPACE/env +$pyvenv $WORKSPACE/env . $WORKSPACE/env/bin/activate # workaround https://github.com/pypa/setuptools/issues/937 pip3 install setuptools==33.1.1 @@ -68,7 +80,7 @@ fdroid=$WORKSPACE/env/bin/fdroid $WORKSPACE/tests/run-tests $apksource # test install using install direct from git repo cd $WORKSPACE rm -rf $WORKSPACE/env -pyvenv $WORKSPACE/env +$pyvenv $WORKSPACE/env . $WORKSPACE/env/bin/activate # workaround https://github.com/pypa/setuptools/issues/937 pip3 install setuptools==33.1.1 From 0a88a97881d928472af4965abbcd9013037e8f60 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 19 Sep 2017 19:58:31 +0200 Subject: [PATCH 08/14] travis-ci: update OSX CI build to work again --- .travis.yml | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 58b28ebe..7a7ce68e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,9 @@ matrix: # this doesn't actually work yet https://github.com/travis-ci/travis-ci/issues/5337 dist: trusty - os: osx - osx_image: xcode8 - env: ANDROID_HOME=/usr/local/opt/android-sdk + osx_image: xcode9 + env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk + env: ANDROID_HOME=/usr/local/share/android-sdk licenses: - 'android-sdk-preview-license-52d11cd2' @@ -24,21 +25,32 @@ licenses: install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update > /dev/null; - brew install android-sdk dash gnu-sed jpeg python3; - sudo pip3 install pep8 pyflakes pylint; + brew install dash bash python3 gradle; + brew install gnu-sed --with-default-names; + brew cask install android-sdk; + + mkdir -p "$ANDROID_HOME/licenses"; + echo -e "\n8933bad161af4178b1185d1a37fbf41ea5269c55" > "$ANDROID_HOME/licenses/android-sdk-license"; + echo -e "\n84831b9409646a918e30573bab4c9c91346d8abd" > "$ANDROID_HOME/licenses/android-sdk-preview-license"; + echo y | $ANDROID_HOME/tools/bin/sdkmanager "platform-tools"; + echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;25.0.2"; + echo y | $ANDROID_HOME/tools/bin/sdkmanager "platforms;android-23"; + sudo pip3 install -e .; sudo rm -rf fdroidserver.egg-info; - echo y | android --verbose update sdk --no-ui --all --filter platform-tools,build-tools-25.0.2; - elif [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - sudo add-apt-repository ppa:guardianproject/fdroidserver -y; - sudo apt-get -q update -y; - sudo apt-get -q install -y --no-install-recommends python3 python3-dev - python3-git python3-pil python3-libcloud python3-logilab-astng - python3-paramiko python3-pip python3-pyasn1 python3-pyasn1-modules - python3-requests python3-venv python3-yaml rsync - pyflakes pylint3 pep8 dash bash ruby libjpeg-dev zlib1g-dev; - sudo pip3 install pylint; - fi + + echo $PATH; + echo $JAVA_HOME; + /usr/libexec/java_home; + java -version; + which java; + javac -version; + which javac; + jarsigner -help; + which jarsigner; + keytool -help; + which keytool; + fi script: - cd tests @@ -46,5 +58,4 @@ script: after_failure: - cd $TRAVIS_BUILD_DIR - - ls -lRa env - ls -lR | curl -F 'clbin=<-' https://clbin.com From 2b5edf2434941d43db4374e9b0c7497b8dc0f742 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 19 Sep 2017 19:58:45 +0200 Subject: [PATCH 09/14] travis-ci: make Trusty builds a first class citizen Trusty is used in Travis-CI, Windows Subsystem for Linux, and many other places, so its an important target platform to test on. This provides testing for the backports in the Launchpad PPAs like: * https://launchpad.net/~fdroid/+archive/ubuntu/fdroidserver * https://launchpad.net/~fdroid/+archive/ubuntu/buildserver --- .travis.yml | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7a7ce68e..7b39e720 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,22 +3,39 @@ language: java matrix: - allow_failures: - - os: linux # this is really about OSX, Ubuntu is just bonus include: - os: linux language: android - sudo: required - # this doesn't actually work yet https://github.com/travis-ci/travis-ci/issues/5337 - dist: trusty - os: osx osx_image: xcode9 env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk env: ANDROID_HOME=/usr/local/share/android-sdk -licenses: - - 'android-sdk-preview-license-52d11cd2' - - 'android-sdk-license-.+' +addons: + apt: + sources: + - sourceline: 'ppa:fdroid/fdroidserver' + packages: + - bash + - dash + - pylint + - pep8 + - python3-dev + - python3-pip + - python3-ruamel.yaml + - python3-setuptools + - python3.4-venv + - libjpeg-dev + - zlib1g-dev + - fdroidserver + +android: + components: + - android-23 # required for `fdroid build` test + - build-tools-25.0.3 # required for `fdroid build` test + licenses: + - 'android-sdk-preview-.+' + - 'android-sdk-license-.+' # the PPA is needed on Ubuntu 14.04 precise, and with python3, trusty too # the pip thing is a hack that can go away with trusty From 373b46ab3fe1c783e5cc8fc5d02b6c1a4ad64390 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 14 Sep 2017 14:34:29 +0200 Subject: [PATCH 10/14] tests: don't run gpgsign tests on Travis CI's OSX, gpg always fails --- tests/run-tests | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index 0ce5e69e..346e8cdd 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -235,13 +235,16 @@ test -e repo/index-v1.jar grep -F ' Date: Tue, 19 Sep 2017 20:01:15 +0200 Subject: [PATCH 11/14] tests: support Java setups where MD5 is not disabled by default For platforms using Java < 1.8.0_133, MD5 is still enabled for JAR signatures. Its just too painful to manage all this, so support this in the tests. --- tests/IsMD5Disabled.java | 22 ++++++++++++++++++++++ tests/run-tests | 5 +++++ 2 files changed, 27 insertions(+) create mode 100644 tests/IsMD5Disabled.java diff --git a/tests/IsMD5Disabled.java b/tests/IsMD5Disabled.java new file mode 100644 index 00000000..4844619f --- /dev/null +++ b/tests/IsMD5Disabled.java @@ -0,0 +1,22 @@ + +import java.security.Security; +import java.util.Locale; + +public class IsMD5Disabled { + public static void main(String[] args) throws Exception { + String daString = Security.getProperty("jdk.jar.disabledAlgorithms"); + String[] algorithms = daString.trim().split(","); + boolean isMD5Disabled = true; + for (String alg : algorithms) { + if (alg.trim().toLowerCase(Locale.US).startsWith("md5")) { + isMD5Disabled = false; + } + } + if (isMD5Disabled) { + System.out.println("MD5 in jdk.jar.disabledAlgorithms: " + daString); + } else { + System.out.println("MD5 allowed for JAR signatures: " + daString); + System.exit(1); + } + } +} diff --git a/tests/run-tests b/tests/run-tests index 346e8cdd..27dc651e 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -46,6 +46,11 @@ have_git_2_3() { python3 -c "import sys; from distutils.version import LooseVersion as V; sys.exit(V(sys.argv[3]) < V('2.3'))" `git --version` } +is_MD5_disabled() { + javac $WORKSPACE/tests/IsMD5Disabled.java && java -cp $WORKSPACE/tests IsMD5Disabled + return $? +} + #------------------------------------------------------------------------------# # "main" From 2764c43fa2410eee5572817c27253ed85e74bd6b Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 19 Sep 2017 16:41:50 +0200 Subject: [PATCH 12/14] tests: use ci-test-app with current Gradle Android Plugin This works around the gradle 2.x bug where versions newer than 2.9 cannot run things. This also then specifies the version of Gradle Android Plugin that is included in Debian/stretch. --- tests/run-tests | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index 27dc651e..b33fa319 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -177,23 +177,23 @@ $fdroid update #------------------------------------------------------------------------------# -echo_header 'run `fdroid build` in fresh git checkout from import.TestCase' +echo_header 'run "fdroid build" in fresh git checkout from import.TestCase' cd $WORKSPACE/tests/tmp/importer git remote update -p git clean -fdx # stick with known working commit, in case future commits break things for this code -git reset --hard cecf00c08aec56ae7a5eba444150c4d1ae868814 +git reset --hard 985aa135524ab7dd1e70335fd47b22fa628b81b3 if [ -d $ANDROID_HOME/platforms/android-23 ]; then echo "build_tools = '`ls -1 $ANDROID_HOME/build-tools/ | sort -n | tail -1`'" > config.py echo "force_build_tools = True" >> config.py $fdroid build --verbose org.fdroid.ci.test.app:300 else - echo 'WARNING: Skipping `fdroid build` test since android-23 is missing!' + echo 'WARNING: Skipping "fdroid build" test since android-23 is missing!' fi #------------------------------------------------------------------------------# -echo_header 'copy git import and run `fdroid scanner` on it' +echo_header 'copy git import and run "fdroid scanner" on it' REPOROOT=`create_test_dir` cd $REPOROOT From a9a696b199d63ed33ef0f22822af32c5fbb18e88 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sun, 17 Sep 2017 12:54:43 +0200 Subject: [PATCH 13/14] travis-ci: test on older OSX versions --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7b39e720..ff034b68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,14 @@ matrix: osx_image: xcode9 env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk env: ANDROID_HOME=/usr/local/share/android-sdk + - os: osx + osx_image: xcode7.3 + env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk + env: ANDROID_HOME=/usr/local/share/android-sdk + - os: osx + osx_image: xcode6.4 + env: ANDROID_SDK_ROOT=/usr/local/share/android-sdk + env: ANDROID_HOME=/usr/local/share/android-sdk addons: apt: From 86a3ef5355c828e2dc56b586fc6f5d690ca5afda Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 19 Sep 2017 20:10:25 +0200 Subject: [PATCH 14/14] travis-ci: always reinstall Java on OSX to get recent version The Travis images have quite old versions of Java currently. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ff034b68..80ce86a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,8 +50,9 @@ android: install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update > /dev/null; - brew install dash bash python3 gradle; + brew install dash bash python3 gradle jenv; brew install gnu-sed --with-default-names; + brew cask reinstall java; brew cask install android-sdk; mkdir -p "$ANDROID_HOME/licenses"; @@ -66,6 +67,7 @@ install: echo $PATH; echo $JAVA_HOME; + jenv versions; /usr/libexec/java_home; java -version; which java;