mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-08-15 10:50:09 +02:00
Merge branch 'auto-install-ndk' into 'master'
auto-download missing NDKs if they're known and can be verified Closes #369 and #717 See merge request fdroid/fdroidserver!919
This commit is contained in:
Hans-Christoph Steiner
commit
be9036b1f1
@ -261,7 +261,7 @@ fedora_latest:
|
||||
"cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
|
||||
|
||||
|
||||
gradle:
|
||||
gradle/ndk:
|
||||
image: debian:bullseye
|
||||
<<: *apt-template
|
||||
variables:
|
||||
@ -276,16 +276,18 @@ gradle:
|
||||
python3-git
|
||||
python3-gitlab
|
||||
python3-requests
|
||||
# if this is a merge request fork, then only check if makebuildserver or gradlew-fdroid changed
|
||||
# if this is a merge request fork, then only check if relevant files changed
|
||||
- if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
|
||||
git fetch https://gitlab.com/fdroid/fdroidserver.git;
|
||||
for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
|
||||
test "$f" == "makebuildserver" && export CHANGED="yes";
|
||||
test "$f" == "gradlew-fdroid" && export CHANGED="yes";
|
||||
test "$f" == "fdroidserver/common.py" && export CHANGED="yes";
|
||||
done;
|
||||
test -z "$CHANGED" && exit;
|
||||
fi
|
||||
- ./tests/gradle-release-checksums.py
|
||||
- ./tests/ndk-release-checksums.py
|
||||
|
||||
|
||||
# Run an actual build in a simple, faked version of the buildserver guest VM.
|
||||
|
||||
@ -1,16 +1,6 @@
|
||||
sdk_path: /home/vagrant/android-sdk
|
||||
ndk_paths:
|
||||
r10e: /home/vagrant/android-ndk/r10e
|
||||
r11c: /home/vagrant/android-ndk/r11c
|
||||
r12b: /home/vagrant/android-ndk/r12b
|
||||
r13b: /home/vagrant/android-ndk/r13b
|
||||
r14b: /home/vagrant/android-ndk/r14b
|
||||
r15c: /home/vagrant/android-ndk/r15c
|
||||
r16b: /home/vagrant/android-ndk/r16b
|
||||
r17c: /home/vagrant/android-ndk/r17c
|
||||
r18b: /home/vagrant/android-ndk/r18b
|
||||
r19c: /home/vagrant/android-ndk/r19c
|
||||
r20b: /home/vagrant/android-ndk/r20b
|
||||
r21e: /home/vagrant/android-ndk/r21e
|
||||
r22b: /home/vagrant/android-ndk/r22b
|
||||
|
||||
|
||||
@ -15,7 +15,7 @@ if [ ! -e $NDK_BASE/r10e ]; then
|
||||
mv android-ndk-r10e r10e
|
||||
fi
|
||||
|
||||
for version in r11c r12b r13b r14b r15c r16b r17c r18b r19c r20b r21e r22b; do
|
||||
for version in r21e r22b; do
|
||||
if [ ! -e ${NDK_BASE}/${version} ]; then
|
||||
unzip /vagrant/cache/android-ndk-${version}-linux-x86_64.zip > /dev/null
|
||||
mv android-ndk-${version} ${version}
|
||||
|
||||
@ -5,14 +5,14 @@
|
||||
# Custom path to the Android SDK, defaults to $ANDROID_HOME
|
||||
# sdk_path: $ANDROID_HOME
|
||||
|
||||
# Custom paths to various versions of the Android NDK, defaults to 'r12b' set
|
||||
# to $ANDROID_NDK. Most users will have the latest at $ANDROID_NDK, which is
|
||||
# used by default. If a version is missing or assigned to None, it is assumed
|
||||
# not installed.
|
||||
# Paths to various installed versions of the Android NDK. If a
|
||||
# required version is missing in the buildserver VM, it will be
|
||||
# automatically downloaded and installed into a temporary dir.
|
||||
#
|
||||
# ndk_paths:
|
||||
# r10e: None
|
||||
# r11c: None
|
||||
# r12b: $ANDROID_NDK
|
||||
# r12b: None
|
||||
# r13b: None
|
||||
# r14b: None
|
||||
# r15c: None
|
||||
|
||||
@ -361,13 +361,16 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
|
||||
ndk_path = build.ndk_path()
|
||||
if build.ndk or (build.buildjni and build.buildjni != ['no']):
|
||||
if not ndk_path:
|
||||
logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r12b')
|
||||
logging.critical("Android NDK version '%s' could not be found!" % build.ndk)
|
||||
logging.critical("Configured versions:")
|
||||
for k, v in config['ndk_paths'].items():
|
||||
if k.endswith("_orig"):
|
||||
continue
|
||||
logging.critical(" %s: %s" % (k, v))
|
||||
raise FDroidException()
|
||||
if onserver:
|
||||
common.auto_install_ndk(build)
|
||||
else:
|
||||
raise FDroidException()
|
||||
elif not os.path.isdir(ndk_path):
|
||||
logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
|
||||
raise FDroidException()
|
||||
|
||||
@ -58,6 +58,7 @@ try:
|
||||
except ImportError:
|
||||
import xml.etree.ElementTree as XMLElementTree # nosec this is a fallback only
|
||||
|
||||
from base64 import urlsafe_b64encode
|
||||
from binascii import hexlify
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from distutils.version import LooseVersion
|
||||
@ -68,6 +69,7 @@ from pyasn1.codec.der import decoder, encoder
|
||||
from pyasn1_modules import rfc2315
|
||||
from pyasn1.error import PyAsn1Error
|
||||
|
||||
from . import net
|
||||
import fdroidserver.metadata
|
||||
import fdroidserver.lint
|
||||
from fdroidserver import _
|
||||
@ -116,15 +118,7 @@ orig_path = None
|
||||
|
||||
default_config = {
|
||||
'sdk_path': "$ANDROID_HOME",
|
||||
'ndk_paths': {
|
||||
'r10e': None,
|
||||
'r11c': None,
|
||||
'r12b': "$ANDROID_NDK",
|
||||
'r13b': None,
|
||||
'r14b': None,
|
||||
'r15c': None,
|
||||
'r16b': None,
|
||||
},
|
||||
'ndk_paths': {},
|
||||
'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
|
||||
'java_paths': None,
|
||||
'scan_binary': False,
|
||||
@ -3964,3 +3958,288 @@ def run_yamllint(path, indent=0):
|
||||
for problem in problems:
|
||||
result.append(' ' * indent + path + ':' + str(problem.line) + ': ' + problem.message)
|
||||
return '\n'.join(result)
|
||||
|
||||
|
||||
def sha256sum(filename):
|
||||
'''Calculate the sha256 of the given file'''
|
||||
sha = hashlib.sha256()
|
||||
with open(filename, 'rb') as f:
|
||||
while True:
|
||||
t = f.read(16384)
|
||||
if len(t) == 0:
|
||||
break
|
||||
sha.update(t)
|
||||
return sha.hexdigest()
|
||||
|
||||
|
||||
def sha256base64(filename):
|
||||
'''Calculate the sha256 of the given file as URL-safe base64'''
|
||||
hasher = hashlib.sha256()
|
||||
with open(filename, 'rb') as f:
|
||||
while True:
|
||||
t = f.read(16384)
|
||||
if len(t) == 0:
|
||||
break
|
||||
hasher.update(t)
|
||||
return urlsafe_b64encode(hasher.digest()).decode()
|
||||
|
||||
|
||||
def get_ndk_version(ndk_path):
|
||||
source_properties = os.path.join(ndk_path, 'source.properties')
|
||||
if os.path.exists(source_properties):
|
||||
with open(source_properties) as fp:
|
||||
m = re.search(r'^Pkg.Revision *= *(.+)', fp.read(), flags=re.MULTILINE)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
|
||||
def auto_install_ndk(build):
|
||||
"""auto-install the NDK in the build, this assumes its in a buildserver guest VM
|
||||
|
||||
Download, verify, and install the NDK version as specified via the
|
||||
"ndk:" field in the build entry. As it uncompresses the zipball,
|
||||
this forces the permissions to work for all users, since this
|
||||
might uncompress as root and then be used from a different user.
|
||||
|
||||
This needs to be able to install multiple versions of the NDK,
|
||||
since this is also used in CI builds, where multiple `fdroid build
|
||||
--onserver` calls can run in a single session. The production
|
||||
buildserver is reset between every build.
|
||||
|
||||
The default ANDROID_HOME base dir of /home/vagrant/android-sdk is
|
||||
hard-coded in buildserver/Vagrantfile. The "ndk" subdir is where
|
||||
Android Studio will install the NDK into versioned subdirs.
|
||||
https://developer.android.com/studio/projects/configure-agp-ndk#agp_version_41
|
||||
|
||||
Also, r10e and older cannot be handled via this mechanism because
|
||||
they are packaged differently.
|
||||
|
||||
"""
|
||||
global config
|
||||
if build.get('disable'):
|
||||
return
|
||||
ndk = build.get('ndk')
|
||||
if not ndk:
|
||||
return
|
||||
ndk_path = config.get(ndk)
|
||||
if ndk_path and os.path.isdir(ndk_path):
|
||||
return
|
||||
for ndkdict in NDKS:
|
||||
if ndk == ndkdict['release']:
|
||||
url = ndkdict['url']
|
||||
sha256 = ndkdict['sha256']
|
||||
break
|
||||
ndk_base = os.path.join(config['sdk_path'], 'ndk')
|
||||
logging.info(_('Downloading %s') % url)
|
||||
zipball = os.path.join(
|
||||
tempfile.mkdtemp(prefix='android-ndk-'),
|
||||
os.path.basename(url)
|
||||
)
|
||||
net.download_file(url, zipball)
|
||||
if sha256 != sha256sum(zipball):
|
||||
raise FDroidException('SHA-256 %s does not match expected for %s' % (sha256, url))
|
||||
logging.info(_('Unzipping to %s') % ndk_base)
|
||||
with zipfile.ZipFile(zipball) as zipfp:
|
||||
for info in zipfp.infolist():
|
||||
permbits = info.external_attr >> 16
|
||||
if stat.S_ISDIR(permbits) or stat.S_IXUSR & permbits:
|
||||
zipfp.extract(info.filename, path=ndk_base)
|
||||
os.chmod(os.path.join(ndk_base, info.filename), 0o755) # nosec bandit B103
|
||||
else:
|
||||
zipfp.extract(info.filename, path=ndk_base)
|
||||
os.chmod(os.path.join(ndk_base, info.filename), 0o644) # nosec bandit B103
|
||||
os.remove(zipball)
|
||||
extracted = glob.glob(os.path.join(ndk_base, '*'))[0]
|
||||
version = get_ndk_version(extracted)
|
||||
ndk_dir = os.path.join(ndk_base, version)
|
||||
os.rename(extracted, ndk_dir)
|
||||
if 'ndk_paths' not in config:
|
||||
config['ndk_paths'] = dict()
|
||||
config['ndk_paths'][ndk] = ndk_dir
|
||||
logging.info(_('Set NDK {release} ({version}) up')
|
||||
.format(release=ndk, version=version))
|
||||
|
||||
|
||||
"""Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json"""
|
||||
NDKS = [
|
||||
{
|
||||
"release": "r11",
|
||||
"revision": "11.0.2655954",
|
||||
"sha256": "59ab44f7ee6201df4381844736fdc456134c7f7660151003944a3017a0dcce97",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r11-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r11b",
|
||||
"revision": "11.1.2683735",
|
||||
"sha256": "51d429bfda8bbe038683ed7ae7acc03b39604b84711901b555fe18c698867e53",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r11b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r11c",
|
||||
"revision": "11.2.2725575",
|
||||
"sha256": "ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r12",
|
||||
"revision": "12.0.2931149",
|
||||
"sha256": "7876e3b99f3596a3215ecf4e9f152d24b82dfdf2bbe7d3a38c423ae6a3edee79",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r12-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r12b",
|
||||
"revision": "12.1.2977051",
|
||||
"sha256": "eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r13",
|
||||
"revision": "13.0.3315539",
|
||||
"sha256": "0a1dbd216386399e2979c17a48f65b962bf7ddc0c2311ef35d902b90c298c400",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r13-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r13b",
|
||||
"revision": "13.1.3345770",
|
||||
"sha256": "3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r14",
|
||||
"revision": "14.0.3770861",
|
||||
"sha256": "3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r14b",
|
||||
"revision": "14.1.3816874",
|
||||
"sha256": "0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r15",
|
||||
"revision": "15.0.4075724",
|
||||
"sha256": "078eb7d28c3fcf45841f5baf6e6582e7fd5b73d8e8c4e0101df490f51abd37b6",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r15-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r15b",
|
||||
"revision": "15.1.4119039",
|
||||
"sha256": "d1ce63f68cd806b5a992d4e5aa60defde131c243bf523cdfc5b67990ef0ee0d3",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r15c",
|
||||
"revision": "15.2.4203891",
|
||||
"sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r16",
|
||||
"revision": "16.0.4442984",
|
||||
"sha256": "a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r16b",
|
||||
"revision": "16.1.4479499",
|
||||
"sha256": "bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r17",
|
||||
"revision": "17.0.4754217",
|
||||
"sha256": "ba3d813b47de75bc32a2f3de087f72599c6cb36fdc9686b96f517f5492ff43ca",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r17-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r17b",
|
||||
"revision": "17.1.4828580",
|
||||
"sha256": "5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r17c",
|
||||
"revision": "17.2.4988734",
|
||||
"sha256": "3f541adbd0330a9205ba12697f6d04ec90752c53d6b622101a2a8a856e816589",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r18b",
|
||||
"revision": "18.1.5063045",
|
||||
"sha256": "4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r19",
|
||||
"revision": "19.0.5232133",
|
||||
"sha256": "c0a2425206191252197b97ea5fcc7eab9f693a576e69ef4773a9ed1690feed53",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r19b",
|
||||
"revision": "19.1.5304403",
|
||||
"sha256": "0fbb1645d0f1de4dde90a4ff79ca5ec4899c835e729d692f433fda501623257a",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r19c",
|
||||
"revision": "19.2.5345600",
|
||||
"sha256": "4c62514ec9c2309315fd84da6d52465651cdb68605058f231f1e480fcf2692e1",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r20",
|
||||
"revision": "20.0.5594570",
|
||||
"sha256": "57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r20-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r20b",
|
||||
"revision": "20.1.5948944",
|
||||
"sha256": "8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r21",
|
||||
"revision": "21.0.6113669",
|
||||
"sha256": "b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r21b",
|
||||
"revision": "21.1.6352462",
|
||||
"sha256": "0c7af5dd23c5d2564915194e71b1053578438ac992958904703161c7672cbed7",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r21b-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r21c",
|
||||
"revision": "21.2.6472646",
|
||||
"sha256": "214ebfcfa5108ba78f5b2cc8db4d575068f9c973ac7f27d2fa1987dfdb76c9e7",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r21c-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r21d",
|
||||
"revision": "21.3.6528147",
|
||||
"sha256": "dd6dc090b6e2580206c64bcee499bc16509a5d017c6952dcd2bed9072af67cbd",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r21e",
|
||||
"revision": "21.4.7075529",
|
||||
"sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r22",
|
||||
"revision": "22.0.7026061",
|
||||
"sha256": "d37fc69cd81e5660234a686e20adef39bc0244086e4d66525a40af771c020718",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r22-linux-x86_64.zip"
|
||||
},
|
||||
{
|
||||
"release": "r22b",
|
||||
"revision": "22.1.7171670",
|
||||
"sha256": "ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b",
|
||||
"url": "https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip"
|
||||
}
|
||||
]
|
||||
|
||||
@ -957,7 +957,6 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
|
||||
% repo_icon)
|
||||
os.makedirs(os.path.dirname(iconfilename), exist_ok=True)
|
||||
try:
|
||||
import qrcode
|
||||
qrcode.make(common.config['repo_url']).save(iconfilename)
|
||||
except Exception:
|
||||
exampleicon = os.path.join(common.get_examples_dir(),
|
||||
|
||||
@ -66,7 +66,6 @@ def main():
|
||||
help=_("Do not prompt for Android SDK path, just fail"))
|
||||
options = parser.parse_args()
|
||||
|
||||
aapt = None
|
||||
fdroiddir = os.getcwd()
|
||||
test_config = dict()
|
||||
examplesdir = common.get_examples_dir()
|
||||
@ -227,9 +226,6 @@ def main():
|
||||
msg = '\n'
|
||||
msg += _('Built repo based in "%s" with this config:') % fdroiddir
|
||||
msg += '\n\n Android SDK:\t\t\t' + config['sdk_path']
|
||||
if aapt:
|
||||
msg += '\n Android SDK Build Tools:\t' + os.path.dirname(aapt)
|
||||
msg += '\n Android NDK r12b (optional):\t$ANDROID_NDK'
|
||||
msg += '\n ' + _('Keystore for signing key:\t') + keystore
|
||||
if repo_keyalias is not None:
|
||||
msg += '\n Alias for key in store:\t' + repo_keyalias
|
||||
|
||||
@ -326,13 +326,7 @@ class Build(dict):
|
||||
return 'ant'
|
||||
|
||||
def ndk_path(self):
|
||||
version = self.ndk
|
||||
if not version:
|
||||
version = 'r12b' # falls back to latest
|
||||
paths = fdroidserver.common.config['ndk_paths']
|
||||
if version not in paths:
|
||||
return ''
|
||||
return paths[version]
|
||||
return fdroidserver.common.config['ndk_paths'].get(self.ndk, '')
|
||||
|
||||
|
||||
flagtypes = {
|
||||
@ -890,10 +884,6 @@ def write_yaml(mf, app):
|
||||
_yaml_bools_plus_lists.extend(_yaml_bools_false)
|
||||
_yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
|
||||
|
||||
def _class_as_dict_representer(dumper, data):
|
||||
'''Creates a YAML representation of a App/Build instance'''
|
||||
return dumper.represent_dict(data)
|
||||
|
||||
def _field_to_yaml(typ, value):
|
||||
if typ is TYPE_STRING:
|
||||
if value in _yaml_bools_plus_lists:
|
||||
|
||||
@ -36,7 +36,6 @@ import yaml
|
||||
import copy
|
||||
from datetime import datetime
|
||||
from argparse import ArgumentParser
|
||||
from base64 import urlsafe_b64encode
|
||||
try:
|
||||
from yaml import CSafeLoader as SafeLoader
|
||||
except ImportError:
|
||||
@ -590,30 +589,6 @@ def get_icon_bytes(apkzip, iconsrc):
|
||||
return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
|
||||
|
||||
|
||||
def sha256sum(filename):
|
||||
'''Calculate the sha256 of the given file'''
|
||||
sha = hashlib.sha256()
|
||||
with open(filename, 'rb') as f:
|
||||
while True:
|
||||
t = f.read(16384)
|
||||
if len(t) == 0:
|
||||
break
|
||||
sha.update(t)
|
||||
return sha.hexdigest()
|
||||
|
||||
|
||||
def sha256base64(filename):
|
||||
'''Calculate the sha256 of the given file as URL-safe base64'''
|
||||
hasher = hashlib.sha256()
|
||||
with open(filename, 'rb') as f:
|
||||
while True:
|
||||
t = f.read(16384)
|
||||
if len(t) == 0:
|
||||
break
|
||||
hasher.update(t)
|
||||
return urlsafe_b64encode(hasher.digest()).decode()
|
||||
|
||||
|
||||
def has_known_vulnerability(filename):
|
||||
"""checks for known vulnerabilities in the APK
|
||||
|
||||
@ -723,7 +698,7 @@ def insert_obbs(repodir, apps, apks):
|
||||
obbWarnDelete(f, _('OBB file has newer versionCode({integer}) than any APK:')
|
||||
.format(integer=str(versionCode)))
|
||||
continue
|
||||
obbsha256 = sha256sum(f)
|
||||
obbsha256 = common.sha256sum(f)
|
||||
obbs.append((packagename, versionCode, obbfile, obbsha256))
|
||||
|
||||
for apk in apks:
|
||||
@ -1267,7 +1242,7 @@ def insert_localized_app_metadata(apps):
|
||||
if not os.path.samefile(f, basepath):
|
||||
os.unlink(f)
|
||||
else:
|
||||
sha256 = sha256base64(f)
|
||||
sha256 = common.sha256base64(f)
|
||||
filename = base + '_' + sha256 + '.' + extension
|
||||
index_file = os.path.join(os.path.dirname(f), filename)
|
||||
if not os.path.exists(index_file):
|
||||
@ -1313,7 +1288,7 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
||||
raise FDroidException(_('{path} is zero size!')
|
||||
.format(path=filename))
|
||||
|
||||
shasum = sha256sum(filename)
|
||||
shasum = common.sha256sum(filename)
|
||||
usecache = False
|
||||
if name_utf8 in apkcache:
|
||||
repo_file = apkcache[name_utf8]
|
||||
@ -1378,7 +1353,7 @@ def scan_apk(apk_file):
|
||||
:return A dict containing APK metadata
|
||||
"""
|
||||
apk = {
|
||||
'hash': sha256sum(apk_file),
|
||||
'hash': common.sha256sum(apk_file),
|
||||
'hashType': 'sha256',
|
||||
'uses-permission': [],
|
||||
'uses-permission-sdk-23': [],
|
||||
@ -1613,7 +1588,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
|
||||
usecache = False
|
||||
if apkfilename in apkcache:
|
||||
apk = apkcache[apkfilename]
|
||||
if apk.get('hash') == sha256sum(apkfile):
|
||||
if apk.get('hash') == common.sha256sum(apkfile):
|
||||
logging.debug(_("Reading {apkfilename} from cache")
|
||||
.format(apkfilename=apkfilename))
|
||||
usecache = True
|
||||
@ -2067,7 +2042,6 @@ def create_metadata_from_template(apk):
|
||||
template: field sort order, empty field value, formatting, etc.
|
||||
'''
|
||||
|
||||
import yaml
|
||||
if os.path.exists('template.yml'):
|
||||
with open('template.yml') as f:
|
||||
metatxt = f.read()
|
||||
|
||||
@ -7,7 +7,6 @@ import os
|
||||
import re
|
||||
import requests
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
projectbasedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
@ -293,26 +293,6 @@ CACHE_FILES = [
|
||||
'0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'),
|
||||
('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
|
||||
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
|
||||
'ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip',
|
||||
'eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip',
|
||||
'3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip',
|
||||
'0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
|
||||
'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip',
|
||||
'bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip',
|
||||
'3f541adbd0330a9205ba12697f6d04ec90752c53d6b622101a2a8a856e816589'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip',
|
||||
'4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip',
|
||||
'4c62514ec9c2309315fd84da6d52465651cdb68605058f231f1e480fcf2692e1'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip',
|
||||
'8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip',
|
||||
'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip',
|
||||
|
||||
@ -12,10 +12,12 @@ import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
import yaml
|
||||
import zipfile
|
||||
from unittest import mock
|
||||
|
||||
localmodule = os.path.realpath(
|
||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
|
||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
||||
)
|
||||
print('localmodule: ' + localmodule)
|
||||
if localmodule not in sys.path:
|
||||
sys.path.insert(0, localmodule)
|
||||
@ -26,6 +28,14 @@ import fdroidserver.metadata
|
||||
import fdroidserver.scanner
|
||||
|
||||
|
||||
class FakeProcess:
|
||||
output = 'fake output'
|
||||
returncode = 0
|
||||
|
||||
def __init__(self, args, **kwargs):
|
||||
print('FakeFDroidPopen', args, kwargs)
|
||||
|
||||
|
||||
class BuildTest(unittest.TestCase):
|
||||
'''fdroidserver/build.py'''
|
||||
|
||||
@ -94,9 +104,106 @@ class BuildTest(unittest.TestCase):
|
||||
self.assertEqual(versionCode, vc)
|
||||
self.assertEqual(versionName, vn)
|
||||
|
||||
def test_build_local_ndk(self):
|
||||
"""Test if `fdroid build` detects installed NDKs and auto-installs when missing"""
|
||||
testdir = tempfile.mkdtemp(
|
||||
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
|
||||
)
|
||||
os.chdir(testdir)
|
||||
|
||||
config = {'ndk_paths': {}, 'sdk_path': tempfile.mkdtemp(prefix='android-sdk-')}
|
||||
fdroidserver.common.config = config
|
||||
fdroidserver.build.config = config
|
||||
fdroidserver.build.options = mock.Mock()
|
||||
fdroidserver.build.options.scan_binary = False
|
||||
fdroidserver.build.options.notarball = True
|
||||
fdroidserver.build.options.skipscan = True
|
||||
|
||||
app = fdroidserver.metadata.App()
|
||||
app.id = 'mocked.app.id'
|
||||
build = fdroidserver.metadata.Build()
|
||||
build.commit = '1.0'
|
||||
build.output = app.id + '.apk'
|
||||
build.versionCode = '1'
|
||||
build.versionName = '1.0'
|
||||
build.ndk = 'r21e' # aka 21.4.7075529
|
||||
vcs = mock.Mock()
|
||||
|
||||
def make_fake_apk(output, build):
|
||||
with open(build.output, 'w') as fp:
|
||||
fp.write('APK PLACEHOLDER')
|
||||
return output
|
||||
|
||||
def fake_download_file(_ignored, local_filename):
|
||||
_ignored # silence the linters
|
||||
with zipfile.ZipFile(local_filename, 'x') as zipfp:
|
||||
zipfp.writestr(
|
||||
'android-ndk-r21e/source.properties',
|
||||
'Pkg.Revision = 21.4.7075529\n',
|
||||
)
|
||||
|
||||
# use "as _ignored" just to make a pretty layout
|
||||
with mock.patch(
|
||||
'fdroidserver.common.replace_build_vars', wraps=make_fake_apk
|
||||
) as _ignored, mock.patch(
|
||||
'fdroidserver.common.get_native_code', return_value='x86'
|
||||
) as _ignored, mock.patch(
|
||||
'fdroidserver.common.get_apk_id',
|
||||
return_value=(app.id, build.versionCode, build.versionName),
|
||||
) as _ignored, mock.patch(
|
||||
'fdroidserver.common.is_apk_and_debuggable', return_value=False
|
||||
) as _ignored, mock.patch(
|
||||
'fdroidserver.common.sha256sum',
|
||||
return_value='ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
|
||||
) as _ignored, mock.patch(
|
||||
'fdroidserver.common.is_apk_and_debuggable', return_value=False
|
||||
) as _ignored, mock.patch(
|
||||
'fdroidserver.build.FDroidPopen', FakeProcess
|
||||
) as _ignored, mock.patch(
|
||||
'fdroidserver.net.download_file', wraps=fake_download_file
|
||||
) as _ignored:
|
||||
_ignored # silence the linters
|
||||
with self.assertRaises(
|
||||
fdroidserver.exception.FDroidException,
|
||||
msg="No NDK setup, `fdroid build` should fail with error",
|
||||
):
|
||||
fdroidserver.build.build_local(
|
||||
app,
|
||||
build,
|
||||
vcs,
|
||||
build_dir=testdir,
|
||||
output_dir=testdir,
|
||||
log_dir=None,
|
||||
srclib_dir=None,
|
||||
extlib_dir=None,
|
||||
tmp_dir=None,
|
||||
force=False,
|
||||
onserver=False,
|
||||
refresh=False,
|
||||
)
|
||||
# now run `fdroid buid --onserver`
|
||||
self.assertTrue('r21e' not in config['ndk_paths'])
|
||||
fdroidserver.build.build_local(
|
||||
app,
|
||||
build,
|
||||
vcs,
|
||||
build_dir=testdir,
|
||||
output_dir=testdir,
|
||||
log_dir=os.getcwd(),
|
||||
srclib_dir=None,
|
||||
extlib_dir=None,
|
||||
tmp_dir=None,
|
||||
force=False,
|
||||
onserver=True,
|
||||
refresh=False,
|
||||
)
|
||||
self.assertTrue(os.path.exists(config['ndk_paths']['r21e']))
|
||||
|
||||
def test_build_local_clean(self):
|
||||
"""Test if `fdroid build` cleans ant and gradle build products"""
|
||||
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
|
||||
testdir = tempfile.mkdtemp(
|
||||
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
|
||||
)
|
||||
os.chdir(testdir)
|
||||
|
||||
config = dict()
|
||||
@ -214,14 +321,9 @@ class BuildTest(unittest.TestCase):
|
||||
|
||||
def test_failed_verifies_are_not_in_unsigned(self):
|
||||
|
||||
class FakeProcess:
|
||||
output = 'fake output'
|
||||
returncode = 0
|
||||
|
||||
def __init__(self, args, **kwargs):
|
||||
print('FakeFDroidPopen', args, kwargs)
|
||||
|
||||
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
|
||||
testdir = tempfile.mkdtemp(
|
||||
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
|
||||
)
|
||||
os.chdir(testdir)
|
||||
sdk_path = os.path.join(testdir, 'android-sdk')
|
||||
self.create_fake_android_home(sdk_path)
|
||||
|
||||
140
tests/ndk-release-checksums.py
Executable file
140
tests/ndk-release-checksums.py
Executable file
@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import git
|
||||
import gitlab
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import requests
|
||||
import subprocess
|
||||
from colorama import Fore, Style
|
||||
|
||||
|
||||
checksums = None
|
||||
versions = dict()
|
||||
|
||||
while not checksums:
|
||||
r = requests.get(
|
||||
'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json'
|
||||
)
|
||||
if r.status_code == 200:
|
||||
checksums = r.json()
|
||||
|
||||
with open('fdroidserver/common.py') as fp:
|
||||
common_py = fp.read()
|
||||
|
||||
to_compile = re.search(r'\nNDKS = [^\]]+\]', common_py).group()
|
||||
if not to_compile:
|
||||
exit(1)
|
||||
code = compile(to_compile, '<string>', 'exec')
|
||||
config = {}
|
||||
exec(code, None, config) # nosec this is just a CI script
|
||||
|
||||
ndks = []
|
||||
errors = 0
|
||||
release = None
|
||||
revision = None
|
||||
for k, entries in checksums.items():
|
||||
if k.startswith('https://dl.google.com/android/repository/android-ndk'):
|
||||
m = re.search(r'-(r[1-9][0-9]?[a-z]?)-linux', k)
|
||||
if m:
|
||||
for entry in entries:
|
||||
if 'source.properties' in entry:
|
||||
n = re.search(
|
||||
r'[1-9][0-9]\.[0-9]\.[0-9]{7}', entry['source.properties']
|
||||
)
|
||||
if n:
|
||||
release = m.group(1)
|
||||
revision = n.group()
|
||||
ndks.append(
|
||||
{
|
||||
'url': k,
|
||||
'release': release,
|
||||
'revision': revision,
|
||||
'sha256': checksums[k][0]['sha256'],
|
||||
}
|
||||
)
|
||||
for d in config['NDKS']:
|
||||
if k == d['url']:
|
||||
sha256 = d['sha256']
|
||||
found = False
|
||||
for entry in entries:
|
||||
if sha256 == entry['sha256']:
|
||||
found = True
|
||||
if not found:
|
||||
print(
|
||||
Fore.RED
|
||||
+ (
|
||||
'ERROR: checksum mismatch: %s != %s'
|
||||
% (sha256, entry['sha256'])
|
||||
)
|
||||
+ Style.RESET_ALL
|
||||
)
|
||||
errors += 1
|
||||
|
||||
with open('fdroidserver/common.py', 'w') as fp:
|
||||
fp.write(
|
||||
common_py.replace(
|
||||
to_compile, '\nNDKS = ' + json.dumps(ndks, indent=4, sort_keys=True)
|
||||
)
|
||||
)
|
||||
|
||||
if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid':
|
||||
p = subprocess.run(['git', '--no-pager', 'diff'])
|
||||
print(p.stdout)
|
||||
exit(errors)
|
||||
|
||||
|
||||
# This only runs after commits are pushed to fdroid/fdroidserver
|
||||
git_repo = git.repo.Repo('.')
|
||||
modified = git_repo.git().ls_files(modified=True).split()
|
||||
if git_repo.is_dirty() and 'fdroidserver/common.py' in modified:
|
||||
branch = git_repo.create_head(os.path.basename(__file__), force=True)
|
||||
branch.checkout()
|
||||
git_repo.index.add(['fdroidserver/common.py'])
|
||||
author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org')
|
||||
git_repo.index.commit('Android NDK %s (%s)' % (release, revision), author=author)
|
||||
project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME')
|
||||
url = 'https://gitlab-ci-token:%s@%s/%s.git' % (
|
||||
os.getenv('PERSONAL_ACCESS_TOKEN'),
|
||||
os.getenv('CI_SERVER_HOST'),
|
||||
project_path,
|
||||
)
|
||||
remote_name = 'fdroid-bot'
|
||||
try:
|
||||
remote = git_repo.create_remote(remote_name, url)
|
||||
except git.exc.GitCommandError:
|
||||
remote = git.remote.Remote(git_repo, remote_name)
|
||||
remote.set_url(url)
|
||||
remote.push(force=True)
|
||||
git.remote.Remote.rm(git_repo, remote_name)
|
||||
|
||||
private_token = os.getenv('PERSONAL_ACCESS_TOKEN')
|
||||
if not private_token:
|
||||
print(
|
||||
Fore.RED
|
||||
+ 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!'
|
||||
+ Style.RESET_ALL
|
||||
)
|
||||
exit(1)
|
||||
gl = gitlab.Gitlab(
|
||||
os.getenv('CI_SERVER_URL'), api_version=4, private_token=private_token
|
||||
)
|
||||
project = gl.projects.get(project_path, lazy=True)
|
||||
description = (
|
||||
'see <https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json>'
|
||||
'\n\n<p><small>generated by <a href="%s/-/jobs/%s">GitLab CI Job #%s</a></small></p>'
|
||||
% (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID'))
|
||||
)
|
||||
mr = project.mergerequests.create(
|
||||
{
|
||||
'source_branch': branch.name,
|
||||
'target_project_id': 36527, # fdroid/fdroidserver
|
||||
'target_branch': 'master',
|
||||
'title': 'update to gradle v' + version,
|
||||
'description': description,
|
||||
'labels': ['fdroid-bot', 'buildserver'],
|
||||
'remove_source_branch': True,
|
||||
}
|
||||
)
|
||||
mr.save()
|
||||
Loading…
Reference in New Issue
Block a user