kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-08-15 10:50:09 +02:00
Code Activity

Merge branch 'modernize-ndk-handling' into 'master'

Browse Source 
buildserver: standardize SDK install location /opt/android-sdk

Closes #902

See merge request fdroid/fdroidserver!927
This commit is contained in:
Hans-Christoph Steiner 2021-05-28 07:35:38 +00:00
commit 22c83c9142
10 changed files with 181 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
buildserver/config.buildserver.yml
View File

@ -1,8 +1,4 @@
sdk_path: /opt/android-sdk
ndk_paths:
r10e: /opt/android-sdk/ndk/r10e
r21e: /opt/android-sdk/ndk/21.4.7075529
r22b: /opt/android-sdk/ndk/22.0.7026061
java_paths:
8: /usr/lib/jvm/java-8-openjdk-amd64

5
buildserver/provision-android-ndk
View File

@ -10,11 +10,6 @@ NDK_BASE=$1
test -e $NDK_BASE || mkdir -p $NDK_BASE
cd $NDK_BASE
if [ ! -e $NDK_BASE/r10e ]; then
7zr x /vagrant/cache/android-ndk-r10e-linux-x86_64.bin > /dev/null
mv android-ndk-r10e r10e
fi
for version in r21e r22b; do
if [ ! -e ${NDK_BASE}/${version} ]; then
unzip /vagrant/cache/android-ndk-${version}-linux-x86_64.zip > /dev/null

1
buildserver/provision-apt-get-install
View File

@ -96,7 +96,6 @@ packages="
openjdk-8-jre-headless
openjdk-8-jdk-headless
optipng
p7zip
pkg-config
python-gnupg
python-lxml

28
examples/config.yml
View File

@ -5,24 +5,20 @@
# Custom path to the Android SDK, defaults to $ANDROID_HOME
# sdk_path: $ANDROID_HOME
# Paths to various installed versions of the Android NDK. If a
# required version is missing in the buildserver VM, it will be
# automatically downloaded and installed into a temporary dir.
# Paths to installed versions of the Android NDK. This will be
# automatically filled out from well known sources like
# $ANDROID_HOME/ndk-bundle and $ANDROID_HOME/ndk/*. If a required
# version is missing in the buildserver VM, it will be automatically
# downloaded and installed into the standard $ANDROID_HOME/ndk/
# directory. Manually setting it here will override the auto-detected
# values. The keys can either be the "release" (e.g. r21e) or the
# "revision" (e.g. 21.4.7075529).
#
# ndk_paths:
# r10e: None
# r11c: None
# r12b: None
# r13b: None
# r14b: None
# r15c: None
# r16b: None
# r17c: None
# r18b: None
# r19c: None
# r20b: None
# r21e: None
# r22b: None
# r10e: $ANDROID_HOME/android-ndk-r10e
# r17: ""
# 21.4.7075529: ~/Android/Ndk
# r22b: null
# Directory to store downloaded tools in (i.e. gradle versions)
# By default, these are stored in ~/.cache/fdroidserver

6
fdroidserver/build.py
View File

@ -237,7 +237,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
try:
cmd_stdout = chan.makefile('rb', 1024)
output = bytes()
output += common.get_android_tools_version_log(build.ndk_path()).encode()
output += common.get_android_tools_version_log().encode()
while not chan.exit_status_ready():
line = cmd_stdout.readline()
if line:
@ -402,7 +402,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
log_path = os.path.join(log_dir,
common.get_toolsversion_logname(app, build))
with open(log_path, 'w') as f:
f.write(common.get_android_tools_version_log(build.ndk_path()))
f.write(common.get_android_tools_version_log())
else:
if build.sudo:
logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s\nThese commands were skipped because fdroid build is not running on a dedicated build server.'
@ -1088,7 +1088,7 @@ def main():
build_starttime = common.get_wiki_timestamp()
tools_version_log = ''
if not options.onserver:
tools_version_log = common.get_android_tools_version_log(build.ndk_path())
tools_version_log = common.get_android_tools_version_log()
common.write_running_status_json(status_output)
try:

86
fdroidserver/common.py
View File

@ -214,6 +214,14 @@ def _add_java_paths_to_config(pathlist, thisconfig):
def fill_config_defaults(thisconfig):
"""Fill in the global config dict with relevant defaults
For config values that have a path that can be expanded, e.g. an
env var or a ~/, this will store the original value using "_orig"
appended to the key name so that if the config gets written out,
it will preserve the original, unexpanded string.
"""
for k, v in default_config.items():
if k not in thisconfig:
thisconfig[k] = v
@ -281,6 +289,28 @@ def fill_config_defaults(thisconfig):
thisconfig[k][k2] = exp
thisconfig[k][k2 + '_orig'] = v
ndk_paths = thisconfig.get('ndk_paths', {})
ndk_bundle = os.path.join(thisconfig['sdk_path'], 'ndk-bundle')
if os.path.exists(ndk_bundle):
version = get_ndk_version(ndk_bundle)
if version not in ndk_paths:
ndk_paths[version] = ndk_bundle
ndk_dir = os.path.join(thisconfig['sdk_path'], 'ndk')
if os.path.exists(ndk_dir):
for ndk in glob.glob(os.path.join(ndk_dir, '*')):
version = get_ndk_version(ndk)
if version not in ndk_paths:
ndk_paths[version] = ndk
for k in list(ndk_paths.keys()):
if not re.match(r'r[1-9][0-9]*[a-z]?', k):
for ndkdict in NDKS:
if k == ndkdict.get('revision'):
ndk_paths[ndkdict['release']] = ndk_paths.pop(k)
break
def regsub_file(pattern, repl, path):
with open(path, 'rb') as f:
@ -3842,7 +3872,7 @@ def get_wiki_timestamp(timestamp=None):
return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
def get_android_tools_versions(ndk_path=None):
def get_android_tools_versions():
'''get a list of the versions of all installed Android SDK/NDK components'''
global config
@ -3850,11 +3880,9 @@ def get_android_tools_versions(ndk_path=None):
if sdk_path[-1] != '/':
sdk_path += '/'
components = []
if ndk_path:
ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
if os.path.isfile(ndk_release_txt):
with open(ndk_release_txt, 'r') as fp:
components.append((os.path.basename(ndk_path), fp.read()[:-1]))
for ndk_path in config.get('ndk_paths', []):
version = get_ndk_version(ndk_path)
components.append((os.path.basename(ndk_path), str(version)))
pattern = re.compile(r'^Pkg.Revision *= *(.+)', re.MULTILINE)
for root, dirs, files in os.walk(sdk_path):
@ -3868,10 +3896,10 @@ def get_android_tools_versions(ndk_path=None):
return components
def get_android_tools_version_log(ndk_path=None):
def get_android_tools_version_log():
'''get a list of the versions of all installed Android SDK/NDK components'''
log = '== Installed Android Tools ==\n\n'
components = get_android_tools_versions(ndk_path)
components = get_android_tools_versions()
for name, version in sorted(components):
log += '* ' + name + ' (' + version + ')\n'
@ -3985,12 +4013,23 @@ def sha256base64(filename):
def get_ndk_version(ndk_path):
"""Get the version info from the metadata in the NDK package
Since r11, the info is nice and easy to find in
sources.properties. Before, there was a kludgey format in
RELEASE.txt. This is only needed for r10e.
"""
source_properties = os.path.join(ndk_path, 'source.properties')
release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
if os.path.exists(source_properties):
with open(source_properties) as fp:
m = re.search(r'^Pkg.Revision *= *(.+)', fp.read(), flags=re.MULTILINE)
if m:
return m.group(1)
elif os.path.exists(release_txt):
with open(release_txt) as fp:
return fp.read().split('-')[0]
def auto_install_ndk(build):
@ -4021,6 +4060,32 @@ def auto_install_ndk(build):
ndk = build.get('ndk')
if not ndk:
return
if isinstance(ndk, str):
_install_ndk(ndk)
elif isinstance(ndk, list):
for n in ndk:
_install_ndk(n)
else:
BuildException(_('Invalid ndk: entry in build: "{ndk}"')
.format(ndk=str(ndk)))
def _install_ndk(ndk):
"""Install specified NDK if it is not already installed
Parameters
----------
ndk
The NDK version to install, either in "release" form (r21e) or
"revision" form (21.4.7075529).
"""
if re.match(r'[1-9][0-9.]+[0-9]', ndk):
for ndkdict in NDKS:
if ndk == ndkdict['revision']:
ndk = ndkdict['release']
break
ndk_path = config.get(ndk)
if ndk_path and os.path.isdir(ndk_path):
return
@ -4062,6 +4127,11 @@ def auto_install_ndk(build):
"""Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json"""
NDKS = [
{
"release": "r10e",
"sha256": "ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c",
"url": "https://dl.google.com/android/repository/android-ndk-r10e-linux-x86_64.zip"
},
{
"release": "r11",
"revision": "11.0.2655954",

6
fdroidserver/metadata.py
View File

@ -326,7 +326,11 @@ class Build(dict):
return 'ant'
def ndk_path(self):
return fdroidserver.common.config['ndk_paths'].get(self.ndk, '')
"""Returns the path to the first configured NDK or an empty string"""
ndk = self.ndk
if isinstance(ndk, list):
ndk = self.ndk[0]
return fdroidserver.common.config['ndk_paths'].get(ndk, '')
flagtypes = {

2
makebuildserver
View File

@ -291,8 +291,6 @@ CACHE_FILES = [
'dccda8aa069563c8ba2f6cdfd0777df0e34a5b4d15138ca8b9757e94f4e8a8cb'),
('https://services.gradle.org/distributions/gradle-7.0.2-bin.zip',
'0e46229820205440b48a5501122002842b82886e76af35f0f3a069243dca4b3c'),
('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
('https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip',
'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e'),
('https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip',

2
tests/build.TestCase
View File

@ -362,7 +362,7 @@ class BuildTest(unittest.TestCase):
with mock.patch(
'fdroidserver.common.force_exit', lambda *args: None
) as a, mock.patch(
'fdroidserver.common.get_android_tools_version_log', lambda s: 'fake'
'fdroidserver.common.get_android_tools_version_log', lambda: 'fake'
) as b, mock.patch(
'fdroidserver.common.FDroidPopen', FakeProcess
) as c, mock.patch(

82
tests/common.TestCase
View File

@ -1796,6 +1796,88 @@ class CommonTest(unittest.TestCase):
fdroidserver.common.metadata_find_developer_signing_files(appid, vc)
)
def test_auto_install_ndk(self):
"""Test all possible field data types for build.ndk"""
build = fdroidserver.metadata.Build()
none_entry = mock.Mock()
with mock.patch('fdroidserver.common._install_ndk', none_entry):
fdroidserver.common.auto_install_ndk(build)
none_entry.assert_not_called()
empty_list = mock.Mock()
build.ndk = []
with mock.patch('fdroidserver.common._install_ndk', empty_list):
fdroidserver.common.auto_install_ndk(build)
empty_list.assert_not_called()
release_entry = mock.Mock()
build.ndk = 'r21e'
with mock.patch('fdroidserver.common._install_ndk', release_entry):
fdroidserver.common.auto_install_ndk(build)
release_entry.assert_called_once_with('r21e')
revision_entry = mock.Mock()
build.ndk = '21.4.7075529'
with mock.patch('fdroidserver.common._install_ndk', revision_entry):
fdroidserver.common.auto_install_ndk(build)
revision_entry.assert_called_once_with('21.4.7075529')
list_entry = mock.Mock()
calls = []
build.ndk = ['r10e', '11.0.2655954', 'r12b', 'r21e']
for n in build.ndk:
calls.append(mock.call(n))
with mock.patch('fdroidserver.common._install_ndk', list_entry):
fdroidserver.common.auto_install_ndk(build)
list_entry.assert_has_calls(calls)
@unittest.skip("This test downloads and unzips a 1GB file.")
def test_install_ndk(self):
"""NDK r10e is a special case since its missing source.properties"""
sdk_path = tempfile.mkdtemp(
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
)
config = {'sdk_path': sdk_path}
fdroidserver.common.config = config
fdroidserver.common._install_ndk('r10e')
r10e = os.path.join(sdk_path, 'ndk', 'r10e')
self.assertEqual('r10e', fdroidserver.common.get_ndk_version(r10e))
fdroidserver.common.fill_config_defaults(config)
self.assertEqual({'r10e': r10e}, config['ndk_paths'])
def test_fill_config_defaults(self):
"""Test the auto-detection of NDKs installed in standard paths"""
sdk_path = tempfile.mkdtemp(
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
)
ndk_bundle = os.path.join(sdk_path, 'ndk-bundle')
os.makedirs(ndk_bundle)
with open(os.path.join(ndk_bundle, 'source.properties'), 'w') as fp:
fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 17.2.4988734\n')
config = {'sdk_path': sdk_path}
fdroidserver.common.fill_config_defaults(config)
self.assertEqual({'r17c': ndk_bundle}, config['ndk_paths'])
r21e = os.path.join(sdk_path, 'ndk', '21.4.7075529')
os.makedirs(r21e)
with open(os.path.join(r21e, 'source.properties'), 'w') as fp:
fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 21.4.7075529\n')
config = {'sdk_path': sdk_path}
fdroidserver.common.fill_config_defaults(config)
self.assertEqual({'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths'])
r10e = os.path.join(sdk_path, 'ndk', 'r10e')
os.makedirs(r10e)
with open(os.path.join(r10e, 'RELEASE.TXT'), 'w') as fp:
fp.write('r10e-rc4 (64-bit)\n')
config = {'sdk_path': sdk_path}
fdroidserver.common.fill_config_defaults(config)
self.assertEqual(
{'r10e': r10e, 'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths']
)
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))