kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-04 22:40:12 +01:00
Code Activity

Merge branch 'bug-fix-grab-bag' into 'master'

Browse Source 
bug fix grab bag; and, complete 'build' JSON status

Closes #344

See merge request fdroid/fdroidserver!875
This commit is contained in:
Hans-Christoph Steiner 2021-03-09 20:33:20 +00:00
commit 77c0c8bc5a
22 changed files with 162 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -41,13 +41,19 @@ makebuildserver.config.py
/tests/OBBMainTwoVersions.apk
/tests/archive/categories.txt
/tests/archive/icons*
/tests/archive/index.css
/tests/archive/index.html
/tests/archive/index.jar
/tests/archive/index.png
/tests/archive/index_unsigned.jar
/tests/archive/index.xml
/tests/archive/index-v1.jar
/tests/archive/index-v1.json
/tests/metadata/org.videolan.vlc/en-US/icon*.png
/tests/repo/index.css
/tests/repo/index.html
/tests/repo/index.jar
/tests/repo/index.png
/tests/repo/index_unsigned.jar
/tests/repo/index-v1.jar
/tests/repo/info.guardianproject.urzip/

7
MANIFEST.in
View File

@ -551,6 +551,13 @@ include tests/dump_internal_metadata_format.py
include tests/exception.TestCase
include tests/extra/manual-vmtools-test.py
include tests/funding-usernames.yaml
include tests/get_android_tools_versions/android-ndk/android-ndk-r21d/source.properties
include tests/get_android_tools_versions/android-ndk/r11c/source.properties
include tests/get_android_tools_versions/android-ndk/r17c/source.properties
include tests/get_android_tools_versions/android-sdk/patcher/v4/source.properties
include tests/get_android_tools_versions/android-sdk/platforms/android-30/source.properties
include tests/get_android_tools_versions/android-sdk/skiaparser/1/source.properties
include tests/get_android_tools_versions/android-sdk/tools/source.properties
include tests/getsig/getsig.java
include tests/getsig/make.sh
include tests/getsig/run.sh

2
examples/fdroid_fetchsrclibs.py
View File

@ -21,7 +21,7 @@ def main():
options = parser.parse_args()
common.options = options
pkgs = common.read_pkg_args(options.appid, True)
allapps = metadata.read_metadata(pkgs)
allapps = metadata.read_metadata(pkgs, check_vcs=True)
apps = common.read_app_args(options.appid, allapps, True)
srclib_dir = os.path.join('build', 'srclib')
os.makedirs(srclib_dir, exist_ok=True)

10
fdroidserver/build.py
View File

@ -83,7 +83,6 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
'cat /home/vagrant/buildserverid'],
cwd='builder').strip().decode()
status_output['buildserverid'] = buildserverid
logging.debug(_('Fetched buildserverid from VM: {buildserverid}')
.format(buildserverid=buildserverid))
except Exception as e:
@ -1013,7 +1012,7 @@ def main():
# Read all app and srclib metadata
pkgs = common.read_pkg_args(options.appid, True)
allapps = metadata.read_metadata(pkgs, options.refresh, sort_by_time=True)
allapps = metadata.read_metadata(pkgs, options.refresh, sort_by_time=True, check_vcs=True)
apps = common.read_app_args(options.appid, allapps, True)
for appid, app in list(apps.items()):
@ -1266,12 +1265,14 @@ def main():
.format(id=fdroidserverid))
if os.cpu_count():
txt += "* host processors: %d\n" % os.cpu_count()
status_output['hostOsCpuCount'] = os.cpu_count()
if os.path.isfile('/proc/meminfo') and os.access('/proc/meminfo', os.R_OK):
with open('/proc/meminfo') as fp:
for line in fp:
m = re.search(r'MemTotal:\s*([0-9].*)', line)
if m:
txt += "* host RAM: %s\n" % m.group(1)
status_output['hostProcMeminfoMemTotal'] = m.group(1)
break
fdroid_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
buildserver_config = os.path.join(fdroid_path, 'makebuildserver.config.py')
@ -1281,9 +1282,11 @@ def main():
m = re.search(r'cpus\s*=\s*([0-9].*)', line)
if m:
txt += "* guest processors: %s\n" % m.group(1)
status_output['guestVagrantVmCpus'] = m.group(1)
m = re.search(r'memory\s*=\s*([0-9].*)', line)
if m:
txt += "* guest RAM: %s MB\n" % m.group(1)
status_output['guestVagrantVmMemory'] = m.group(1)
txt += "* successful builds: %d\n" % len(build_succeeded)
txt += "* failed builds: %d\n" % len(failed_builds)
txt += "\n\n"
@ -1291,6 +1294,9 @@ def main():
newpage = site.Pages['build']
newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
if buildserverid:
status_output['buildserver'] = {'commitId': buildserverid}
if not options.onserver:
common.write_status_json(status_output)

2
fdroidserver/common.py
View File

@ -3821,7 +3821,7 @@ def get_android_tools_versions(ndk_path=None):
with open(ndk_release_txt, 'r') as fp:
components.append((os.path.basename(ndk_path), fp.read()[:-1]))
pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
pattern = re.compile(r'^Pkg.Revision *= *(.+)', re.MULTILINE)
for root, dirs, files in os.walk(sdk_path):
if 'source.properties' in files:
source_properties = os.path.join(root, 'source.properties')

31
fdroidserver/index.py
View File

@ -563,12 +563,25 @@ def make_v1(apps, packages, repodir, repodict, requestsdict, fdroid_signing_key_
json.dump(output, fp, default=_index_encoder_default)
if common.options.nosign:
_copy_to_local_copy_dir(repodir, index_file)
logging.debug(_('index-v1 must have a signature, use `fdroid signindex` to create it!'))
else:
signindex.config = common.config
signindex.sign_index_v1(repodir, json_name)
def _copy_to_local_copy_dir(repodir, f):
local_copy_dir = common.config.get('local_copy_dir', '')
if os.path.exists(local_copy_dir):
destdir = os.path.join(local_copy_dir, repodir)
if not os.path.exists(destdir):
os.mkdir(destdir)
shutil.copy2(f, destdir, follow_symlinks=False)
elif local_copy_dir:
raise FDroidException(_('"local_copy_dir" {path} does not exist!')
.format(path=local_copy_dir))
def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
"""Sorts the supplied list to ensure a deterministic sort order for
package entries in the index file. This sort-order also expresses
@ -577,7 +590,6 @@ def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
:param packages: list of packages which need to be sorted before but into index file.
"""
GROUP_DEV_SIGNED = 1
GROUP_FDROID_SIGNED = 2
GROUP_OTHER_SIGNED = 3
@ -585,22 +597,22 @@ def v1_sort_packages(packages, fdroid_signing_key_fingerprints):
def v1_sort_keys(package):
packageName = package.get('packageName', None)
sig = package.get('signer', None)
signer = package.get('signer', None)
dev_sig = common.metadata_find_developer_signature(packageName)
dev_signer = common.metadata_find_developer_signature(packageName)
group = GROUP_OTHER_SIGNED
if dev_sig and dev_sig == sig:
if dev_signer and dev_signer == signer:
group = GROUP_DEV_SIGNED
else:
fdroidsig = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
if fdroidsig and fdroidsig == sig:
fdroid_signer = fdroid_signing_key_fingerprints.get(packageName, {}).get('signer')
if fdroid_signer and fdroid_signer == signer:
group = GROUP_FDROID_SIGNED
versionCode = None
if package.get('versionCode', None):
versionCode = -int(package['versionCode'])
return(packageName, group, sig, versionCode)
return(packageName, group, signer, versionCode)
packages.sort(key=v1_sort_keys)
@ -705,11 +717,11 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
if name_from_apk is None:
name_from_apk = apk.get('name')
for versionCode, apksforver in apksbyversion.items():
fdroidsig = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
fdroid_signer = fdroid_signing_key_fingerprints.get(appid, {}).get('signer')
fdroid_signed_apk = None
name_match_apk = None
for x in apksforver:
if fdroidsig and x.get('signer', None) == fdroidsig:
if fdroid_signer and x.get('signer', None) == fdroid_signer:
fdroid_signed_apk = x
if common.apk_release_filename.match(x.get('apkName', '')):
name_match_apk = x
@ -926,6 +938,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
# Sign the index...
signed = os.path.join(repodir, 'index.jar')
if common.options.nosign:
_copy_to_local_copy_dir(repodir, os.path.join(repodir, jar_output))
# Remove old signed index if not signing
if os.path.exists(signed):
os.remove(signed)

4
fdroidserver/metadata.py
View File

@ -541,7 +541,7 @@ def read_srclibs():
srclibs[srclibname] = parse_yaml_srclib(metadatapath)
def read_metadata(appids={}, refresh=True, sort_by_time=False):
def read_metadata(appids={}, refresh=True, sort_by_time=False, check_vcs=False):
"""Return a list of App instances sorted newest first
This reads all of the metadata files in a 'data' repository, then
@ -597,7 +597,7 @@ def read_metadata(appids={}, refresh=True, sort_by_time=False):
if appid in apps:
_warn_or_exception(_("Found multiple metadata files for {appid}")
.format(appid=appid))
app = parse_metadata(metadatapath, appid in appids, refresh)
app = parse_metadata(metadatapath, check_vcs, refresh)
check_metadata(app)
apps[app.id] = app

3
fdroidserver/scanner.py
View File

@ -106,6 +106,9 @@ def scan_binary(apkfile):
logging.info("Scanning APK for known non-free classes.")
result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False)
problems = 0
if result.returncode != 0:
problems += 1
logging.error(result.output)
for suspect, regexp in CODE_SIGNATURES.items():
matches = regexp.findall(result.output)
if matches:

76
fdroidserver/update.py
View File

@ -1423,7 +1423,7 @@ def _get_apk_icons_src(apkfile, icon_name):
"""
icons_src = dict()
density_re = re.compile(r'^res/(.*)/{}\.(png|xml)$'.format(icon_name))
density_re = re.compile(r'^res/(.*)/{}\.png$'.format(icon_name))
with zipfile.ZipFile(apkfile) as zf:
for filename in zf.namelist():
m = density_re.match(filename)
@ -1686,7 +1686,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
.format(apkfilename=apkfile) + str(e))
# extract icons from APK zip file
iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])
iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
try:
empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
finally:
@ -1773,8 +1773,6 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
if m and m.group(4) == 'png':
density = screen_resolutions[m.group(2)]
pngs[m.group(3) + '/' + density] = m.group(0)
icon_type = None
empty_densities = []
for density in screen_densities:
if density not in apk['icons_src']:
@ -1782,7 +1780,7 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
continue
icon_src = apk['icons_src'][density]
icon_dir = get_icon_dir(repo_dir, density)
icon_type = '.png'
icon_dest = os.path.join(icon_dir, icon_filename)
# Extract the icon files per density
if icon_src.endswith('.xml'):
@ -1793,48 +1791,44 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
icon_src = name
if icon_src.endswith('.xml'):
empty_densities.append(density)
icon_type = '.xml'
icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
continue
try:
with open(icon_dest, 'wb') as f:
f.write(get_icon_bytes(apkzip, icon_src))
apk['icons'][density] = icon_filename + icon_type
apk['icons'][density] = icon_filename
except (zipfile.BadZipFile, ValueError, KeyError) as e:
logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
del apk['icons_src'][density]
empty_densities.append(density)
# '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"
if '-1' in apk['icons_src']:
if '-1' in apk['icons_src'] and not apk['icons_src']['-1'].endswith('.xml'):
icon_src = apk['icons_src']['-1']
icon_type = icon_src[-4:]
icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)
icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
with open(icon_path, 'wb') as f:
f.write(get_icon_bytes(apkzip, icon_src))
if icon_type == '.png':
im = None
try:
im = Image.open(icon_path)
dpi = px_to_dpi(im.size[0])
for density in screen_densities:
if density in apk['icons']:
break
if density == screen_densities[-1] or dpi >= int(density):
apk['icons'][density] = icon_filename + icon_type
shutil.move(icon_path,
os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
empty_densities.remove(density)
break
except Exception as e:
logging.warning(_("Failed reading {path}: {error}")
.format(path=icon_path, error=e))
finally:
if im and hasattr(im, 'close'):
im.close()
im = None
try:
im = Image.open(icon_path)
dpi = px_to_dpi(im.size[0])
for density in screen_densities:
if density in apk['icons']:
break
if density == screen_densities[-1] or dpi >= int(density):
apk['icons'][density] = icon_filename
shutil.move(icon_path,
os.path.join(get_icon_dir(repo_dir, density), icon_filename))
empty_densities.remove(density)
break
except Exception as e:
logging.warning(_("Failed reading {path}: {error}")
.format(path=icon_path, error=e))
finally:
if im and hasattr(im, 'close'):
im.close()
if apk['icons']:
apk['icon'] = icon_filename + icon_type
apk['icon'] = icon_filename
return empty_densities
@ -1849,7 +1843,6 @@ def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
:param repo_dir: The directory of the APK's repository
"""
icon_filename += '.png'
# First try resizing down to not lose quality
last_density = None
for density in screen_densities:
@ -2324,7 +2317,13 @@ def main():
options.use_date_from_apk)
cachechanged = cachechanged or fcachechanged
apks += files
appid_has_apks = set()
appid_has_repo_files = set()
for apk in apks:
if apk['apkName'].endswith('.apk'):
appid_has_apks.add(apk['packageName'])
else:
appid_has_repo_files.add(apk['packageName'])
if apk['packageName'] not in apps:
if options.create_metadata:
create_metadata_from_template(apk)
@ -2343,6 +2342,15 @@ def main():
else:
logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.'))
mismatch_errors = ''
for appid in appid_has_apks:
if appid in appid_has_repo_files:
appid_files = ', '.join(glob.glob(os.path.join('repo', appid + '_[0-9]*.*')))
mismatch_errors += (_('{appid} has both APKs and files: {files}')
.format(appid=appid, files=appid_files)) + '\n'
if mismatch_errors:
raise FDroidException(mismatch_errors)
# Scan the archive repo for apks as well
if len(repodirs) > 1:
archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)

15
tests/common.TestCase
View File

@ -1627,6 +1627,21 @@ class CommonTest(unittest.TestCase):
{'AutoName': testvalue, 'id': 'nope'}]:
self.assertEqual(testvalue, fdroidserver.common.get_app_display_name(app))
def test_get_android_tools_versions(self):
sdk_path = os.path.join(self.basedir, 'get_android_tools_versions')
fdroidserver.common.config = {'sdk_path': sdk_path}
components = fdroidserver.common.get_android_tools_versions()
expected = (
('android-ndk/android-ndk-r21d', '21.3.6528147'),
('android-ndk/r11c', '11.2.2725575'),
('android-ndk/r17c', '17.2.4988734'),
('android-sdk/patcher/v4', '1'),
('android-sdk/platforms/android-30', '3'),
('android-sdk/skiaparser/1', '6'),
('android-sdk/tools', '26.1.1'),
)
self.assertSequenceEqual(expected, sorted(components))
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))

2
tests/get_android_tools_versions/android-ndk/android-ndk-r21d/source.properties Normal file
View File

@ -0,0 +1,2 @@
Pkg.Desc = Android NDK
Pkg.Revision = 21.3.6528147

2
tests/get_android_tools_versions/android-ndk/r11c/source.properties Normal file
View File

@ -0,0 +1,2 @@
Pkg.Desc = Android NDK
Pkg.Revision = 11.2.2725575

2
tests/get_android_tools_versions/android-ndk/r17c/source.properties Normal file
View File

@ -0,0 +1,2 @@
Pkg.Desc = Android NDK
Pkg.Revision = 17.2.4988734

18
tests/get_android_tools_versions/android-sdk/patcher/v4/source.properties Normal file
View File

@ -0,0 +1,18 @@
#
# Copyright (C) 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
Pkg.Revision=1
Pkg.Path=patcher;v4
Pkg.Desc=SDK Patch Applier v4

9
tests/get_android_tools_versions/android-sdk/platforms/android-30/source.properties Normal file
View File

@ -0,0 +1,9 @@
Pkg.Desc=Android SDK Platform 11
Pkg.UserSrc=false
Platform.Version=11
Platform.CodeName=
Pkg.Revision=3
AndroidVersion.ApiLevel=30
Layoutlib.Api=15
Layoutlib.Revision=1
Platform.MinToolsRev=22

3
tests/get_android_tools_versions/android-sdk/skiaparser/1/source.properties Executable file
View File

@ -0,0 +1,3 @@
Pkg.Revision=6
Pkg.Path=skiaparser;1
Pkg.Desc=Layout Inspector image server for API 29-30

6
tests/get_android_tools_versions/android-sdk/tools/source.properties Normal file
View File

@ -0,0 +1,6 @@
Pkg.UserSrc=false
Pkg.Revision=26.1.1
Platform.MinPlatformToolsRev=20
Pkg.Dependencies=emulator
Pkg.Path=tools
Pkg.Desc=Android SDK Tools

9
tests/index.TestCase
View File

@ -395,6 +395,15 @@ class IndexTest(unittest.TestCase):
self.maxDiff = None
self.assertEquals(css, pretty_css)
def test_v1_sort_packages_with_invalid(self):
i = [{'packageName': 'org.smssecure.smssecure',
'apkName': 'smssecure-custom.fake',
'signer': None,
'versionCode': 11111}]
fdroidserver.index.v1_sort_packages(
i, fdroidserver.common.load_stats_fdroid_signing_key_fingerprints())
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))

1
tests/repo/index-v1.json
View File

@ -54,7 +54,6 @@
"name": "Caffeine Tile",
"summary": "Test app for extracting icons when an XML one is default",
"added": 1539129600000,
"icon": "info.zwanenburg.caffeinetile.4.xml",
"packageName": "info.zwanenburg.caffeinetile",
"lastUpdated": 1539129600000
},

1
tests/repo/index.xml
View File

@ -53,7 +53,6 @@ Features:
<lastupdated>2018-10-10</lastupdated>
<name>Caffeine Tile</name>
<summary>Test app for extracting icons when an XML one is default</summary>
<icon>info.zwanenburg.caffeinetile.4.xml</icon>
<desc>No description available</desc>
<license>Unknown</license>
<categories>Development</categories>

5
tests/run-tests
View File

@ -735,8 +735,9 @@ cd $REPOROOT
$fdroid init
$fdroid update --create-metadata --verbose
$fdroid readmeta
$fdroid deploy --local-copy-dir=/tmp/fdroid
$fdroid deploy --local-copy-dir=/tmp/fdroid --verbose
LOCAL_COPY_DIR=`create_test_dir`/fdroid
$fdroid deploy --local-copy-dir=$LOCAL_COPY_DIR
$fdroid deploy --local-copy-dir=$LOCAL_COPY_DIR --verbose
# now test the errors work
set +e

3
tests/update.TestCase
View File

@ -707,8 +707,7 @@ class UpdateTest(unittest.TestCase):
apk_info = fdroidserver.update.scan_apk('repo/info.zwanenburg.caffeinetile_4.apk')
self.assertEqual(apk_info.get('versionName'), '1.3')
self.assertEqual(apk_info['icons_src'], {'160': 'res/drawable/ic_coffee_on.xml',
'-1': 'res/drawable/ic_coffee_on.xml'})
self.assertEqual(apk_info['icons_src'], {})
apk_info = fdroidserver.update.scan_apk('repo/com.politedroid_6.apk')
self.assertEqual(apk_info.get('versionName'), '1.5')