mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
refactor update.py:main
This makes update.py:main a lot saner by removing a lot of the implicit assumptions between the different stages of generating the repository index. * mostly unify repo and archive processing, that means the archive is now actually getting the same treatment regarding i.e. fastlane data. Previously the archive didn't get considered at all here. * already filter the list of apps to include in a repo in update.py and give that prefiltered list to index. This makes sure we actually only copy fastlane/triple-t/etc. stuff for apps ending up in the index. This both, can save a lot of time if there are a lot of old /build dirs lying around and doesn't clutter /repo with things that aren't referenced from the index. Closes fdroid/fdroidserver#524
This commit is contained in:
parent
e66683720b
commit
d720c99ae5
@ -21,7 +21,6 @@
|
|||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
import collections
|
import collections
|
||||||
import copy
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
@ -41,7 +40,7 @@ from . import metadata
|
|||||||
from . import net
|
from . import net
|
||||||
from . import signindex
|
from . import signindex
|
||||||
from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
|
from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
|
||||||
from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
|
from fdroidserver.exception import FDroidException, VerificationException
|
||||||
|
|
||||||
|
|
||||||
def make(apps, apks, repodir, archive):
|
def make(apps, apks, repodir, archive):
|
||||||
@ -49,8 +48,9 @@ def make(apps, apks, repodir, archive):
|
|||||||
|
|
||||||
This requires properly initialized options and config objects.
|
This requires properly initialized options and config objects.
|
||||||
|
|
||||||
:param apps: fully populated apps list
|
:param apps: OrderedDict of apps to go into the index, each app should have
|
||||||
:param apks: full populated apks list
|
at least one associated apk
|
||||||
|
:param apks: list of apks to go into the index
|
||||||
:param repodir: the repo directory
|
:param repodir: the repo directory
|
||||||
:param archive: True if this is the archive repo, False if it's the
|
:param archive: True if this is the archive repo, False if it's the
|
||||||
main one.
|
main one.
|
||||||
@ -60,6 +60,12 @@ def make(apps, apks, repodir, archive):
|
|||||||
if not common.options.nosign:
|
if not common.options.nosign:
|
||||||
common.assert_config_keystore(common.config)
|
common.assert_config_keystore(common.config)
|
||||||
|
|
||||||
|
# Historically the index has been sorted by App Name, so we enforce this ordering here
|
||||||
|
sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper())
|
||||||
|
sortedapps = collections.OrderedDict()
|
||||||
|
for appid in sortedids:
|
||||||
|
sortedapps[appid] = apps[appid]
|
||||||
|
|
||||||
repodict = collections.OrderedDict()
|
repodict = collections.OrderedDict()
|
||||||
repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
|
repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
|
||||||
repodict['version'] = METADATA_VERSION
|
repodict['version'] = METADATA_VERSION
|
||||||
@ -100,24 +106,6 @@ def make(apps, apks, repodir, archive):
|
|||||||
if mirrors:
|
if mirrors:
|
||||||
repodict['mirrors'] = mirrors
|
repodict['mirrors'] = mirrors
|
||||||
|
|
||||||
# Historically the index has been sorted by App Name, so we enforce this ordering here
|
|
||||||
sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper())
|
|
||||||
|
|
||||||
appsWithPackages = collections.OrderedDict()
|
|
||||||
for packageName in sortedids:
|
|
||||||
app = apps[packageName]
|
|
||||||
if app['Disabled']:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# only include apps with packages
|
|
||||||
for apk in apks:
|
|
||||||
if apk['packageName'] == packageName:
|
|
||||||
newapp = copy.copy(app) # update wiki needs unmodified description
|
|
||||||
newapp['Description'] = metadata.description_html(app['Description'],
|
|
||||||
metadata.DescriptionResolver(apps))
|
|
||||||
appsWithPackages[packageName] = newapp
|
|
||||||
break
|
|
||||||
|
|
||||||
requestsdict = collections.OrderedDict()
|
requestsdict = collections.OrderedDict()
|
||||||
for command in ('install', 'uninstall'):
|
for command in ('install', 'uninstall'):
|
||||||
packageNames = []
|
packageNames = []
|
||||||
@ -133,9 +121,9 @@ def make(apps, apks, repodir, archive):
|
|||||||
|
|
||||||
fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
|
fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
|
||||||
|
|
||||||
make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
|
make_v0(sortedapps, apks, repodir, repodict, requestsdict,
|
||||||
fdroid_signing_key_fingerprints)
|
fdroid_signing_key_fingerprints)
|
||||||
make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
|
make_v1(sortedapps, apks, repodir, repodict, requestsdict,
|
||||||
fdroid_signing_key_fingerprints)
|
fdroid_signing_key_fingerprints)
|
||||||
|
|
||||||
|
|
||||||
|
@ -2166,6 +2166,71 @@ def create_metadata_from_template(apk):
|
|||||||
logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
|
logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
|
||||||
|
|
||||||
|
|
||||||
|
def read_names_from_apks(apps, apks):
|
||||||
|
"""This is a stripped down copy of apply_info_from_latest_apk that only parses app names"""
|
||||||
|
for appid, app in apps.items():
|
||||||
|
bestver = UNSET_VERSION_CODE
|
||||||
|
for apk in apks:
|
||||||
|
if apk['packageName'] == appid:
|
||||||
|
if apk['versionCode'] > bestver:
|
||||||
|
bestver = apk['versionCode']
|
||||||
|
bestapk = apk
|
||||||
|
|
||||||
|
if bestver == UNSET_VERSION_CODE:
|
||||||
|
if app.Name is None:
|
||||||
|
app.Name = app.AutoName or appid
|
||||||
|
app.icon = None
|
||||||
|
else:
|
||||||
|
if app.Name is None:
|
||||||
|
app.Name = bestapk['name']
|
||||||
|
|
||||||
|
|
||||||
|
def render_app_descriptions(apps, all_apps):
|
||||||
|
"""
|
||||||
|
Renders the app html description.
|
||||||
|
For resolving inter-app links it needs the full list of apps, even if they end up in
|
||||||
|
separate repos (i.e. archive or per app repos).
|
||||||
|
"""
|
||||||
|
for app in apps.values():
|
||||||
|
app['Description'] = metadata.description_html(app['Description'], metadata.DescriptionResolver(all_apps))
|
||||||
|
|
||||||
|
|
||||||
|
def get_apps_with_packages(apps, apks):
|
||||||
|
"""Returns a deepcopy of that subset apps that actually has any associated packages. Skips disabled apps."""
|
||||||
|
appsWithPackages = collections.OrderedDict()
|
||||||
|
for packageName in apps:
|
||||||
|
app = apps[packageName]
|
||||||
|
if app['Disabled']:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# only include apps with packages
|
||||||
|
for apk in apks:
|
||||||
|
if apk['packageName'] == packageName:
|
||||||
|
newapp = copy.copy(app)
|
||||||
|
appsWithPackages[packageName] = newapp
|
||||||
|
break
|
||||||
|
return appsWithPackages
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_apps(apps, apks, repodir):
|
||||||
|
"""Encapsulates all necessary preparation steps before we can build an index out of apps and apks.
|
||||||
|
|
||||||
|
:param apps: All apps as read from metadata
|
||||||
|
:param apks: list of apks that belong into repo, this gets modified in place
|
||||||
|
:param repodir: the target repository directory, metadata files will be copied here
|
||||||
|
:return: the relevant subset of apps (as a deepcopy)
|
||||||
|
"""
|
||||||
|
apps_with_packages = get_apps_with_packages(apps, apks)
|
||||||
|
apply_info_from_latest_apk(apps_with_packages, apks)
|
||||||
|
render_app_descriptions(apps_with_packages, apps)
|
||||||
|
insert_funding_yml_donation_links(apps)
|
||||||
|
copy_triple_t_store_metadata(apps_with_packages, repodir)
|
||||||
|
insert_obbs(repodir, apps_with_packages, apks)
|
||||||
|
translate_per_build_anti_features(apps_with_packages, apks)
|
||||||
|
insert_localized_app_metadata(apps_with_packages, repodir)
|
||||||
|
return apps_with_packages
|
||||||
|
|
||||||
|
|
||||||
config = None
|
config = None
|
||||||
options = None
|
options = None
|
||||||
start_timestamp = time.gmtime()
|
start_timestamp = time.gmtime()
|
||||||
@ -2302,12 +2367,6 @@ def main():
|
|||||||
else:
|
else:
|
||||||
logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.'))
|
logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.'))
|
||||||
|
|
||||||
insert_funding_yml_donation_links(apps)
|
|
||||||
copy_triple_t_store_metadata(apps, repodirs[0])
|
|
||||||
insert_obbs(repodirs[0], apps, apks)
|
|
||||||
insert_localized_app_metadata(apps, repodirs[0])
|
|
||||||
translate_per_build_anti_features(apps, apks)
|
|
||||||
|
|
||||||
# Scan the archive repo for apks as well
|
# Scan the archive repo for apks as well
|
||||||
if len(repodirs) > 1:
|
if len(repodirs) > 1:
|
||||||
archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
|
archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
|
||||||
@ -2316,8 +2375,18 @@ def main():
|
|||||||
else:
|
else:
|
||||||
archapks = []
|
archapks = []
|
||||||
|
|
||||||
# Apply information from latest apks to the application and update dates
|
# We need app.Name populated for all apps regardless of which repo they end up in
|
||||||
apply_info_from_latest_apk(apps, apks + archapks)
|
# for the old-style inter-app links, so let's do it before we do anything else.
|
||||||
|
# This will be done again (as part of apply_info_from_latest_apk) for repo and archive
|
||||||
|
# separately later on, but it's fairly cheap anyway.
|
||||||
|
read_names_from_apks(apps, apks + archapks)
|
||||||
|
|
||||||
|
if len(repodirs) > 1:
|
||||||
|
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
|
||||||
|
archived_apps = prepare_apps(apps, archapks, repodirs[1])
|
||||||
|
index.make(archived_apps, archapks, repodirs[1], True)
|
||||||
|
|
||||||
|
repoapps = prepare_apps(apps, apks, repodirs[0])
|
||||||
|
|
||||||
# APKs are placed into multiple repos based on the app package, providing
|
# APKs are placed into multiple repos based on the app package, providing
|
||||||
# per-app subscription feeds for nightly builds and things like it
|
# per-app subscription feeds for nightly builds and things like it
|
||||||
@ -2333,20 +2402,10 @@ def main():
|
|||||||
logging.info(_('Skipping index generation for {appid}').format(appid=appid))
|
logging.info(_('Skipping index generation for {appid}').format(appid=appid))
|
||||||
return
|
return
|
||||||
|
|
||||||
if len(repodirs) > 1:
|
|
||||||
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
|
|
||||||
|
|
||||||
# Make the index for the main repo...
|
# Make the index for the main repo...
|
||||||
index.make(apps, apks, repodirs[0], False)
|
index.make(repoapps, apks, repodirs[0], False)
|
||||||
make_categories_txt(repodirs[0], categories)
|
make_categories_txt(repodirs[0], categories)
|
||||||
|
|
||||||
# If there's an archive repo, make the index for it. We already scanned it
|
|
||||||
# earlier on.
|
|
||||||
if len(repodirs) > 1:
|
|
||||||
archived_apps = copy.deepcopy(apps)
|
|
||||||
apply_info_from_latest_apk(archived_apps, archapks)
|
|
||||||
index.make(archived_apps, archapks, repodirs[1], True)
|
|
||||||
|
|
||||||
git_remote = config.get('binary_transparency_remote')
|
git_remote = config.get('binary_transparency_remote')
|
||||||
if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
|
if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
|
||||||
from . import btlog
|
from . import btlog
|
||||||
|
Loading…
Reference in New Issue
Block a user