kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-17 10:40:12 +02:00
Code Activity

Improved metadata handling

Browse Source 
Two main points: Firstly, there is no longer a random mapping between
app['key'] and a corresponding field in the metadata file - the key and
field are now the same. Secondly, more information (including comments)
is retrieved from the metadata, to facilitate being able to re-write it
which is necessary for various support utilities.
This commit is contained in:
Ciaran Gultnieks 2012-01-10 18:57:07 +00:00
parent c9c824f57d
commit c469f0feed
5 changed files with 157 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
build.py
View File

@ -74,13 +74,13 @@ if not os.path.isdir(build_dir):
for app in apps:
if app['disabled']:
if app['Disabled']:
print "Skipping %s: disabled" % app['id']
elif not app['builds']:
print "Skipping %s: no builds specified" % app['id']
if (app['disabled'] is None and app['repo'] != ''
and app['repotype'] != '' and (options.package is None or
if (app['Disabled'] is None and app['Repo'] != ''
and app['Repo Type'] != '' and (options.package is None or
options.package == app['id']) and len(app['builds']) > 0):
print "Processing " + app['id']
@ -88,7 +88,7 @@ for app in apps:
build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['repotype'], app['repo'], build_dir)
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
refreshed_source = False

2
checkmarket2.py
View File

@ -63,7 +63,7 @@ for app in apps:
print "...couldn't find version code"
elif not version:
print "...couldn't find version"
elif vercode == app['marketvercode'] and version == app['marketversion']:
elif vercode == app['Market Version Code'] and version == app['Market Version']:
print "...up to date"
else:
print '...updating to version:' + version + ' vercode:' + vercode

200
common.py
View File

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# common.py - part of the FDroid server tools
# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -245,15 +245,53 @@ class vcs_bzr(vcs):
raise VCSException("Bzr update failed")
# Get the type expected for a given metadata field.
def metafieldtype(name):
if name == 'Description':
return 'multiline'
if name == 'Requires Root':
return 'flag'
if name == 'Build Version':
return 'build'
if name == 'Use Built':
return 'obsolete'
return 'string'
# Parse metadata for a single application.
#
# 'metafile' - the filename to read. The package id for the application comes
# from this filename.
#
# Returns a dictionary containing all the details of the application. There are
# two major kinds of information in the dictionary. Keys beginning with capital
# letters correspond directory to identically named keys in the metadata file.
# Keys beginning with lower case letters are generated in one way or another,
# and are not found verbatim in the metadata.
#
# Known keys not originating from the metadata are:
#
# 'id' - the application's package ID
# 'builds' - a list of dictionaries containing build information
# for each defined build
# 'comments' - a list of comments from the metadata file. Each is
# a tuple of the form (field, comment) where field is
# the name of the field it preceded in the metadata
# file
# 'name' - the application's name, as displayed. This will
# always be None on return from this parser. The name
# is discovered from built APK files.
#
def parse_metadata(metafile, **kw):
def parse_buildline(value):
def parse_buildline(lines):
value = "".join(lines)
parts = [p.replace("\\,", ",")
for p in re.split(r"(?<!\\),", value)]
if len(parts) < 3:
raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
thisbuild = {}
thisbuild['origlines'] = lines
thisbuild['version'] = parts[0]
thisbuild['vercode'] = parts[1]
thisbuild['commit'] = parts[2]
@ -268,29 +306,37 @@ def parse_metadata(metafile, **kw):
thisinfo['id'] = metafile.name[9:-4]
if kw.get("verbose", False):
print "Reading metadata for " + thisinfo['id']
thisinfo['description'] = ''
thisinfo['name'] = None
thisinfo['summary'] = ''
thisinfo['license'] = 'Unknown'
thisinfo['web'] = ''
thisinfo['source'] = ''
thisinfo['tracker'] = ''
thisinfo['donate'] = None
thisinfo['disabled'] = None
thisinfo['antifeatures'] = None
thisinfo['marketversion'] = ''
thisinfo['marketvercode'] = '0'
thisinfo['repotype'] = ''
thisinfo['repo'] = ''
# Defaults for fields that come from metadata...
thisinfo['Description'] = ''
thisinfo['Summary'] = ''
thisinfo['License'] = 'Unknown'
thisinfo['Web Site'] = ''
thisinfo['Source Code'] = ''
thisinfo['Issue Tracker'] = ''
thisinfo['Donate'] = None
thisinfo['Disabled'] = None
thisinfo['AntiFeatures'] = None
thisinfo['Market Version'] = ''
thisinfo['Market Version Code'] = '0'
thisinfo['Repo Type'] = ''
thisinfo['Repo'] = ''
thisinfo['Requires Root'] = False
# General defaults...
thisinfo['builds'] = []
thisinfo['requiresroot'] = False
thisinfo['name'] = None
thisinfo['comments'] = []
mode = 0
buildline = []
buildlines = []
curcomments = []
for line in metafile:
line = line.rstrip('\r\n')
if line.startswith("#"):
continue
if mode == 0:
curcomments.append(line)
elif mode == 0:
if len(line) == 0:
continue
index = line.find(':')
@ -298,83 +344,77 @@ def parse_metadata(metafile, **kw):
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
field = line[:index]
value = line[index+1:]
if field == 'Description':
for comment in curcomments:
thisinfo['comments'].append((field,comment))
fieldtype = metafieldtype(field)
if fieldtype == 'multiline':
mode = 1
elif field == 'Name':
thisinfo['name'] = value
elif field == 'Summary':
thisinfo['summary'] = value
elif field == 'Source Code':
thisinfo['source'] = value
elif field == 'License':
thisinfo['license'] = value
elif field == 'Category':
thisinfo['category'] = value
elif field == 'Web Site':
thisinfo['web'] = value
elif field == 'Issue Tracker':
thisinfo['tracker'] = value
elif field == 'Donate':
thisinfo['donate'] = value
elif field == 'Disabled':
thisinfo['disabled'] = value
elif field == 'Use Built':
pass #Ignoring this - will be removed
elif field == 'AntiFeatures':
parts = value.split(",")
for part in parts:
if (part != "Ads" and
part != "Tracking" and
part != "NonFreeNet" and
part != "NonFreeDep" and
part != "NonFreeAdd"):
raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
+ metafile.name)
thisinfo['antifeatures'] = value
elif field == 'Market Version':
thisinfo['marketversion'] = value
elif field == 'Market Version Code':
thisinfo['marketvercode'] = value
elif field == 'Repo Type':
thisinfo['repotype'] = value
elif field == 'Repo':
thisinfo['repo'] = value
elif field == 'Build Version':
if len(value) > 0:
raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
elif fieldtype == 'string':
thisinfo[field] = value
elif fieldtype == 'flag':
if value == 'Yes':
thisinfo[field] = True
elif value == 'No':
thisinfo[field] = False
else:
raise MetaDataException("Expected Yes or No for " + field + " in " + metafile.name)
elif fieldtype == 'build':
if value.endswith("\\"):
mode = 2
buildline = [value[:-1]]
buildlines = [value[:-1]]
else:
thisinfo['builds'].append(parse_buildline(value))
elif field == "Requires Root":
if value == "Yes":
thisinfo['requiresroot'] = True
thisinfo['builds'].append(parse_buildline([value]))
elif fieldtype == 'obsolete':
pass # Just throw it away!
else:
raise MetaDataException("Unrecognised field " + field + " in " + metafile.name)
elif mode == 1: # multi-line description
raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
elif mode == 1: # Multiline field
if line == '.':
mode = 0
else:
if len(line) == 0:
thisinfo['description'] += '\n\n'
thisinfo[field] += '\n\n'
else:
if (not thisinfo['description'].endswith('\n') and
len(thisinfo['description']) > 0):
thisinfo['description'] += ' '
thisinfo['description'] += line
elif mode == 2: # line continuation
if (not thisinfo[field].endswith('\n') and
len(thisinfo[field]) > 0):
thisinfo[field] += ' '
thisinfo[field] += line
elif mode == 2: # Line continuation mode in Build Version
if line.endswith("\\"):
buildline.append(line[:-1])
buildlines.append(line[:-1])
else:
buildline.append(line)
buildlines.append(line)
thisinfo['builds'].append(
parse_buildline("".join(buildline)))
parse_buildline(buildlines))
mode = 0
if mode == 1:
raise MetaDataException("Description not terminated in " + metafile.name)
if len(thisinfo['description']) == 0:
thisinfo['description'] = 'No description available'
raise MetaDataException(field + " not terminated in " + metafile.name)
elif mode == 2:
raise MetaDataException("Unterminated continuation in " + metafile.name)
if len(thisinfo['Description']) == 0:
thisinfo['Description'] = 'No description available'
# Ensure all AntiFeatures are recognised...
if thisinfo['AntiFeatures']:
parts = thisinfo['AntiFeatures'].split(",")
for part in parts:
if (part != "Ads" and
part != "Tracking" and
part != "NonFreeNet" and
part != "NonFreeDep" and
part != "NonFreeAdd"):
raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
+ metafile.name)
return thisinfo
def read_metadata(verbose=False):
apps = []
for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):

4
scanner.py
View File

@ -55,7 +55,7 @@ for app in apps:
skip = False
if options.package and app['id'] != options.package:
skip = True
elif app['disabled']:
elif app['Disabled']:
print "Skipping %s: disabled" % app['id']
skip = True
elif not app['builds']:
@ -71,7 +71,7 @@ for app in apps:
build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['repotype'], app['repo'], build_dir)
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
refreshed_source = False

60
update.py
View File

@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
#
# update.py - part of the FDroid server tools
# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com
# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
@ -196,7 +196,7 @@ for app in apps:
if app['name'] is None:
app['name'] = app['id']
app['icon'] = ''
if app['disabled'] is None:
if app['Disabled'] is None:
print "WARNING: Application " + app['id'] + " has no packages"
else:
if app['name'] is None:
@ -283,14 +283,14 @@ apps_nopkg = 0
for app in apps:
if app['disabled'] is None:
if app['Disabled'] is None:
# Get a list of the apks for this app...
gotmarketver = False
apklist = []
for apk in apks:
if apk['id'] == app['id']:
if str(apk['versioncode']) == app['marketvercode']:
if str(apk['versioncode']) == app['Market Version Code']:
gotmarketver = True
apklist.append(apk)
@ -304,22 +304,22 @@ for app in apps:
addElement('id', app['id'], doc, apel)
addElement('name', app['name'], doc, apel)
addElement('summary', app['summary'], doc, apel)
addElement('summary', app['Summary'], doc, apel)
addElement('icon', app['icon'], doc, apel)
addElement('description', app['description'], doc, apel)
addElement('license', app['license'], doc, apel)
if 'category' in app:
addElement('category', app['category'], doc, apel)
addElement('web', app['web'], doc, apel)
addElement('source', app['source'], doc, apel)
addElement('tracker', app['tracker'], doc, apel)
if app['donate'] != None:
addElement('donate', app['donate'], doc, apel)
addElement('marketversion', app['marketversion'], doc, apel)
addElement('marketvercode', app['marketvercode'], doc, apel)
if not (app['antifeatures'] is None):
addElement('antifeatures', app['antifeatures'], doc, apel)
if app['requiresroot']:
addElement('description', app['Description'], doc, apel)
addElement('license', app['License'], doc, apel)
if 'Category' in app:
addElement('category', app['Category'], doc, apel)
addElement('web', app['Web Site'], doc, apel)
addElement('source', app['Source Code'], doc, apel)
addElement('tracker', app['Issue Tracker'], doc, apel)
if app['Donate'] != None:
addElement('donate', app['Donate'], doc, apel)
addElement('marketversion', app['Market Version'], doc, apel)
addElement('marketvercode', app['Market Version Code'], doc, apel)
if not (app['AntiFeatures'] is None):
addElement('antifeatures', app['AntiFeatures'], doc, apel)
if app['Requires Root']:
addElement('requirements', 'root', doc, apel)
# Sort the apk list into version order, just so the web site
@ -370,38 +370,38 @@ for app in apps:
if options.buildreport:
if len(app['builds']) == 0:
print ("WARNING: No builds defined for " + app['id'] +
" Source: " + app['source'])
" Source: " + app['Source Code'])
warnings += 1
else:
if app['marketvercode'] != '0':
if app['Market Version Code'] != '0':
gotbuild = False
for build in app['builds']:
if build['vercode'] == app['marketvercode']:
if build['vercode'] == app['Market Version Code']:
gotbuild = True
if not gotbuild:
print ("WARNING: No build data for market version of "
+ app['id'] + " (" + app['marketversion']
+ ") " + app['source'])
+ app['id'] + " (" + app['Market Version']
+ ") " + app['Source Code'])
warnings += 1
# If we don't have the market version, check if there is a build
# with a commit ID starting with '!' - this means we can't build it
# for some reason, and don't want hassling about it...
if not gotmarketver and app['marketvercode'] != '0':
if not gotmarketver and app['Market Version Code'] != '0':
for build in app['builds']:
if build['vercode'] == app['marketvercode']:
if build['vercode'] == app['Market Version Code']:
gotmarketver = True
# Output a message of harassment if we don't have the market version:
if not gotmarketver and app['marketvercode'] != '0':
addr = app['source']
print "WARNING: Don't have market version (" + app['marketversion'] + ") of " + app['name']
if not gotmarketver and app['Market Version Code'] != '0':
addr = app['Source Code']
print "WARNING: Don't have market version (" + app['Market Version'] + ") of " + app['name']
print " (" + app['id'] + ") " + addr
warnings += 1
if options.verbose:
# A bit of extra debug info, basically for diagnosing
# app developer mistakes:
print " Market vercode:" + app['marketvercode']
print " Market vercode:" + app['Market Version Code']
print " Got:"
for apk in apks:
if apk['id'] == app['id']: