mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-13 02:30:11 +01:00
First metadata checks rewrite; New metadata.py module
This commit is contained in:
Daniel Martí
parent
b6f5c15a7c
commit
fb38eb6b8c
@ -29,7 +29,7 @@ import time
|
||||
import json
|
||||
from optparse import OptionParser
|
||||
|
||||
import common
|
||||
import common, metadata
|
||||
from common import BuildException, VCSException, FDroidPopen
|
||||
|
||||
def get_builder_vm_id():
|
||||
@ -816,7 +816,7 @@ def main():
|
||||
sys.exit(1)
|
||||
|
||||
# Get all apps...
|
||||
apps = common.read_metadata(xref=not options.onserver)
|
||||
apps = metadata.read_metadata(xref=not options.onserver)
|
||||
|
||||
log_dir = 'logs'
|
||||
if not os.path.isdir(log_dir):
|
||||
|
||||
@ -28,7 +28,7 @@ from optparse import OptionParser
|
||||
import traceback
|
||||
import HTMLParser
|
||||
from distutils.version import LooseVersion
|
||||
import common
|
||||
import common, metadata
|
||||
from common import BuildException
|
||||
from common import VCSException
|
||||
|
||||
@ -295,7 +295,7 @@ def main():
|
||||
config = common.read_config(options)
|
||||
|
||||
# Get all apps...
|
||||
apps = common.read_metadata(options.verbose)
|
||||
apps = metadata.read_metadata(options.verbose)
|
||||
|
||||
# Filter apps according to command-line options
|
||||
if options.package:
|
||||
@ -453,7 +453,7 @@ def main():
|
||||
|
||||
if writeit:
|
||||
metafile = os.path.join('metadata', app['id'] + '.txt')
|
||||
common.write_metadata(metafile, app)
|
||||
metadata.write_metadata(metafile, app)
|
||||
if options.commit and logmsg:
|
||||
print "Commiting update for " + metafile
|
||||
gitcmd = ["git", "commit", "-m",
|
||||
|
||||
@ -23,20 +23,15 @@ import stat
|
||||
import subprocess
|
||||
import time
|
||||
import operator
|
||||
import cgi
|
||||
import Queue
|
||||
import threading
|
||||
import magic
|
||||
|
||||
import metadata
|
||||
|
||||
config = None
|
||||
options = None
|
||||
|
||||
# These can only contain 'yes' or 'no'
|
||||
bool_keys = (
|
||||
'submodules', 'oldsdkloc',
|
||||
'forceversion', 'forcevercode',
|
||||
'fixtrans', 'fixapos', 'novcheck')
|
||||
|
||||
def read_config(opts, config_file='config.py'):
|
||||
"""Read the repository config
|
||||
|
||||
@ -52,7 +47,7 @@ def read_config(opts, config_file='config.py'):
|
||||
sys.exit(2)
|
||||
st = os.stat(config_file)
|
||||
if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
|
||||
print("WARNING: unsafe permissions on config.py (should be 0600)!")
|
||||
print "WARNING: unsafe permissions on config.py (should be 0600)!"
|
||||
|
||||
options = opts
|
||||
if not hasattr(options, 'verbose'):
|
||||
@ -73,6 +68,11 @@ def read_config(opts, config_file='config.py'):
|
||||
execfile(config_file, config)
|
||||
return config
|
||||
|
||||
def getapkname(app, build):
|
||||
return "%s_%s.apk" % (app['id'], build['vercode'])
|
||||
|
||||
def getsrcname(app, build):
|
||||
return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
|
||||
|
||||
def getvcs(vcstype, remote, local):
|
||||
if vcstype == 'git':
|
||||
@ -95,7 +95,7 @@ def getsrclibvcs(name):
|
||||
srclib_path = os.path.join('srclibs', name + ".txt")
|
||||
if not os.path.exists(srclib_path):
|
||||
raise VCSException("Missing srclib " + name)
|
||||
return parse_srclib(srclib_path)['Repo Type']
|
||||
return metadata.parse_srclib(srclib_path)['Repo Type']
|
||||
|
||||
class vcs:
|
||||
def __init__(self, remote, local):
|
||||
@ -457,625 +457,6 @@ class vcs_bzr(vcs):
|
||||
return [tag.split(' ')[0].strip() for tag in
|
||||
p.communicate()[0].splitlines()]
|
||||
|
||||
|
||||
# Get the type expected for a given metadata field.
|
||||
def metafieldtype(name):
|
||||
if name in ['Description', 'Maintainer Notes']:
|
||||
return 'multiline'
|
||||
if name == 'Requires Root':
|
||||
return 'flag'
|
||||
if name == 'Build Version':
|
||||
return 'build'
|
||||
if name == 'Build':
|
||||
return 'buildv2'
|
||||
if name == 'Use Built':
|
||||
return 'obsolete'
|
||||
return 'string'
|
||||
|
||||
|
||||
# Parse metadata for a single application.
|
||||
#
|
||||
# 'metafile' - the filename to read. The package id for the application comes
|
||||
# from this filename. Pass None to get a blank entry.
|
||||
#
|
||||
# Returns a dictionary containing all the details of the application. There are
|
||||
# two major kinds of information in the dictionary. Keys beginning with capital
|
||||
# letters correspond directory to identically named keys in the metadata file.
|
||||
# Keys beginning with lower case letters are generated in one way or another,
|
||||
# and are not found verbatim in the metadata.
|
||||
#
|
||||
# Known keys not originating from the metadata are:
|
||||
#
|
||||
# 'id' - the application's package ID
|
||||
# 'builds' - a list of dictionaries containing build information
|
||||
# for each defined build
|
||||
# 'comments' - a list of comments from the metadata file. Each is
|
||||
# a tuple of the form (field, comment) where field is
|
||||
# the name of the field it preceded in the metadata
|
||||
# file. Where field is None, the comment goes at the
|
||||
# end of the file. Alternatively, 'build:version' is
|
||||
# for a comment before a particular build version.
|
||||
# 'descriptionlines' - original lines of description as formatted in the
|
||||
# metadata file.
|
||||
#
|
||||
def parse_metadata(metafile):
|
||||
|
||||
def parse_buildline(lines):
|
||||
value = "".join(lines)
|
||||
parts = [p.replace("\\,", ",")
|
||||
for p in re.split(r"(?<!\\),", value)]
|
||||
if len(parts) < 3:
|
||||
raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
|
||||
thisbuild = {}
|
||||
thisbuild['origlines'] = lines
|
||||
thisbuild['version'] = parts[0]
|
||||
thisbuild['vercode'] = parts[1]
|
||||
try:
|
||||
int(thisbuild['vercode'])
|
||||
except:
|
||||
raise MetaDataException("Invalid version code for build in " + metafile.name)
|
||||
if parts[2].startswith('!'):
|
||||
# For backwards compatibility, handle old-style disabling,
|
||||
# including attempting to extract the commit from the message
|
||||
thisbuild['disable'] = parts[2][1:]
|
||||
commit = 'unknown - see disabled'
|
||||
index = parts[2].rfind('at ')
|
||||
if index != -1:
|
||||
commit = parts[2][index+3:]
|
||||
if commit.endswith(')'):
|
||||
commit = commit[:-1]
|
||||
thisbuild['commit'] = commit
|
||||
else:
|
||||
thisbuild['commit'] = parts[2]
|
||||
for p in parts[3:]:
|
||||
pk, pv = p.split('=', 1)
|
||||
thisbuild[pk.strip()] = pv
|
||||
|
||||
return thisbuild
|
||||
|
||||
def add_comments(key):
|
||||
if not curcomments:
|
||||
return
|
||||
for comment in curcomments:
|
||||
thisinfo['comments'].append((key, comment))
|
||||
del curcomments[:]
|
||||
|
||||
|
||||
thisinfo = {}
|
||||
if metafile:
|
||||
if not isinstance(metafile, file):
|
||||
metafile = open(metafile, "r")
|
||||
thisinfo['id'] = metafile.name[9:-4]
|
||||
else:
|
||||
thisinfo['id'] = None
|
||||
|
||||
# Defaults for fields that come from metadata...
|
||||
thisinfo['Name'] = None
|
||||
thisinfo['Auto Name'] = ''
|
||||
thisinfo['Categories'] = 'None'
|
||||
thisinfo['Description'] = []
|
||||
thisinfo['Summary'] = ''
|
||||
thisinfo['License'] = 'Unknown'
|
||||
thisinfo['Web Site'] = ''
|
||||
thisinfo['Source Code'] = ''
|
||||
thisinfo['Issue Tracker'] = ''
|
||||
thisinfo['Donate'] = None
|
||||
thisinfo['FlattrID'] = None
|
||||
thisinfo['Bitcoin'] = None
|
||||
thisinfo['Litecoin'] = None
|
||||
thisinfo['Disabled'] = None
|
||||
thisinfo['AntiFeatures'] = None
|
||||
thisinfo['Archive Policy'] = None
|
||||
thisinfo['Update Check Mode'] = 'None'
|
||||
thisinfo['Vercode Operation'] = None
|
||||
thisinfo['Auto Update Mode'] = 'None'
|
||||
thisinfo['Current Version'] = ''
|
||||
thisinfo['Current Version Code'] = '0'
|
||||
thisinfo['Repo Type'] = ''
|
||||
thisinfo['Repo'] = ''
|
||||
thisinfo['Requires Root'] = False
|
||||
thisinfo['No Source Since'] = ''
|
||||
|
||||
# General defaults...
|
||||
thisinfo['builds'] = []
|
||||
thisinfo['comments'] = []
|
||||
|
||||
if metafile is None:
|
||||
return thisinfo
|
||||
|
||||
mode = 0
|
||||
buildlines = []
|
||||
curcomments = []
|
||||
curbuild = None
|
||||
|
||||
for line in metafile:
|
||||
line = line.rstrip('\r\n')
|
||||
if mode == 3:
|
||||
if not any(line.startswith(s) for s in (' ', '\t')):
|
||||
if 'commit' not in curbuild and 'disable' not in curbuild:
|
||||
raise MetaDataException("No commit specified for {0} in {1}".format(
|
||||
curbuild['version'], metafile.name))
|
||||
thisinfo['builds'].append(curbuild)
|
||||
add_comments('build:' + curbuild['version'])
|
||||
mode = 0
|
||||
else:
|
||||
if line.endswith('\\'):
|
||||
buildlines.append(line[:-1].lstrip())
|
||||
else:
|
||||
buildlines.append(line.lstrip())
|
||||
bl = ''.join(buildlines)
|
||||
bv = bl.split('=', 1)
|
||||
if len(bv) != 2:
|
||||
raise MetaDataException("Invalid build flag at {0} in {1}".
|
||||
format(buildlines[0], metafile.name))
|
||||
name, val = bv
|
||||
if name in curbuild:
|
||||
raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
|
||||
format(name, curbuild['version'], metafile.name))
|
||||
curbuild[name] = val.lstrip()
|
||||
buildlines = []
|
||||
|
||||
if mode == 0:
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
curcomments.append(line)
|
||||
continue
|
||||
index = line.find(':')
|
||||
if index == -1:
|
||||
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
|
||||
field = line[:index]
|
||||
value = line[index+1:]
|
||||
|
||||
# Translate obsolete fields...
|
||||
if field == 'Market Version':
|
||||
field = 'Current Version'
|
||||
if field == 'Market Version Code':
|
||||
field = 'Current Version Code'
|
||||
|
||||
fieldtype = metafieldtype(field)
|
||||
if fieldtype not in ['build', 'buildv2']:
|
||||
add_comments(field)
|
||||
if fieldtype == 'multiline':
|
||||
mode = 1
|
||||
thisinfo[field] = []
|
||||
if value:
|
||||
raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
|
||||
elif fieldtype == 'string':
|
||||
if field == 'Category' and thisinfo['Categories'] == 'None':
|
||||
thisinfo['Categories'] = value.replace(';',',')
|
||||
thisinfo[field] = value
|
||||
elif fieldtype == 'flag':
|
||||
if value == 'Yes':
|
||||
thisinfo[field] = True
|
||||
elif value == 'No':
|
||||
thisinfo[field] = False
|
||||
else:
|
||||
raise MetaDataException("Expected Yes or No for " + field + " in " + metafile.name)
|
||||
elif fieldtype == 'build':
|
||||
if value.endswith("\\"):
|
||||
mode = 2
|
||||
buildlines = [value[:-1]]
|
||||
else:
|
||||
thisinfo['builds'].append(parse_buildline([value]))
|
||||
add_comments('build:' + thisinfo['builds'][-1]['version'])
|
||||
elif fieldtype == 'buildv2':
|
||||
curbuild = {}
|
||||
vv = value.split(',')
|
||||
if len(vv) != 2:
|
||||
raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
|
||||
format(value, metafile.name))
|
||||
curbuild['version'] = vv[0]
|
||||
curbuild['vercode'] = vv[1]
|
||||
try:
|
||||
int(curbuild['vercode'])
|
||||
except:
|
||||
raise MetaDataException("Invalid version code for build in " + metafile.name)
|
||||
buildlines = []
|
||||
mode = 3
|
||||
elif fieldtype == 'obsolete':
|
||||
pass # Just throw it away!
|
||||
else:
|
||||
raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
|
||||
elif mode == 1: # Multiline field
|
||||
if line == '.':
|
||||
mode = 0
|
||||
else:
|
||||
thisinfo[field].append(line)
|
||||
elif mode == 2: # Line continuation mode in Build Version
|
||||
if line.endswith("\\"):
|
||||
buildlines.append(line[:-1])
|
||||
else:
|
||||
buildlines.append(line)
|
||||
thisinfo['builds'].append(
|
||||
parse_buildline(buildlines))
|
||||
add_comments('build:' + thisinfo['builds'][-1]['version'])
|
||||
mode = 0
|
||||
add_comments(None)
|
||||
|
||||
for key in bool_keys:
|
||||
for build in thisinfo['builds']:
|
||||
if key not in build:
|
||||
build[key] = False
|
||||
continue
|
||||
if build[key] == 'yes':
|
||||
build[key] = True
|
||||
elif build[key] == 'no':
|
||||
build[key] = False
|
||||
else:
|
||||
raise MetaDataException("Invalid value %s assigned to boolean build flag %s"
|
||||
% (build[key], key))
|
||||
|
||||
# Mode at end of file should always be 0...
|
||||
if mode == 1:
|
||||
raise MetaDataException(field + " not terminated in " + metafile.name)
|
||||
elif mode == 2:
|
||||
raise MetaDataException("Unterminated continuation in " + metafile.name)
|
||||
elif mode == 3:
|
||||
raise MetaDataException("Unterminated build in " + metafile.name)
|
||||
|
||||
if not thisinfo['Description']:
|
||||
thisinfo['Description'].append('No description available')
|
||||
|
||||
# Validate archive policy...
|
||||
if thisinfo['Archive Policy']:
|
||||
if not thisinfo['Archive Policy'].endswith(' versions'):
|
||||
raise MetaDataException("Invalid archive policy")
|
||||
try:
|
||||
versions = int(thisinfo['Archive Policy'][:-9])
|
||||
if versions < 1 or versions > 20:
|
||||
raise MetaDataException("Silly number of versions for archive policy")
|
||||
except:
|
||||
raise MetaDataException("Incomprehensible number of versions for archive policy")
|
||||
|
||||
# Ensure all AntiFeatures are recognised...
|
||||
if thisinfo['AntiFeatures']:
|
||||
parts = thisinfo['AntiFeatures'].split(",")
|
||||
for part in parts:
|
||||
if (part != "Ads" and
|
||||
part != "Tracking" and
|
||||
part != "NonFreeNet" and
|
||||
part != "NonFreeDep" and
|
||||
part != "NonFreeAdd"):
|
||||
raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
|
||||
+ metafile.name)
|
||||
|
||||
return thisinfo
|
||||
|
||||
def getvercode(build):
|
||||
return "%s" % (build['vercode'])
|
||||
|
||||
def getapkname(app, build):
|
||||
return "%s_%s.apk" % (app['id'], getvercode(build))
|
||||
|
||||
def getsrcname(app, build):
|
||||
return "%s_%s_src.tar.gz" % (app['id'], getvercode(build))
|
||||
|
||||
# Write a metadata file.
|
||||
#
|
||||
# 'dest' - The path to the output file
|
||||
# 'app' - The app data
|
||||
def write_metadata(dest, app):
|
||||
|
||||
def writecomments(key):
|
||||
written = 0
|
||||
for pf, comment in app['comments']:
|
||||
if pf == key:
|
||||
mf.write(comment + '\n')
|
||||
written += 1
|
||||
if options.verbose and written > 0:
|
||||
print "...writing comments for " + (key if key else 'EOF')
|
||||
|
||||
def writefield(field, value=None):
|
||||
writecomments(field)
|
||||
if value is None:
|
||||
value = app[field]
|
||||
mf.write(field + ':' + value + '\n')
|
||||
|
||||
mf = open(dest, 'w')
|
||||
if app['Disabled']:
|
||||
writefield('Disabled')
|
||||
if app['AntiFeatures']:
|
||||
writefield('AntiFeatures')
|
||||
writefield('Categories')
|
||||
writefield('License')
|
||||
writefield('Web Site')
|
||||
writefield('Source Code')
|
||||
writefield('Issue Tracker')
|
||||
if app['Donate']:
|
||||
writefield('Donate')
|
||||
if app['FlattrID']:
|
||||
writefield('FlattrID')
|
||||
if app['Bitcoin']:
|
||||
writefield('Bitcoin')
|
||||
if app['Litecoin']:
|
||||
writefield('Litecoin')
|
||||
mf.write('\n')
|
||||
if app['Name']:
|
||||
writefield('Name')
|
||||
if app['Auto Name']:
|
||||
writefield('Auto Name')
|
||||
writefield('Summary')
|
||||
writefield('Description', '')
|
||||
for line in app['Description']:
|
||||
mf.write(line + '\n')
|
||||
mf.write('.\n')
|
||||
mf.write('\n')
|
||||
if app['Requires Root']:
|
||||
writefield('Requires Root', 'Yes')
|
||||
mf.write('\n')
|
||||
if app['Repo Type']:
|
||||
writefield('Repo Type')
|
||||
writefield('Repo')
|
||||
mf.write('\n')
|
||||
for build in app['builds']:
|
||||
writecomments('build:' + build['version'])
|
||||
mf.write('Build:')
|
||||
mf.write("%s,%s\n" % (
|
||||
build['version'],
|
||||
getvercode(build)))
|
||||
|
||||
# This defines the preferred order for the build items - as in the
|
||||
# manual, they're roughly in order of application.
|
||||
keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
|
||||
'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk',
|
||||
'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
|
||||
'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
|
||||
'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
|
||||
'preassemble', 'bindir', 'antcommand', 'novcheck']
|
||||
|
||||
def write_builditem(key, value):
|
||||
if key not in ['version', 'vercode', 'origlines']:
|
||||
if key in bool_keys:
|
||||
if not value:
|
||||
return
|
||||
value = 'yes'
|
||||
if options.verbose:
|
||||
print "...writing {0} : {1}".format(key, value)
|
||||
outline = ' %s=' % key
|
||||
outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
|
||||
outline += '\n'
|
||||
mf.write(outline)
|
||||
|
||||
for key in keyorder:
|
||||
if key in build:
|
||||
write_builditem(key, build[key])
|
||||
for key, value in build.iteritems():
|
||||
if not key in keyorder:
|
||||
write_builditem(key, value)
|
||||
mf.write('\n')
|
||||
|
||||
if 'Maintainer Notes' in app:
|
||||
writefield('Maintainer Notes', '')
|
||||
for line in app['Maintainer Notes']:
|
||||
mf.write(line + '\n')
|
||||
mf.write('.\n')
|
||||
mf.write('\n')
|
||||
|
||||
|
||||
if app['Archive Policy']:
|
||||
writefield('Archive Policy')
|
||||
writefield('Auto Update Mode')
|
||||
writefield('Update Check Mode')
|
||||
if app['Vercode Operation']:
|
||||
writefield('Vercode Operation')
|
||||
if 'Update Check Data' in app:
|
||||
writefield('Update Check Data')
|
||||
if app['Current Version']:
|
||||
writefield('Current Version')
|
||||
writefield('Current Version Code')
|
||||
mf.write('\n')
|
||||
if app['No Source Since']:
|
||||
writefield('No Source Since')
|
||||
mf.write('\n')
|
||||
writecomments(None)
|
||||
mf.close()
|
||||
|
||||
|
||||
# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
|
||||
# returned by the parse_metadata function.
|
||||
def read_metadata(xref=True, package=None):
|
||||
apps = []
|
||||
for basedir in ('metadata', 'tmp'):
|
||||
if not os.path.exists(basedir):
|
||||
os.makedirs(basedir)
|
||||
for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
|
||||
if package is None or metafile == os.path.join('metadata', package + '.txt'):
|
||||
try:
|
||||
appinfo = parse_metadata(metafile)
|
||||
except Exception, e:
|
||||
raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
|
||||
apps.append(appinfo)
|
||||
|
||||
if xref:
|
||||
# Parse all descriptions at load time, just to ensure cross-referencing
|
||||
# errors are caught early rather than when they hit the build server.
|
||||
def linkres(link):
|
||||
for app in apps:
|
||||
if app['id'] == link:
|
||||
return ("fdroid.app:" + link, "Dummy name - don't know yet")
|
||||
raise MetaDataException("Cannot resolve app id " + link)
|
||||
for app in apps:
|
||||
try:
|
||||
description_html(app['Description'], linkres)
|
||||
except Exception, e:
|
||||
raise MetaDataException("Problem with description of " + app['id'] +
|
||||
" - " + str(e))
|
||||
|
||||
return apps
|
||||
|
||||
# Formatter for descriptions. Create an instance, and call parseline() with
|
||||
# each line of the description source from the metadata. At the end, call
|
||||
# end() and then text_plain, text_wiki and text_html will contain the result.
|
||||
class DescriptionFormatter:
|
||||
stNONE = 0
|
||||
stPARA = 1
|
||||
stUL = 2
|
||||
stOL = 3
|
||||
bold = False
|
||||
ital = False
|
||||
state = stNONE
|
||||
text_plain = ''
|
||||
text_wiki = ''
|
||||
text_html = ''
|
||||
linkResolver = None
|
||||
def __init__(self, linkres):
|
||||
self.linkResolver = linkres
|
||||
def endcur(self, notstates=None):
|
||||
if notstates and self.state in notstates:
|
||||
return
|
||||
if self.state == self.stPARA:
|
||||
self.endpara()
|
||||
elif self.state == self.stUL:
|
||||
self.endul()
|
||||
elif self.state == self.stOL:
|
||||
self.endol()
|
||||
def endpara(self):
|
||||
self.text_plain += '\n'
|
||||
self.text_html += '</p>'
|
||||
self.state = self.stNONE
|
||||
def endul(self):
|
||||
self.text_html += '</ul>'
|
||||
self.state = self.stNONE
|
||||
def endol(self):
|
||||
self.text_html += '</ol>'
|
||||
self.state = self.stNONE
|
||||
|
||||
def formatted(self, txt, html):
|
||||
formatted = ''
|
||||
if html:
|
||||
txt = cgi.escape(txt)
|
||||
while True:
|
||||
index = txt.find("''")
|
||||
if index == -1:
|
||||
return formatted + txt
|
||||
formatted += txt[:index]
|
||||
txt = txt[index:]
|
||||
if txt.startswith("'''"):
|
||||
if html:
|
||||
if self.bold:
|
||||
formatted += '</b>'
|
||||
else:
|
||||
formatted += '<b>'
|
||||
self.bold = not self.bold
|
||||
txt = txt[3:]
|
||||
else:
|
||||
if html:
|
||||
if self.ital:
|
||||
formatted += '</i>'
|
||||
else:
|
||||
formatted += '<i>'
|
||||
self.ital = not self.ital
|
||||
txt = txt[2:]
|
||||
|
||||
|
||||
def linkify(self, txt):
|
||||
linkified_plain = ''
|
||||
linkified_html = ''
|
||||
while True:
|
||||
index = txt.find("[")
|
||||
if index == -1:
|
||||
return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
|
||||
linkified_plain += self.formatted(txt[:index], False)
|
||||
linkified_html += self.formatted(txt[:index], True)
|
||||
txt = txt[index:]
|
||||
if txt.startswith("[["):
|
||||
index = txt.find("]]")
|
||||
if index == -1:
|
||||
raise MetaDataException("Unterminated ]]")
|
||||
url = txt[2:index]
|
||||
if self.linkResolver:
|
||||
url, urltext = self.linkResolver(url)
|
||||
else:
|
||||
urltext = url
|
||||
linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
|
||||
linkified_plain += urltext
|
||||
txt = txt[index+2:]
|
||||
else:
|
||||
index = txt.find("]")
|
||||
if index == -1:
|
||||
raise MetaDataException("Unterminated ]")
|
||||
url = txt[1:index]
|
||||
index2 = url.find(' ')
|
||||
if index2 == -1:
|
||||
urltxt = url
|
||||
else:
|
||||
urltxt = url[index2 + 1:]
|
||||
url = url[:index2]
|
||||
linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
|
||||
linkified_plain += urltxt
|
||||
if urltxt != url:
|
||||
linkified_plain += ' (' + url + ')'
|
||||
txt = txt[index+1:]
|
||||
|
||||
def addtext(self, txt):
|
||||
p, h = self.linkify(txt)
|
||||
self.text_plain += p
|
||||
self.text_html += h
|
||||
|
||||
def parseline(self, line):
|
||||
self.text_wiki += line + '\n'
|
||||
if not line:
|
||||
self.endcur()
|
||||
elif line.startswith('*'):
|
||||
self.endcur([self.stUL])
|
||||
if self.state != self.stUL:
|
||||
self.text_html += '<ul>'
|
||||
self.state = self.stUL
|
||||
self.text_html += '<li>'
|
||||
self.text_plain += '*'
|
||||
self.addtext(line[1:])
|
||||
self.text_html += '</li>'
|
||||
elif line.startswith('#'):
|
||||
self.endcur([self.stOL])
|
||||
if self.state != self.stOL:
|
||||
self.text_html += '<ol>'
|
||||
self.state = self.stOL
|
||||
self.text_html += '<li>'
|
||||
self.text_plain += '*' #TODO: lazy - put the numbers in!
|
||||
self.addtext(line[1:])
|
||||
self.text_html += '</li>'
|
||||
else:
|
||||
self.endcur([self.stPARA])
|
||||
if self.state == self.stNONE:
|
||||
self.text_html += '<p>'
|
||||
self.state = self.stPARA
|
||||
elif self.state == self.stPARA:
|
||||
self.text_html += ' '
|
||||
self.text_plain += ' '
|
||||
self.addtext(line)
|
||||
|
||||
def end(self):
|
||||
self.endcur()
|
||||
|
||||
# Parse multiple lines of description as written in a metadata file, returning
|
||||
# a single string in plain text format.
|
||||
def description_plain(lines, linkres):
|
||||
ps = DescriptionFormatter(linkres)
|
||||
for line in lines:
|
||||
ps.parseline(line)
|
||||
ps.end()
|
||||
return ps.text_plain
|
||||
|
||||
# Parse multiple lines of description as written in a metadata file, returning
|
||||
# a single string in wiki format. Used for the Maintainer Notes field as well,
|
||||
# because it's the same format.
|
||||
def description_wiki(lines):
|
||||
ps = DescriptionFormatter(None)
|
||||
for line in lines:
|
||||
ps.parseline(line)
|
||||
ps.end()
|
||||
return ps.text_wiki
|
||||
|
||||
# Parse multiple lines of description as written in a metadata file, returning
|
||||
# a single string in HTML format.
|
||||
def description_html(lines,linkres):
|
||||
ps = DescriptionFormatter(linkres)
|
||||
for line in lines:
|
||||
ps.parseline(line)
|
||||
ps.end()
|
||||
return ps.text_html
|
||||
|
||||
def retrieve_string(xml_dir, string):
|
||||
if not string.startswith('@string/'):
|
||||
return string.replace("\\'","'")
|
||||
@ -1248,48 +629,6 @@ class VCSException(Exception):
|
||||
def __str__(self):
|
||||
return repr(self.value)
|
||||
|
||||
class MetaDataException(Exception):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.value)
|
||||
|
||||
def parse_srclib(metafile, **kw):
|
||||
|
||||
thisinfo = {}
|
||||
if metafile and not isinstance(metafile, file):
|
||||
metafile = open(metafile, "r")
|
||||
|
||||
# Defaults for fields that come from metadata
|
||||
thisinfo['Repo Type'] = ''
|
||||
thisinfo['Repo'] = ''
|
||||
thisinfo['Subdir'] = None
|
||||
thisinfo['Prepare'] = None
|
||||
thisinfo['Srclibs'] = None
|
||||
thisinfo['Update Project'] = None
|
||||
|
||||
if metafile is None:
|
||||
return thisinfo
|
||||
|
||||
for line in metafile:
|
||||
line = line.rstrip('\r\n')
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
|
||||
index = line.find(':')
|
||||
if index == -1:
|
||||
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
|
||||
field = line[:index]
|
||||
value = line[index+1:]
|
||||
|
||||
if field == "Subdir":
|
||||
thisinfo[field] = value.split(',')
|
||||
else:
|
||||
thisinfo[field] = value
|
||||
|
||||
return thisinfo
|
||||
|
||||
# Get the specified source library.
|
||||
# Returns the path to it. Normally this is the path to be used when referencing
|
||||
# it, which may be a subdirectory of the actual project. If you want the base
|
||||
@ -1314,7 +653,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, basepath=False,
|
||||
if not os.path.exists(srclib_path):
|
||||
raise BuildException('srclib ' + name + ' not found.')
|
||||
|
||||
srclib = parse_srclib(srclib_path)
|
||||
srclib = metadata.parse_srclib(srclib_path)
|
||||
|
||||
sdir = os.path.join(srclib_dir, name)
|
||||
|
||||
|
||||
@ -22,7 +22,7 @@ import os
|
||||
import shutil
|
||||
import urllib
|
||||
from optparse import OptionParser
|
||||
import common
|
||||
import common, metadata
|
||||
|
||||
# Get the repo type and address from the given web page. The page is scanned
|
||||
# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and
|
||||
@ -114,7 +114,7 @@ def main():
|
||||
os.makedirs(tmp_dir)
|
||||
|
||||
# Get all apps...
|
||||
apps = common.read_metadata()
|
||||
apps = metadata.read_metadata()
|
||||
|
||||
# Figure out what kind of project it is...
|
||||
projecttype = None
|
||||
@ -249,7 +249,7 @@ def main():
|
||||
sys.exit(1)
|
||||
|
||||
# Construct the metadata...
|
||||
app = common.parse_metadata(None)
|
||||
app = metadata.parse_metadata(None)
|
||||
app['id'] = package
|
||||
app['Web Site'] = website
|
||||
app['Source Code'] = sourcecode
|
||||
@ -281,7 +281,7 @@ def main():
|
||||
f.write(repotype + ' ' + repo)
|
||||
|
||||
metafile = os.path.join('metadata', package + '.txt')
|
||||
common.write_metadata(metafile, app)
|
||||
metadata.write_metadata(metafile, app)
|
||||
print "Wrote " + metafile
|
||||
|
||||
|
||||
|
||||
714
fdroidserver/metadata.py
Normal file
714
fdroidserver/metadata.py
Normal file
@ -0,0 +1,714 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# common.py - part of the FDroid server tools
|
||||
# Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
|
||||
# Copyright (C) 2013 Daniel Martí <mvdan@mvdan.cc>
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os, re, glob
|
||||
import cgi
|
||||
|
||||
class MetaDataException(Exception):
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def __str__(self):
|
||||
return repr(self.value)
|
||||
|
||||
class FieldType():
|
||||
def __init__(self, name, matching, sep, fields, attrs):
|
||||
self.name = name
|
||||
if type(matching) is str:
|
||||
self.matching = re.compile(matching)
|
||||
elif type(matching) is list:
|
||||
self.matching = matching
|
||||
self.sep = sep
|
||||
self.fields = fields
|
||||
self.attrs = attrs
|
||||
|
||||
def _assert_regex(self, values, appid):
|
||||
for v in values:
|
||||
if not self.matching.match(v):
|
||||
raise MetaDataException("'%s' is not a valid %s in %s"
|
||||
% (v, self.name, appid))
|
||||
|
||||
def _assert_list(self, values, appid):
|
||||
for v in values:
|
||||
if v not in self.matching:
|
||||
raise MetaDataException("'%s' is not a valid %s in %s"
|
||||
% (v, self.name, appid))
|
||||
|
||||
def check(self, value, appid):
|
||||
if type(value) is not str or not value:
|
||||
return
|
||||
if self.sep is not None:
|
||||
values = value.split(self.sep)
|
||||
else:
|
||||
values = [value]
|
||||
if type(self.matching) is list:
|
||||
self._assert_list(values, appid)
|
||||
else:
|
||||
self._assert_regex(values, appid)
|
||||
|
||||
|
||||
valuetypes = {
|
||||
'int' : FieldType("Integer",
|
||||
r'^[0-9]+$', None,
|
||||
[ 'FlattrID' ],
|
||||
[ 'vercode' ]),
|
||||
|
||||
'http' : FieldType("HTTP link",
|
||||
r'^http[s]?://.+$', None,
|
||||
[ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
|
||||
|
||||
'bitcoin' : FieldType("Bitcoin address",
|
||||
r'^[a-zA-Z0-9]{27,34}$', None,
|
||||
[ "Bitcoin" ],
|
||||
[ ]),
|
||||
|
||||
'litecoin' : FieldType("Litecoin address",
|
||||
r'^[a-zA-Z0-9]{27,34}$', None,
|
||||
[ "Bitcoin" ],
|
||||
[ ]),
|
||||
|
||||
'bool' : FieldType("Boolean",
|
||||
['yes', 'no'], None,
|
||||
[ ],
|
||||
[ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
|
||||
'fixtrans', 'fixapos', 'novcheck' ]),
|
||||
|
||||
'Bool' : FieldType("Boolean",
|
||||
['Yes', 'No'], None,
|
||||
[ "Requires Root" ],
|
||||
[ ]),
|
||||
|
||||
'antifeatures' : FieldType("Anti-Feature",
|
||||
[ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd" ], ',',
|
||||
[ "AntiFeatures" ],
|
||||
[ ]),
|
||||
}
|
||||
|
||||
def check_metadata(info):
|
||||
|
||||
# Generic fields and attributes
|
||||
for k, t in valuetypes.iteritems():
|
||||
for field in [f for f in t.fields if f in info]:
|
||||
t.check(info[field], info['id'])
|
||||
if k == 'Bool':
|
||||
info[field] = info[field] == "Yes"
|
||||
for build in info['builds']:
|
||||
for attr in [a for a in t.attrs if a in build]:
|
||||
t.check(build[attr], info['id'])
|
||||
if k == 'bool':
|
||||
info[field] = info[field] == "yes"
|
||||
|
||||
# Special fields
|
||||
if info['Archive Policy']:
|
||||
if not re.match(r'^[0-9]+ versions$', info['Archive Policy']):
|
||||
raise MetaDataException("Invalid archive policy '%s' in %s"
|
||||
% (info['Archive Policy'], info["id"]))
|
||||
versions = int(info['Archive Policy'][:-9])
|
||||
if versions < 1 or versions > 20:
|
||||
raise MetaDataException("Silly number of versions '%s' for archive policy in %s"
|
||||
% (versions, info["id"]))
|
||||
|
||||
# Formatter for descriptions. Create an instance, and call parseline() with
|
||||
# each line of the description source from the metadata. At the end, call
|
||||
# end() and then text_plain, text_wiki and text_html will contain the result.
|
||||
class DescriptionFormatter:
|
||||
stNONE = 0
|
||||
stPARA = 1
|
||||
stUL = 2
|
||||
stOL = 3
|
||||
bold = False
|
||||
ital = False
|
||||
state = stNONE
|
||||
text_plain = ''
|
||||
text_wiki = ''
|
||||
text_html = ''
|
||||
linkResolver = None
|
||||
def __init__(self, linkres):
|
||||
self.linkResolver = linkres
|
||||
def endcur(self, notstates=None):
|
||||
if notstates and self.state in notstates:
|
||||
return
|
||||
if self.state == self.stPARA:
|
||||
self.endpara()
|
||||
elif self.state == self.stUL:
|
||||
self.endul()
|
||||
elif self.state == self.stOL:
|
||||
self.endol()
|
||||
def endpara(self):
|
||||
self.text_plain += '\n'
|
||||
self.text_html += '</p>'
|
||||
self.state = self.stNONE
|
||||
def endul(self):
|
||||
self.text_html += '</ul>'
|
||||
self.state = self.stNONE
|
||||
def endol(self):
|
||||
self.text_html += '</ol>'
|
||||
self.state = self.stNONE
|
||||
|
||||
def formatted(self, txt, html):
|
||||
formatted = ''
|
||||
if html:
|
||||
txt = cgi.escape(txt)
|
||||
while True:
|
||||
index = txt.find("''")
|
||||
if index == -1:
|
||||
return formatted + txt
|
||||
formatted += txt[:index]
|
||||
txt = txt[index:]
|
||||
if txt.startswith("'''"):
|
||||
if html:
|
||||
if self.bold:
|
||||
formatted += '</b>'
|
||||
else:
|
||||
formatted += '<b>'
|
||||
self.bold = not self.bold
|
||||
txt = txt[3:]
|
||||
else:
|
||||
if html:
|
||||
if self.ital:
|
||||
formatted += '</i>'
|
||||
else:
|
||||
formatted += '<i>'
|
||||
self.ital = not self.ital
|
||||
txt = txt[2:]
|
||||
|
||||
|
||||
def linkify(self, txt):
|
||||
linkified_plain = ''
|
||||
linkified_html = ''
|
||||
while True:
|
||||
index = txt.find("[")
|
||||
if index == -1:
|
||||
return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
|
||||
linkified_plain += self.formatted(txt[:index], False)
|
||||
linkified_html += self.formatted(txt[:index], True)
|
||||
txt = txt[index:]
|
||||
if txt.startswith("[["):
|
||||
index = txt.find("]]")
|
||||
if index == -1:
|
||||
raise MetaDataException("Unterminated ]]")
|
||||
url = txt[2:index]
|
||||
if self.linkResolver:
|
||||
url, urltext = self.linkResolver(url)
|
||||
else:
|
||||
urltext = url
|
||||
linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
|
||||
linkified_plain += urltext
|
||||
txt = txt[index+2:]
|
||||
else:
|
||||
index = txt.find("]")
|
||||
if index == -1:
|
||||
raise MetaDataException("Unterminated ]")
|
||||
url = txt[1:index]
|
||||
index2 = url.find(' ')
|
||||
if index2 == -1:
|
||||
urltxt = url
|
||||
else:
|
||||
urltxt = url[index2 + 1:]
|
||||
url = url[:index2]
|
||||
linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
|
||||
linkified_plain += urltxt
|
||||
if urltxt != url:
|
||||
linkified_plain += ' (' + url + ')'
|
||||
txt = txt[index+1:]
|
||||
|
||||
def addtext(self, txt):
|
||||
p, h = self.linkify(txt)
|
||||
self.text_plain += p
|
||||
self.text_html += h
|
||||
|
||||
def parseline(self, line):
|
||||
self.text_wiki += "%s\n" % line
|
||||
if not line:
|
||||
self.endcur()
|
||||
elif line.startswith('*'):
|
||||
self.endcur([self.stUL])
|
||||
if self.state != self.stUL:
|
||||
self.text_html += '<ul>'
|
||||
self.state = self.stUL
|
||||
self.text_html += '<li>'
|
||||
self.text_plain += '*'
|
||||
self.addtext(line[1:])
|
||||
self.text_html += '</li>'
|
||||
elif line.startswith('#'):
|
||||
self.endcur([self.stOL])
|
||||
if self.state != self.stOL:
|
||||
self.text_html += '<ol>'
|
||||
self.state = self.stOL
|
||||
self.text_html += '<li>'
|
||||
self.text_plain += '*' #TODO: lazy - put the numbers in!
|
||||
self.addtext(line[1:])
|
||||
self.text_html += '</li>'
|
||||
else:
|
||||
self.endcur([self.stPARA])
|
||||
if self.state == self.stNONE:
|
||||
self.text_html += '<p>'
|
||||
self.state = self.stPARA
|
||||
elif self.state == self.stPARA:
|
||||
self.text_html += ' '
|
||||
self.text_plain += ' '
|
||||
self.addtext(line)
|
||||
|
||||
def end(self):
|
||||
self.endcur()
|
||||
|
||||
# Parse multiple lines of description as written in a metadata file, returning
|
||||
# a single string in plain text format.
|
||||
def description_plain(lines, linkres):
|
||||
ps = DescriptionFormatter(linkres)
|
||||
for line in lines:
|
||||
ps.parseline(line)
|
||||
ps.end()
|
||||
return ps.text_plain
|
||||
|
||||
# Parse multiple lines of description as written in a metadata file, returning
|
||||
# a single string in wiki format. Used for the Maintainer Notes field as well,
|
||||
# because it's the same format.
|
||||
def description_wiki(lines):
|
||||
ps = DescriptionFormatter(None)
|
||||
for line in lines:
|
||||
ps.parseline(line)
|
||||
ps.end()
|
||||
return ps.text_wiki
|
||||
|
||||
# Parse multiple lines of description as written in a metadata file, returning
|
||||
# a single string in HTML format.
|
||||
def description_html(lines,linkres):
|
||||
ps = DescriptionFormatter(linkres)
|
||||
for line in lines:
|
||||
ps.parseline(line)
|
||||
ps.end()
|
||||
return ps.text_html
|
||||
|
||||
def parse_srclib(metafile, **kw):
|
||||
|
||||
thisinfo = {}
|
||||
if metafile and not isinstance(metafile, file):
|
||||
metafile = open(metafile, "r")
|
||||
|
||||
# Defaults for fields that come from metadata
|
||||
thisinfo['Repo Type'] = ''
|
||||
thisinfo['Repo'] = ''
|
||||
thisinfo['Subdir'] = None
|
||||
thisinfo['Prepare'] = None
|
||||
thisinfo['Srclibs'] = None
|
||||
thisinfo['Update Project'] = None
|
||||
|
||||
if metafile is None:
|
||||
return thisinfo
|
||||
|
||||
for line in metafile:
|
||||
line = line.rstrip('\r\n')
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
|
||||
index = line.find(':')
|
||||
if index == -1:
|
||||
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
|
||||
field = line[:index]
|
||||
value = line[index+1:]
|
||||
|
||||
if field == "Subdir":
|
||||
thisinfo[field] = value.split(',')
|
||||
else:
|
||||
thisinfo[field] = value
|
||||
|
||||
return thisinfo
|
||||
|
||||
# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
|
||||
# returned by the parse_metadata function.
|
||||
def read_metadata(xref=True, package=None):
|
||||
apps = []
|
||||
for basedir in ('metadata', 'tmp'):
|
||||
if not os.path.exists(basedir):
|
||||
os.makedirs(basedir)
|
||||
for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
|
||||
if package is None or metafile == os.path.join('metadata', package + '.txt'):
|
||||
try:
|
||||
appinfo = parse_metadata(metafile)
|
||||
except Exception, e:
|
||||
raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
|
||||
check_metadata(appinfo)
|
||||
apps.append(appinfo)
|
||||
|
||||
if xref:
|
||||
# Parse all descriptions at load time, just to ensure cross-referencing
|
||||
# errors are caught early rather than when they hit the build server.
|
||||
def linkres(link):
|
||||
for app in apps:
|
||||
if app['id'] == link:
|
||||
return ("fdroid.app:" + link, "Dummy name - don't know yet")
|
||||
raise MetaDataException("Cannot resolve app id " + link)
|
||||
for app in apps:
|
||||
try:
|
||||
description_html(app['Description'], linkres)
|
||||
except Exception, e:
|
||||
raise MetaDataException("Problem with description of " + app['id'] +
|
||||
" - " + str(e))
|
||||
|
||||
return apps
|
||||
|
||||
# Get the type expected for a given metadata field.
|
||||
def metafieldtype(name):
|
||||
if name in ['Description', 'Maintainer Notes']:
|
||||
return 'multiline'
|
||||
if name == 'Build Version':
|
||||
return 'build'
|
||||
if name == 'Build':
|
||||
return 'buildv2'
|
||||
if name == 'Use Built':
|
||||
return 'obsolete'
|
||||
return 'string'
|
||||
|
||||
# Parse metadata for a single application.
|
||||
#
|
||||
# 'metafile' - the filename to read. The package id for the application comes
|
||||
# from this filename. Pass None to get a blank entry.
|
||||
#
|
||||
# Returns a dictionary containing all the details of the application. There are
|
||||
# two major kinds of information in the dictionary. Keys beginning with capital
|
||||
# letters correspond directory to identically named keys in the metadata file.
|
||||
# Keys beginning with lower case letters are generated in one way or another,
|
||||
# and are not found verbatim in the metadata.
|
||||
#
|
||||
# Known keys not originating from the metadata are:
|
||||
#
|
||||
# 'id' - the application's package ID
|
||||
# 'builds' - a list of dictionaries containing build information
|
||||
# for each defined build
|
||||
# 'comments' - a list of comments from the metadata file. Each is
|
||||
# a tuple of the form (field, comment) where field is
|
||||
# the name of the field it preceded in the metadata
|
||||
# file. Where field is None, the comment goes at the
|
||||
# end of the file. Alternatively, 'build:version' is
|
||||
# for a comment before a particular build version.
|
||||
# 'descriptionlines' - original lines of description as formatted in the
|
||||
# metadata file.
|
||||
#
|
||||
def parse_metadata(metafile):
|
||||
|
||||
def parse_buildline(lines):
|
||||
value = "".join(lines)
|
||||
parts = [p.replace("\\,", ",")
|
||||
for p in re.split(r"(?<!\\),", value)]
|
||||
if len(parts) < 3:
|
||||
raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
|
||||
thisbuild = {}
|
||||
thisbuild['origlines'] = lines
|
||||
thisbuild['version'] = parts[0]
|
||||
thisbuild['vercode'] = parts[1]
|
||||
if parts[2].startswith('!'):
|
||||
# For backwards compatibility, handle old-style disabling,
|
||||
# including attempting to extract the commit from the message
|
||||
thisbuild['disable'] = parts[2][1:]
|
||||
commit = 'unknown - see disabled'
|
||||
index = parts[2].rfind('at ')
|
||||
if index != -1:
|
||||
commit = parts[2][index+3:]
|
||||
if commit.endswith(')'):
|
||||
commit = commit[:-1]
|
||||
thisbuild['commit'] = commit
|
||||
else:
|
||||
thisbuild['commit'] = parts[2]
|
||||
for p in parts[3:]:
|
||||
pk, pv = p.split('=', 1)
|
||||
thisbuild[pk.strip()] = pv
|
||||
|
||||
return thisbuild
|
||||
|
||||
def add_comments(key):
|
||||
if not curcomments:
|
||||
return
|
||||
for comment in curcomments:
|
||||
thisinfo['comments'].append((key, comment))
|
||||
del curcomments[:]
|
||||
|
||||
|
||||
thisinfo = {}
|
||||
if metafile:
|
||||
if not isinstance(metafile, file):
|
||||
metafile = open(metafile, "r")
|
||||
thisinfo['id'] = metafile.name[9:-4]
|
||||
else:
|
||||
thisinfo['id'] = None
|
||||
|
||||
# Defaults for fields that come from metadata...
|
||||
thisinfo['Name'] = None
|
||||
thisinfo['Auto Name'] = ''
|
||||
thisinfo['Categories'] = 'None'
|
||||
thisinfo['Description'] = []
|
||||
thisinfo['Summary'] = ''
|
||||
thisinfo['License'] = 'Unknown'
|
||||
thisinfo['Web Site'] = ''
|
||||
thisinfo['Source Code'] = ''
|
||||
thisinfo['Issue Tracker'] = ''
|
||||
thisinfo['Donate'] = None
|
||||
thisinfo['FlattrID'] = None
|
||||
thisinfo['Bitcoin'] = None
|
||||
thisinfo['Litecoin'] = None
|
||||
thisinfo['Disabled'] = None
|
||||
thisinfo['AntiFeatures'] = None
|
||||
thisinfo['Archive Policy'] = None
|
||||
thisinfo['Update Check Mode'] = 'None'
|
||||
thisinfo['Vercode Operation'] = None
|
||||
thisinfo['Auto Update Mode'] = 'None'
|
||||
thisinfo['Current Version'] = ''
|
||||
thisinfo['Current Version Code'] = '0'
|
||||
thisinfo['Repo Type'] = ''
|
||||
thisinfo['Repo'] = ''
|
||||
thisinfo['Requires Root'] = False
|
||||
thisinfo['No Source Since'] = ''
|
||||
|
||||
# General defaults...
|
||||
thisinfo['builds'] = []
|
||||
thisinfo['comments'] = []
|
||||
|
||||
if metafile is None:
|
||||
return thisinfo
|
||||
|
||||
mode = 0
|
||||
buildlines = []
|
||||
curcomments = []
|
||||
curbuild = None
|
||||
|
||||
for line in metafile:
|
||||
line = line.rstrip('\r\n')
|
||||
if mode == 3:
|
||||
if not any(line.startswith(s) for s in (' ', '\t')):
|
||||
if 'commit' not in curbuild and 'disable' not in curbuild:
|
||||
raise MetaDataException("No commit specified for {0} in {1}".format(
|
||||
curbuild['version'], metafile.name))
|
||||
thisinfo['builds'].append(curbuild)
|
||||
add_comments('build:' + curbuild['version'])
|
||||
mode = 0
|
||||
else:
|
||||
if line.endswith('\\'):
|
||||
buildlines.append(line[:-1].lstrip())
|
||||
else:
|
||||
buildlines.append(line.lstrip())
|
||||
bl = ''.join(buildlines)
|
||||
bv = bl.split('=', 1)
|
||||
if len(bv) != 2:
|
||||
raise MetaDataException("Invalid build flag at {0} in {1}".
|
||||
format(buildlines[0], metafile.name))
|
||||
name, val = bv
|
||||
if name in curbuild:
|
||||
raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
|
||||
format(name, curbuild['version'], metafile.name))
|
||||
curbuild[name] = val.lstrip()
|
||||
buildlines = []
|
||||
|
||||
if mode == 0:
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
curcomments.append(line)
|
||||
continue
|
||||
index = line.find(':')
|
||||
if index == -1:
|
||||
raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
|
||||
field = line[:index]
|
||||
value = line[index+1:]
|
||||
|
||||
# Translate obsolete fields...
|
||||
if field == 'Market Version':
|
||||
field = 'Current Version'
|
||||
if field == 'Market Version Code':
|
||||
field = 'Current Version Code'
|
||||
|
||||
fieldtype = metafieldtype(field)
|
||||
if fieldtype not in ['build', 'buildv2']:
|
||||
add_comments(field)
|
||||
if fieldtype == 'multiline':
|
||||
mode = 1
|
||||
thisinfo[field] = []
|
||||
if value:
|
||||
raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
|
||||
elif fieldtype == 'string':
|
||||
if field == 'Category' and thisinfo['Categories'] == 'None':
|
||||
thisinfo['Categories'] = value.replace(';',',')
|
||||
thisinfo[field] = value
|
||||
elif fieldtype == 'build':
|
||||
if value.endswith("\\"):
|
||||
mode = 2
|
||||
buildlines = [value[:-1]]
|
||||
else:
|
||||
thisinfo['builds'].append(parse_buildline([value]))
|
||||
add_comments('build:' + thisinfo['builds'][-1]['version'])
|
||||
elif fieldtype == 'buildv2':
|
||||
curbuild = {}
|
||||
vv = value.split(',')
|
||||
if len(vv) != 2:
|
||||
raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
|
||||
format(value, metafile.name))
|
||||
curbuild['version'] = vv[0]
|
||||
curbuild['vercode'] = vv[1]
|
||||
buildlines = []
|
||||
mode = 3
|
||||
elif fieldtype == 'obsolete':
|
||||
pass # Just throw it away!
|
||||
else:
|
||||
raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
|
||||
elif mode == 1: # Multiline field
|
||||
if line == '.':
|
||||
mode = 0
|
||||
else:
|
||||
thisinfo[field].append(line)
|
||||
elif mode == 2: # Line continuation mode in Build Version
|
||||
if line.endswith("\\"):
|
||||
buildlines.append(line[:-1])
|
||||
else:
|
||||
buildlines.append(line)
|
||||
thisinfo['builds'].append(
|
||||
parse_buildline(buildlines))
|
||||
add_comments('build:' + thisinfo['builds'][-1]['version'])
|
||||
mode = 0
|
||||
add_comments(None)
|
||||
|
||||
# Mode at end of file should always be 0...
|
||||
if mode == 1:
|
||||
raise MetaDataException(field + " not terminated in " + metafile.name)
|
||||
elif mode == 2:
|
||||
raise MetaDataException("Unterminated continuation in " + metafile.name)
|
||||
elif mode == 3:
|
||||
raise MetaDataException("Unterminated build in " + metafile.name)
|
||||
|
||||
if not thisinfo['Description']:
|
||||
thisinfo['Description'].append('No description available')
|
||||
|
||||
return thisinfo
|
||||
|
||||
# Write a metadata file.
|
||||
#
|
||||
# 'dest' - The path to the output file
|
||||
# 'app' - The app data
|
||||
def write_metadata(dest, app):
|
||||
|
||||
def writecomments(key):
|
||||
written = 0
|
||||
for pf, comment in app['comments']:
|
||||
if pf == key:
|
||||
mf.write("%s\n" % comment)
|
||||
written += 1
|
||||
#if options.verbose and written > 0:
|
||||
#print "...writing comments for " + (key if key else 'EOF')
|
||||
|
||||
def writefield(field, value=None):
|
||||
writecomments(field)
|
||||
if value is None:
|
||||
value = app[field]
|
||||
mf.write("%s:%s\n" % (field, value))
|
||||
|
||||
mf = open(dest, 'w')
|
||||
if app['Disabled']:
|
||||
writefield('Disabled')
|
||||
if app['AntiFeatures']:
|
||||
writefield('AntiFeatures')
|
||||
writefield('Categories')
|
||||
writefield('License')
|
||||
writefield('Web Site')
|
||||
writefield('Source Code')
|
||||
writefield('Issue Tracker')
|
||||
if app['Donate']:
|
||||
writefield('Donate')
|
||||
if app['FlattrID']:
|
||||
writefield('FlattrID')
|
||||
if app['Bitcoin']:
|
||||
writefield('Bitcoin')
|
||||
if app['Litecoin']:
|
||||
writefield('Litecoin')
|
||||
mf.write('\n')
|
||||
if app['Name']:
|
||||
writefield('Name')
|
||||
if app['Auto Name']:
|
||||
writefield('Auto Name')
|
||||
writefield('Summary')
|
||||
writefield('Description', '')
|
||||
for line in app['Description']:
|
||||
mf.write("%s\n" % line)
|
||||
mf.write('.\n')
|
||||
mf.write('\n')
|
||||
if app['Requires Root']:
|
||||
writefield('Requires Root', 'Yes')
|
||||
mf.write('\n')
|
||||
if app['Repo Type']:
|
||||
writefield('Repo Type')
|
||||
writefield('Repo')
|
||||
mf.write('\n')
|
||||
for build in app['builds']:
|
||||
writecomments('build:' + build['version'])
|
||||
mf.write("Build:%s,%s\n" % ( build['version'], build['vercode']))
|
||||
|
||||
# This defines the preferred order for the build items - as in the
|
||||
# manual, they're roughly in order of application.
|
||||
keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
|
||||
'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk',
|
||||
'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
|
||||
'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
|
||||
'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
|
||||
'preassemble', 'bindir', 'antcommand', 'novcheck']
|
||||
|
||||
def write_builditem(key, value):
|
||||
if key not in ['version', 'vercode', 'origlines']:
|
||||
if key in valuetypes['bool'].attrs:
|
||||
if not value:
|
||||
return
|
||||
value = 'yes'
|
||||
#if options.verbose:
|
||||
#print "...writing {0} : {1}".format(key, value)
|
||||
outline = ' %s=' % key
|
||||
outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
|
||||
outline += '\n'
|
||||
mf.write(outline)
|
||||
|
||||
for key in keyorder:
|
||||
if key in build:
|
||||
write_builditem(key, build[key])
|
||||
for key, value in build.iteritems():
|
||||
if not key in keyorder:
|
||||
write_builditem(key, value)
|
||||
mf.write('\n')
|
||||
|
||||
if 'Maintainer Notes' in app:
|
||||
writefield('Maintainer Notes', '')
|
||||
for line in app['Maintainer Notes']:
|
||||
mf.write("%s\n" % line)
|
||||
mf.write('.\n')
|
||||
mf.write('\n')
|
||||
|
||||
|
||||
if app['Archive Policy']:
|
||||
writefield('Archive Policy')
|
||||
writefield('Auto Update Mode')
|
||||
writefield('Update Check Mode')
|
||||
if app['Vercode Operation']:
|
||||
writefield('Vercode Operation')
|
||||
if 'Update Check Data' in app:
|
||||
writefield('Update Check Data')
|
||||
if app['Current Version']:
|
||||
writefield('Current Version')
|
||||
writefield('Current Version Code')
|
||||
mf.write('\n')
|
||||
if app['No Source Since']:
|
||||
writefield('No Source Since')
|
||||
mf.write('\n')
|
||||
writecomments(None)
|
||||
mf.close()
|
||||
|
||||
|
||||
@ -26,7 +26,7 @@ import md5
|
||||
import glob
|
||||
from optparse import OptionParser
|
||||
|
||||
import common
|
||||
import common, metadata
|
||||
from common import BuildException
|
||||
|
||||
config = None
|
||||
@ -74,7 +74,7 @@ def main():
|
||||
# and b) a sane-looking ID that would make its way into the repo.
|
||||
# Nonetheless, to be sure, before publishing we check that there are no
|
||||
# collisions, and refuse to do any publishing if that's the case...
|
||||
apps = common.read_metadata()
|
||||
apps = metadata.read_metadata()
|
||||
allaliases = []
|
||||
for app in apps:
|
||||
m = md5.new()
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
import sys
|
||||
import os
|
||||
from optparse import OptionParser
|
||||
import common
|
||||
import common, metadata
|
||||
|
||||
config = None
|
||||
options = None
|
||||
@ -40,7 +40,7 @@ def main():
|
||||
config = common.read_config(options)
|
||||
|
||||
# Get all apps...
|
||||
apps = common.read_metadata(package=options.package, xref=False)
|
||||
apps = metadata.read_metadata(package=options.package, xref=False)
|
||||
|
||||
if len(apps) == 0 and options.package:
|
||||
print "No such package"
|
||||
@ -48,7 +48,7 @@ def main():
|
||||
|
||||
for app in apps:
|
||||
print "Writing " + app['id']
|
||||
common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
|
||||
metadata.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
|
||||
|
||||
print "Finished."
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@ import sys
|
||||
import os
|
||||
import traceback
|
||||
from optparse import OptionParser
|
||||
import common
|
||||
import common, metadata
|
||||
from common import BuildException
|
||||
from common import VCSException
|
||||
|
||||
@ -45,7 +45,7 @@ def main():
|
||||
config = common.read_config(options)
|
||||
|
||||
# Get all apps...
|
||||
apps = common.read_metadata()
|
||||
apps = metadata.read_metadata()
|
||||
|
||||
# Filter apps according to command-line options
|
||||
if options.package:
|
||||
|
||||
@ -25,7 +25,7 @@ import traceback
|
||||
import glob
|
||||
from optparse import OptionParser
|
||||
import paramiko
|
||||
import common
|
||||
import common, metadata
|
||||
import socket
|
||||
import subprocess
|
||||
|
||||
@ -58,7 +58,7 @@ def main():
|
||||
sys.exit(1)
|
||||
|
||||
# Get all metadata-defined apps...
|
||||
metaapps = common.read_metadata(options.verbose)
|
||||
metaapps = metadata.read_metadata(options.verbose)
|
||||
|
||||
statsdir = 'stats'
|
||||
logsdir = os.path.join(statsdir, 'logs')
|
||||
|
||||
@ -30,7 +30,7 @@ import pickle
|
||||
from xml.dom.minidom import Document
|
||||
from optparse import OptionParser
|
||||
import time
|
||||
import common
|
||||
import common, metadata
|
||||
from common import MetaDataException
|
||||
from PIL import Image
|
||||
|
||||
@ -75,11 +75,11 @@ def update_wiki(apps, apks):
|
||||
wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n"
|
||||
|
||||
wikidata += "=Description=\n"
|
||||
wikidata += common.description_wiki(app['Description']) + "\n"
|
||||
wikidata += metadata.description_wiki(app['Description']) + "\n"
|
||||
|
||||
wikidata += "=Maintainer Notes=\n"
|
||||
if 'Maintainer Notes' in app:
|
||||
wikidata += common.description_wiki(app['Maintainer Notes']) + "\n"
|
||||
wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n"
|
||||
wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id'])
|
||||
|
||||
# Get a list of all packages for this application...
|
||||
@ -232,7 +232,7 @@ def update_wiki(apps, apks):
|
||||
def delete_disabled_builds(apps, apkcache, repodirs):
|
||||
"""Delete disabled build outputs.
|
||||
|
||||
:param apps: list of all applications, as per common.read_metadata
|
||||
:param apps: list of all applications, as per metadata.read_metadata
|
||||
:param apkcache: current apk cache information
|
||||
:param repodirs: the repo directories to process
|
||||
"""
|
||||
@ -268,7 +268,7 @@ def resize_icon(iconpath):
|
||||
def resize_all_icons(repodirs):
|
||||
"""Resize all icons that exceed the max size
|
||||
|
||||
:param apps: list of all applications, as per common.read_metadata
|
||||
:param apps: list of all applications, as per metadata.read_metadata
|
||||
:param repodirs: the repo directories to process
|
||||
"""
|
||||
for repodir in repodirs:
|
||||
@ -280,7 +280,7 @@ def scan_apks(apps, apkcache, repodir, knownapks):
|
||||
|
||||
This also extracts the icons.
|
||||
|
||||
:param apps: list of all applications, as per common.read_metadata
|
||||
:param apps: list of all applications, as per metadata.read_metadata
|
||||
:param apkcache: current apk cache information
|
||||
:param repodir: repo directory to scan
|
||||
:param knownapks: known apks info
|
||||
@ -538,7 +538,7 @@ def make_index(apps, apks, repodir, archive, categories):
|
||||
return ("fdroid.app:" + link, app['Name'])
|
||||
raise MetaDataException("Cannot resolve app id " + link)
|
||||
addElement('desc',
|
||||
common.description_html(app['Description'], linkres), doc, apel)
|
||||
metadata.description_html(app['Description'], linkres), doc, apel)
|
||||
addElement('license', app['License'], doc, apel)
|
||||
if 'Categories' in app:
|
||||
appcategories = [c.strip() for c in app['Categories'].split(',')]
|
||||
@ -739,7 +739,7 @@ def main():
|
||||
sys.exit(0)
|
||||
|
||||
# Get all apps...
|
||||
apps = common.read_metadata()
|
||||
apps = metadata.read_metadata()
|
||||
|
||||
# Generate a list of categories...
|
||||
categories = []
|
||||
|
||||
Loading…
Reference in New Issue
Block a user