From fb38eb6b8cd7b2a5468a7b1932a443c83b8bca2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 19 Nov 2013 15:35:16 +0100 Subject: [PATCH 1/5] First metadata checks rewrite; New metadata.py module --- fdroidserver/build.py | 4 +- fdroidserver/checkupdates.py | 6 +- fdroidserver/common.py | 681 +-------------------------------- fdroidserver/import.py | 8 +- fdroidserver/metadata.py | 714 +++++++++++++++++++++++++++++++++++ fdroidserver/publish.py | 4 +- fdroidserver/rewritemeta.py | 6 +- fdroidserver/scanner.py | 4 +- fdroidserver/stats.py | 4 +- fdroidserver/update.py | 16 +- 10 files changed, 750 insertions(+), 697 deletions(-) create mode 100644 fdroidserver/metadata.py diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 7ea32217..114179d7 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -29,7 +29,7 @@ import time import json from optparse import OptionParser -import common +import common, metadata from common import BuildException, VCSException, FDroidPopen def get_builder_vm_id(): @@ -816,7 +816,7 @@ def main(): sys.exit(1) # Get all apps... - apps = common.read_metadata(xref=not options.onserver) + apps = metadata.read_metadata(xref=not options.onserver) log_dir = 'logs' if not os.path.isdir(log_dir): diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 0a417f73..97e0ac6e 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -28,7 +28,7 @@ from optparse import OptionParser import traceback import HTMLParser from distutils.version import LooseVersion -import common +import common, metadata from common import BuildException from common import VCSException @@ -295,7 +295,7 @@ def main(): config = common.read_config(options) # Get all apps... - apps = common.read_metadata(options.verbose) + apps = metadata.read_metadata(options.verbose) # Filter apps according to command-line options if options.package: @@ -453,7 +453,7 @@ def main(): if writeit: metafile = os.path.join('metadata', app['id'] + '.txt') - common.write_metadata(metafile, app) + metadata.write_metadata(metafile, app) if options.commit and logmsg: print "Commiting update for " + metafile gitcmd = ["git", "commit", "-m", diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 1033d17d..8a3298d5 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -23,20 +23,15 @@ import stat import subprocess import time import operator -import cgi import Queue import threading import magic +import metadata + config = None options = None -# These can only contain 'yes' or 'no' -bool_keys = ( - 'submodules', 'oldsdkloc', - 'forceversion', 'forcevercode', - 'fixtrans', 'fixapos', 'novcheck') - def read_config(opts, config_file='config.py'): """Read the repository config @@ -52,7 +47,7 @@ def read_config(opts, config_file='config.py'): sys.exit(2) st = os.stat(config_file) if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO: - print("WARNING: unsafe permissions on config.py (should be 0600)!") + print "WARNING: unsafe permissions on config.py (should be 0600)!" options = opts if not hasattr(options, 'verbose'): @@ -73,6 +68,11 @@ def read_config(opts, config_file='config.py'): execfile(config_file, config) return config +def getapkname(app, build): + return "%s_%s.apk" % (app['id'], build['vercode']) + +def getsrcname(app, build): + return "%s_%s_src.tar.gz" % (app['id'], build['vercode']) def getvcs(vcstype, remote, local): if vcstype == 'git': @@ -95,7 +95,7 @@ def getsrclibvcs(name): srclib_path = os.path.join('srclibs', name + ".txt") if not os.path.exists(srclib_path): raise VCSException("Missing srclib " + name) - return parse_srclib(srclib_path)['Repo Type'] + return metadata.parse_srclib(srclib_path)['Repo Type'] class vcs: def __init__(self, remote, local): @@ -457,625 +457,6 @@ class vcs_bzr(vcs): return [tag.split(' ')[0].strip() for tag in p.communicate()[0].splitlines()] - -# Get the type expected for a given metadata field. -def metafieldtype(name): - if name in ['Description', 'Maintainer Notes']: - return 'multiline' - if name == 'Requires Root': - return 'flag' - if name == 'Build Version': - return 'build' - if name == 'Build': - return 'buildv2' - if name == 'Use Built': - return 'obsolete' - return 'string' - - -# Parse metadata for a single application. -# -# 'metafile' - the filename to read. The package id for the application comes -# from this filename. Pass None to get a blank entry. -# -# Returns a dictionary containing all the details of the application. There are -# two major kinds of information in the dictionary. Keys beginning with capital -# letters correspond directory to identically named keys in the metadata file. -# Keys beginning with lower case letters are generated in one way or another, -# and are not found verbatim in the metadata. -# -# Known keys not originating from the metadata are: -# -# 'id' - the application's package ID -# 'builds' - a list of dictionaries containing build information -# for each defined build -# 'comments' - a list of comments from the metadata file. Each is -# a tuple of the form (field, comment) where field is -# the name of the field it preceded in the metadata -# file. Where field is None, the comment goes at the -# end of the file. Alternatively, 'build:version' is -# for a comment before a particular build version. -# 'descriptionlines' - original lines of description as formatted in the -# metadata file. -# -def parse_metadata(metafile): - - def parse_buildline(lines): - value = "".join(lines) - parts = [p.replace("\\,", ",") - for p in re.split(r"(? 20: - raise MetaDataException("Silly number of versions for archive policy") - except: - raise MetaDataException("Incomprehensible number of versions for archive policy") - - # Ensure all AntiFeatures are recognised... - if thisinfo['AntiFeatures']: - parts = thisinfo['AntiFeatures'].split(",") - for part in parts: - if (part != "Ads" and - part != "Tracking" and - part != "NonFreeNet" and - part != "NonFreeDep" and - part != "NonFreeAdd"): - raise MetaDataException("Unrecognised antifeature '" + part + "' in " \ - + metafile.name) - - return thisinfo - -def getvercode(build): - return "%s" % (build['vercode']) - -def getapkname(app, build): - return "%s_%s.apk" % (app['id'], getvercode(build)) - -def getsrcname(app, build): - return "%s_%s_src.tar.gz" % (app['id'], getvercode(build)) - -# Write a metadata file. -# -# 'dest' - The path to the output file -# 'app' - The app data -def write_metadata(dest, app): - - def writecomments(key): - written = 0 - for pf, comment in app['comments']: - if pf == key: - mf.write(comment + '\n') - written += 1 - if options.verbose and written > 0: - print "...writing comments for " + (key if key else 'EOF') - - def writefield(field, value=None): - writecomments(field) - if value is None: - value = app[field] - mf.write(field + ':' + value + '\n') - - mf = open(dest, 'w') - if app['Disabled']: - writefield('Disabled') - if app['AntiFeatures']: - writefield('AntiFeatures') - writefield('Categories') - writefield('License') - writefield('Web Site') - writefield('Source Code') - writefield('Issue Tracker') - if app['Donate']: - writefield('Donate') - if app['FlattrID']: - writefield('FlattrID') - if app['Bitcoin']: - writefield('Bitcoin') - if app['Litecoin']: - writefield('Litecoin') - mf.write('\n') - if app['Name']: - writefield('Name') - if app['Auto Name']: - writefield('Auto Name') - writefield('Summary') - writefield('Description', '') - for line in app['Description']: - mf.write(line + '\n') - mf.write('.\n') - mf.write('\n') - if app['Requires Root']: - writefield('Requires Root', 'Yes') - mf.write('\n') - if app['Repo Type']: - writefield('Repo Type') - writefield('Repo') - mf.write('\n') - for build in app['builds']: - writecomments('build:' + build['version']) - mf.write('Build:') - mf.write("%s,%s\n" % ( - build['version'], - getvercode(build))) - - # This defines the preferred order for the build items - as in the - # manual, they're roughly in order of application. - keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init', - 'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk', - 'update', 'encoding', 'forceversion', 'forcevercode', 'rm', - 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch', - 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni', - 'preassemble', 'bindir', 'antcommand', 'novcheck'] - - def write_builditem(key, value): - if key not in ['version', 'vercode', 'origlines']: - if key in bool_keys: - if not value: - return - value = 'yes' - if options.verbose: - print "...writing {0} : {1}".format(key, value) - outline = ' %s=' % key - outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')]) - outline += '\n' - mf.write(outline) - - for key in keyorder: - if key in build: - write_builditem(key, build[key]) - for key, value in build.iteritems(): - if not key in keyorder: - write_builditem(key, value) - mf.write('\n') - - if 'Maintainer Notes' in app: - writefield('Maintainer Notes', '') - for line in app['Maintainer Notes']: - mf.write(line + '\n') - mf.write('.\n') - mf.write('\n') - - - if app['Archive Policy']: - writefield('Archive Policy') - writefield('Auto Update Mode') - writefield('Update Check Mode') - if app['Vercode Operation']: - writefield('Vercode Operation') - if 'Update Check Data' in app: - writefield('Update Check Data') - if app['Current Version']: - writefield('Current Version') - writefield('Current Version Code') - mf.write('\n') - if app['No Source Since']: - writefield('No Source Since') - mf.write('\n') - writecomments(None) - mf.close() - - -# Read all metadata. Returns a list of 'app' objects (which are dictionaries as -# returned by the parse_metadata function. -def read_metadata(xref=True, package=None): - apps = [] - for basedir in ('metadata', 'tmp'): - if not os.path.exists(basedir): - os.makedirs(basedir) - for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))): - if package is None or metafile == os.path.join('metadata', package + '.txt'): - try: - appinfo = parse_metadata(metafile) - except Exception, e: - raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e))) - apps.append(appinfo) - - if xref: - # Parse all descriptions at load time, just to ensure cross-referencing - # errors are caught early rather than when they hit the build server. - def linkres(link): - for app in apps: - if app['id'] == link: - return ("fdroid.app:" + link, "Dummy name - don't know yet") - raise MetaDataException("Cannot resolve app id " + link) - for app in apps: - try: - description_html(app['Description'], linkres) - except Exception, e: - raise MetaDataException("Problem with description of " + app['id'] + - " - " + str(e)) - - return apps - -# Formatter for descriptions. Create an instance, and call parseline() with -# each line of the description source from the metadata. At the end, call -# end() and then text_plain, text_wiki and text_html will contain the result. -class DescriptionFormatter: - stNONE = 0 - stPARA = 1 - stUL = 2 - stOL = 3 - bold = False - ital = False - state = stNONE - text_plain = '' - text_wiki = '' - text_html = '' - linkResolver = None - def __init__(self, linkres): - self.linkResolver = linkres - def endcur(self, notstates=None): - if notstates and self.state in notstates: - return - if self.state == self.stPARA: - self.endpara() - elif self.state == self.stUL: - self.endul() - elif self.state == self.stOL: - self.endol() - def endpara(self): - self.text_plain += '\n' - self.text_html += '

' - self.state = self.stNONE - def endul(self): - self.text_html += '' - self.state = self.stNONE - def endol(self): - self.text_html += '' - self.state = self.stNONE - - def formatted(self, txt, html): - formatted = '' - if html: - txt = cgi.escape(txt) - while True: - index = txt.find("''") - if index == -1: - return formatted + txt - formatted += txt[:index] - txt = txt[index:] - if txt.startswith("'''"): - if html: - if self.bold: - formatted += '' - else: - formatted += '' - self.bold = not self.bold - txt = txt[3:] - else: - if html: - if self.ital: - formatted += '' - else: - formatted += '' - self.ital = not self.ital - txt = txt[2:] - - - def linkify(self, txt): - linkified_plain = '' - linkified_html = '' - while True: - index = txt.find("[") - if index == -1: - return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True)) - linkified_plain += self.formatted(txt[:index], False) - linkified_html += self.formatted(txt[:index], True) - txt = txt[index:] - if txt.startswith("[["): - index = txt.find("]]") - if index == -1: - raise MetaDataException("Unterminated ]]") - url = txt[2:index] - if self.linkResolver: - url, urltext = self.linkResolver(url) - else: - urltext = url - linkified_html += '' + cgi.escape(urltext) + '' - linkified_plain += urltext - txt = txt[index+2:] - else: - index = txt.find("]") - if index == -1: - raise MetaDataException("Unterminated ]") - url = txt[1:index] - index2 = url.find(' ') - if index2 == -1: - urltxt = url - else: - urltxt = url[index2 + 1:] - url = url[:index2] - linkified_html += '' + cgi.escape(urltxt) + '' - linkified_plain += urltxt - if urltxt != url: - linkified_plain += ' (' + url + ')' - txt = txt[index+1:] - - def addtext(self, txt): - p, h = self.linkify(txt) - self.text_plain += p - self.text_html += h - - def parseline(self, line): - self.text_wiki += line + '\n' - if not line: - self.endcur() - elif line.startswith('*'): - self.endcur([self.stUL]) - if self.state != self.stUL: - self.text_html += '
    ' - self.state = self.stUL - self.text_html += '
  • ' - self.text_plain += '*' - self.addtext(line[1:]) - self.text_html += '
    • ' - elif line.startswith('#'): - self.endcur([self.stOL]) - if self.state != self.stOL: - self.text_html += '
      ' - self.state = self.stOL - self.text_html += '
    1. ' - self.text_plain += '*' #TODO: lazy - put the numbers in! - self.addtext(line[1:]) - self.text_html += '
      2. ' - else: - self.endcur([self.stPARA]) - if self.state == self.stNONE: - self.text_html += '

      ' - self.state = self.stPARA - elif self.state == self.stPARA: - self.text_html += ' ' - self.text_plain += ' ' - self.addtext(line) - - def end(self): - self.endcur() - -# Parse multiple lines of description as written in a metadata file, returning -# a single string in plain text format. -def description_plain(lines, linkres): - ps = DescriptionFormatter(linkres) - for line in lines: - ps.parseline(line) - ps.end() - return ps.text_plain - -# Parse multiple lines of description as written in a metadata file, returning -# a single string in wiki format. Used for the Maintainer Notes field as well, -# because it's the same format. -def description_wiki(lines): - ps = DescriptionFormatter(None) - for line in lines: - ps.parseline(line) - ps.end() - return ps.text_wiki - -# Parse multiple lines of description as written in a metadata file, returning -# a single string in HTML format. -def description_html(lines,linkres): - ps = DescriptionFormatter(linkres) - for line in lines: - ps.parseline(line) - ps.end() - return ps.text_html - def retrieve_string(xml_dir, string): if not string.startswith('@string/'): return string.replace("\\'","'") @@ -1248,48 +629,6 @@ class VCSException(Exception): def __str__(self): return repr(self.value) -class MetaDataException(Exception): - def __init__(self, value): - self.value = value - - def __str__(self): - return repr(self.value) - -def parse_srclib(metafile, **kw): - - thisinfo = {} - if metafile and not isinstance(metafile, file): - metafile = open(metafile, "r") - - # Defaults for fields that come from metadata - thisinfo['Repo Type'] = '' - thisinfo['Repo'] = '' - thisinfo['Subdir'] = None - thisinfo['Prepare'] = None - thisinfo['Srclibs'] = None - thisinfo['Update Project'] = None - - if metafile is None: - return thisinfo - - for line in metafile: - line = line.rstrip('\r\n') - if not line or line.startswith("#"): - continue - - index = line.find(':') - if index == -1: - raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line) - field = line[:index] - value = line[index+1:] - - if field == "Subdir": - thisinfo[field] = value.split(',') - else: - thisinfo[field] = value - - return thisinfo - # Get the specified source library. # Returns the path to it. Normally this is the path to be used when referencing # it, which may be a subdirectory of the actual project. If you want the base @@ -1314,7 +653,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, basepath=False, if not os.path.exists(srclib_path): raise BuildException('srclib ' + name + ' not found.') - srclib = parse_srclib(srclib_path) + srclib = metadata.parse_srclib(srclib_path) sdir = os.path.join(srclib_dir, name) diff --git a/fdroidserver/import.py b/fdroidserver/import.py index cda1765c..3bb03681 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -22,7 +22,7 @@ import os import shutil import urllib from optparse import OptionParser -import common +import common, metadata # Get the repo type and address from the given web page. The page is scanned # in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and @@ -114,7 +114,7 @@ def main(): os.makedirs(tmp_dir) # Get all apps... - apps = common.read_metadata() + apps = metadata.read_metadata() # Figure out what kind of project it is... projecttype = None @@ -249,7 +249,7 @@ def main(): sys.exit(1) # Construct the metadata... - app = common.parse_metadata(None) + app = metadata.parse_metadata(None) app['id'] = package app['Web Site'] = website app['Source Code'] = sourcecode @@ -281,7 +281,7 @@ def main(): f.write(repotype + ' ' + repo) metafile = os.path.join('metadata', package + '.txt') - common.write_metadata(metafile, app) + metadata.write_metadata(metafile, app) print "Wrote " + metafile diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py new file mode 100644 index 00000000..a3aa9d91 --- /dev/null +++ b/fdroidserver/metadata.py @@ -0,0 +1,714 @@ +# -*- coding: utf-8 -*- +# +# common.py - part of the FDroid server tools +# Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com +# Copyright (C) 2013 Daniel Martí +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os, re, glob +import cgi + +class MetaDataException(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + +class FieldType(): + def __init__(self, name, matching, sep, fields, attrs): + self.name = name + if type(matching) is str: + self.matching = re.compile(matching) + elif type(matching) is list: + self.matching = matching + self.sep = sep + self.fields = fields + self.attrs = attrs + + def _assert_regex(self, values, appid): + for v in values: + if not self.matching.match(v): + raise MetaDataException("'%s' is not a valid %s in %s" + % (v, self.name, appid)) + + def _assert_list(self, values, appid): + for v in values: + if v not in self.matching: + raise MetaDataException("'%s' is not a valid %s in %s" + % (v, self.name, appid)) + + def check(self, value, appid): + if type(value) is not str or not value: + return + if self.sep is not None: + values = value.split(self.sep) + else: + values = [value] + if type(self.matching) is list: + self._assert_list(values, appid) + else: + self._assert_regex(values, appid) + + +valuetypes = { + 'int' : FieldType("Integer", + r'^[0-9]+$', None, + [ 'FlattrID' ], + [ 'vercode' ]), + + 'http' : FieldType("HTTP link", + r'^http[s]?://.+$', None, + [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []), + + 'bitcoin' : FieldType("Bitcoin address", + r'^[a-zA-Z0-9]{27,34}$', None, + [ "Bitcoin" ], + [ ]), + + 'litecoin' : FieldType("Litecoin address", + r'^[a-zA-Z0-9]{27,34}$', None, + [ "Bitcoin" ], + [ ]), + + 'bool' : FieldType("Boolean", + ['yes', 'no'], None, + [ ], + [ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode', + 'fixtrans', 'fixapos', 'novcheck' ]), + + 'Bool' : FieldType("Boolean", + ['Yes', 'No'], None, + [ "Requires Root" ], + [ ]), + + 'antifeatures' : FieldType("Anti-Feature", + [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd" ], ',', + [ "AntiFeatures" ], + [ ]), +} + +def check_metadata(info): + + # Generic fields and attributes + for k, t in valuetypes.iteritems(): + for field in [f for f in t.fields if f in info]: + t.check(info[field], info['id']) + if k == 'Bool': + info[field] = info[field] == "Yes" + for build in info['builds']: + for attr in [a for a in t.attrs if a in build]: + t.check(build[attr], info['id']) + if k == 'bool': + info[field] = info[field] == "yes" + + # Special fields + if info['Archive Policy']: + if not re.match(r'^[0-9]+ versions$', info['Archive Policy']): + raise MetaDataException("Invalid archive policy '%s' in %s" + % (info['Archive Policy'], info["id"])) + versions = int(info['Archive Policy'][:-9]) + if versions < 1 or versions > 20: + raise MetaDataException("Silly number of versions '%s' for archive policy in %s" + % (versions, info["id"])) + +# Formatter for descriptions. Create an instance, and call parseline() with +# each line of the description source from the metadata. At the end, call +# end() and then text_plain, text_wiki and text_html will contain the result. +class DescriptionFormatter: + stNONE = 0 + stPARA = 1 + stUL = 2 + stOL = 3 + bold = False + ital = False + state = stNONE + text_plain = '' + text_wiki = '' + text_html = '' + linkResolver = None + def __init__(self, linkres): + self.linkResolver = linkres + def endcur(self, notstates=None): + if notstates and self.state in notstates: + return + if self.state == self.stPARA: + self.endpara() + elif self.state == self.stUL: + self.endul() + elif self.state == self.stOL: + self.endol() + def endpara(self): + self.text_plain += '\n' + self.text_html += '

      ' + self.state = self.stNONE + def endul(self): + self.text_html += '
' + self.state = self.stNONE + def endol(self): + self.text_html += '' + self.state = self.stNONE + + def formatted(self, txt, html): + formatted = '' + if html: + txt = cgi.escape(txt) + while True: + index = txt.find("''") + if index == -1: + return formatted + txt + formatted += txt[:index] + txt = txt[index:] + if txt.startswith("'''"): + if html: + if self.bold: + formatted += '' + else: + formatted += '' + self.bold = not self.bold + txt = txt[3:] + else: + if html: + if self.ital: + formatted += '' + else: + formatted += '' + self.ital = not self.ital + txt = txt[2:] + + + def linkify(self, txt): + linkified_plain = '' + linkified_html = '' + while True: + index = txt.find("[") + if index == -1: + return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True)) + linkified_plain += self.formatted(txt[:index], False) + linkified_html += self.formatted(txt[:index], True) + txt = txt[index:] + if txt.startswith("[["): + index = txt.find("]]") + if index == -1: + raise MetaDataException("Unterminated ]]") + url = txt[2:index] + if self.linkResolver: + url, urltext = self.linkResolver(url) + else: + urltext = url + linkified_html += '' + cgi.escape(urltext) + '' + linkified_plain += urltext + txt = txt[index+2:] + else: + index = txt.find("]") + if index == -1: + raise MetaDataException("Unterminated ]") + url = txt[1:index] + index2 = url.find(' ') + if index2 == -1: + urltxt = url + else: + urltxt = url[index2 + 1:] + url = url[:index2] + linkified_html += '' + cgi.escape(urltxt) + '' + linkified_plain += urltxt + if urltxt != url: + linkified_plain += ' (' + url + ')' + txt = txt[index+1:] + + def addtext(self, txt): + p, h = self.linkify(txt) + self.text_plain += p + self.text_html += h + + def parseline(self, line): + self.text_wiki += "%s\n" % line + if not line: + self.endcur() + elif line.startswith('*'): + self.endcur([self.stUL]) + if self.state != self.stUL: + self.text_html += '
    ' + self.state = self.stUL + self.text_html += '
  • ' + self.text_plain += '*' + self.addtext(line[1:]) + self.text_html += '
    • ' + elif line.startswith('#'): + self.endcur([self.stOL]) + if self.state != self.stOL: + self.text_html += '
      ' + self.state = self.stOL + self.text_html += '
    1. ' + self.text_plain += '*' #TODO: lazy - put the numbers in! + self.addtext(line[1:]) + self.text_html += '
      2. ' + else: + self.endcur([self.stPARA]) + if self.state == self.stNONE: + self.text_html += '

      ' + self.state = self.stPARA + elif self.state == self.stPARA: + self.text_html += ' ' + self.text_plain += ' ' + self.addtext(line) + + def end(self): + self.endcur() + +# Parse multiple lines of description as written in a metadata file, returning +# a single string in plain text format. +def description_plain(lines, linkres): + ps = DescriptionFormatter(linkres) + for line in lines: + ps.parseline(line) + ps.end() + return ps.text_plain + +# Parse multiple lines of description as written in a metadata file, returning +# a single string in wiki format. Used for the Maintainer Notes field as well, +# because it's the same format. +def description_wiki(lines): + ps = DescriptionFormatter(None) + for line in lines: + ps.parseline(line) + ps.end() + return ps.text_wiki + +# Parse multiple lines of description as written in a metadata file, returning +# a single string in HTML format. +def description_html(lines,linkres): + ps = DescriptionFormatter(linkres) + for line in lines: + ps.parseline(line) + ps.end() + return ps.text_html + +def parse_srclib(metafile, **kw): + + thisinfo = {} + if metafile and not isinstance(metafile, file): + metafile = open(metafile, "r") + + # Defaults for fields that come from metadata + thisinfo['Repo Type'] = '' + thisinfo['Repo'] = '' + thisinfo['Subdir'] = None + thisinfo['Prepare'] = None + thisinfo['Srclibs'] = None + thisinfo['Update Project'] = None + + if metafile is None: + return thisinfo + + for line in metafile: + line = line.rstrip('\r\n') + if not line or line.startswith("#"): + continue + + index = line.find(':') + if index == -1: + raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line) + field = line[:index] + value = line[index+1:] + + if field == "Subdir": + thisinfo[field] = value.split(',') + else: + thisinfo[field] = value + + return thisinfo + +# Read all metadata. Returns a list of 'app' objects (which are dictionaries as +# returned by the parse_metadata function. +def read_metadata(xref=True, package=None): + apps = [] + for basedir in ('metadata', 'tmp'): + if not os.path.exists(basedir): + os.makedirs(basedir) + for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))): + if package is None or metafile == os.path.join('metadata', package + '.txt'): + try: + appinfo = parse_metadata(metafile) + except Exception, e: + raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e))) + check_metadata(appinfo) + apps.append(appinfo) + + if xref: + # Parse all descriptions at load time, just to ensure cross-referencing + # errors are caught early rather than when they hit the build server. + def linkres(link): + for app in apps: + if app['id'] == link: + return ("fdroid.app:" + link, "Dummy name - don't know yet") + raise MetaDataException("Cannot resolve app id " + link) + for app in apps: + try: + description_html(app['Description'], linkres) + except Exception, e: + raise MetaDataException("Problem with description of " + app['id'] + + " - " + str(e)) + + return apps + +# Get the type expected for a given metadata field. +def metafieldtype(name): + if name in ['Description', 'Maintainer Notes']: + return 'multiline' + if name == 'Build Version': + return 'build' + if name == 'Build': + return 'buildv2' + if name == 'Use Built': + return 'obsolete' + return 'string' + +# Parse metadata for a single application. +# +# 'metafile' - the filename to read. The package id for the application comes +# from this filename. Pass None to get a blank entry. +# +# Returns a dictionary containing all the details of the application. There are +# two major kinds of information in the dictionary. Keys beginning with capital +# letters correspond directory to identically named keys in the metadata file. +# Keys beginning with lower case letters are generated in one way or another, +# and are not found verbatim in the metadata. +# +# Known keys not originating from the metadata are: +# +# 'id' - the application's package ID +# 'builds' - a list of dictionaries containing build information +# for each defined build +# 'comments' - a list of comments from the metadata file. Each is +# a tuple of the form (field, comment) where field is +# the name of the field it preceded in the metadata +# file. Where field is None, the comment goes at the +# end of the file. Alternatively, 'build:version' is +# for a comment before a particular build version. +# 'descriptionlines' - original lines of description as formatted in the +# metadata file. +# +def parse_metadata(metafile): + + def parse_buildline(lines): + value = "".join(lines) + parts = [p.replace("\\,", ",") + for p in re.split(r"(? 0: + #print "...writing comments for " + (key if key else 'EOF') + + def writefield(field, value=None): + writecomments(field) + if value is None: + value = app[field] + mf.write("%s:%s\n" % (field, value)) + + mf = open(dest, 'w') + if app['Disabled']: + writefield('Disabled') + if app['AntiFeatures']: + writefield('AntiFeatures') + writefield('Categories') + writefield('License') + writefield('Web Site') + writefield('Source Code') + writefield('Issue Tracker') + if app['Donate']: + writefield('Donate') + if app['FlattrID']: + writefield('FlattrID') + if app['Bitcoin']: + writefield('Bitcoin') + if app['Litecoin']: + writefield('Litecoin') + mf.write('\n') + if app['Name']: + writefield('Name') + if app['Auto Name']: + writefield('Auto Name') + writefield('Summary') + writefield('Description', '') + for line in app['Description']: + mf.write("%s\n" % line) + mf.write('.\n') + mf.write('\n') + if app['Requires Root']: + writefield('Requires Root', 'Yes') + mf.write('\n') + if app['Repo Type']: + writefield('Repo Type') + writefield('Repo') + mf.write('\n') + for build in app['builds']: + writecomments('build:' + build['version']) + mf.write("Build:%s,%s\n" % ( build['version'], build['vercode'])) + + # This defines the preferred order for the build items - as in the + # manual, they're roughly in order of application. + keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init', + 'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk', + 'update', 'encoding', 'forceversion', 'forcevercode', 'rm', + 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch', + 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni', + 'preassemble', 'bindir', 'antcommand', 'novcheck'] + + def write_builditem(key, value): + if key not in ['version', 'vercode', 'origlines']: + if key in valuetypes['bool'].attrs: + if not value: + return + value = 'yes' + #if options.verbose: + #print "...writing {0} : {1}".format(key, value) + outline = ' %s=' % key + outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')]) + outline += '\n' + mf.write(outline) + + for key in keyorder: + if key in build: + write_builditem(key, build[key]) + for key, value in build.iteritems(): + if not key in keyorder: + write_builditem(key, value) + mf.write('\n') + + if 'Maintainer Notes' in app: + writefield('Maintainer Notes', '') + for line in app['Maintainer Notes']: + mf.write("%s\n" % line) + mf.write('.\n') + mf.write('\n') + + + if app['Archive Policy']: + writefield('Archive Policy') + writefield('Auto Update Mode') + writefield('Update Check Mode') + if app['Vercode Operation']: + writefield('Vercode Operation') + if 'Update Check Data' in app: + writefield('Update Check Data') + if app['Current Version']: + writefield('Current Version') + writefield('Current Version Code') + mf.write('\n') + if app['No Source Since']: + writefield('No Source Since') + mf.write('\n') + writecomments(None) + mf.close() + + diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 4d984916..8e824d26 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -26,7 +26,7 @@ import md5 import glob from optparse import OptionParser -import common +import common, metadata from common import BuildException config = None @@ -74,7 +74,7 @@ def main(): # and b) a sane-looking ID that would make its way into the repo. # Nonetheless, to be sure, before publishing we check that there are no # collisions, and refuse to do any publishing if that's the case... - apps = common.read_metadata() + apps = metadata.read_metadata() allaliases = [] for app in apps: m = md5.new() diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 0565e3d3..30dfb6e8 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -20,7 +20,7 @@ import sys import os from optparse import OptionParser -import common +import common, metadata config = None options = None @@ -40,7 +40,7 @@ def main(): config = common.read_config(options) # Get all apps... - apps = common.read_metadata(package=options.package, xref=False) + apps = metadata.read_metadata(package=options.package, xref=False) if len(apps) == 0 and options.package: print "No such package" @@ -48,7 +48,7 @@ def main(): for app in apps: print "Writing " + app['id'] - common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app) + metadata.write_metadata(os.path.join('metadata', app['id']) + '.txt', app) print "Finished." diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index a9498254..5fb7fc36 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -21,7 +21,7 @@ import sys import os import traceback from optparse import OptionParser -import common +import common, metadata from common import BuildException from common import VCSException @@ -45,7 +45,7 @@ def main(): config = common.read_config(options) # Get all apps... - apps = common.read_metadata() + apps = metadata.read_metadata() # Filter apps according to command-line options if options.package: diff --git a/fdroidserver/stats.py b/fdroidserver/stats.py index 311d835b..be62f650 100644 --- a/fdroidserver/stats.py +++ b/fdroidserver/stats.py @@ -25,7 +25,7 @@ import traceback import glob from optparse import OptionParser import paramiko -import common +import common, metadata import socket import subprocess @@ -58,7 +58,7 @@ def main(): sys.exit(1) # Get all metadata-defined apps... - metaapps = common.read_metadata(options.verbose) + metaapps = metadata.read_metadata(options.verbose) statsdir = 'stats' logsdir = os.path.join(statsdir, 'logs') diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 10e1a91d..e6203130 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -30,7 +30,7 @@ import pickle from xml.dom.minidom import Document from optparse import OptionParser import time -import common +import common, metadata from common import MetaDataException from PIL import Image @@ -75,11 +75,11 @@ def update_wiki(apps, apks): wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n" wikidata += "=Description=\n" - wikidata += common.description_wiki(app['Description']) + "\n" + wikidata += metadata.description_wiki(app['Description']) + "\n" wikidata += "=Maintainer Notes=\n" if 'Maintainer Notes' in app: - wikidata += common.description_wiki(app['Maintainer Notes']) + "\n" + wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n" wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id']) # Get a list of all packages for this application... @@ -232,7 +232,7 @@ def update_wiki(apps, apks): def delete_disabled_builds(apps, apkcache, repodirs): """Delete disabled build outputs. - :param apps: list of all applications, as per common.read_metadata + :param apps: list of all applications, as per metadata.read_metadata :param apkcache: current apk cache information :param repodirs: the repo directories to process """ @@ -268,7 +268,7 @@ def resize_icon(iconpath): def resize_all_icons(repodirs): """Resize all icons that exceed the max size - :param apps: list of all applications, as per common.read_metadata + :param apps: list of all applications, as per metadata.read_metadata :param repodirs: the repo directories to process """ for repodir in repodirs: @@ -280,7 +280,7 @@ def scan_apks(apps, apkcache, repodir, knownapks): This also extracts the icons. - :param apps: list of all applications, as per common.read_metadata + :param apps: list of all applications, as per metadata.read_metadata :param apkcache: current apk cache information :param repodir: repo directory to scan :param knownapks: known apks info @@ -538,7 +538,7 @@ def make_index(apps, apks, repodir, archive, categories): return ("fdroid.app:" + link, app['Name']) raise MetaDataException("Cannot resolve app id " + link) addElement('desc', - common.description_html(app['Description'], linkres), doc, apel) + metadata.description_html(app['Description'], linkres), doc, apel) addElement('license', app['License'], doc, apel) if 'Categories' in app: appcategories = [c.strip() for c in app['Categories'].split(',')] @@ -739,7 +739,7 @@ def main(): sys.exit(0) # Get all apps... - apps = common.read_metadata() + apps = metadata.read_metadata() # Generate a list of categories... categories = [] From fa3a591f15bd50f2824403f78d69c16c9d2a0cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 19 Nov 2013 15:38:47 +0100 Subject: [PATCH 2/5] Make Archive Policy a standard fieldtype too --- fdroidserver/metadata.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index a3aa9d91..f02079be 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -83,6 +83,11 @@ valuetypes = { [ "Bitcoin" ], [ ]), + 'archive' : FieldType("Archive Policy", + r'^[0-9]+ versions$', None, + [ "Archive Policy" ], + [ ]), + 'bool' : FieldType("Boolean", ['yes', 'no'], None, [ ], @@ -101,8 +106,6 @@ valuetypes = { } def check_metadata(info): - - # Generic fields and attributes for k, t in valuetypes.iteritems(): for field in [f for f in t.fields if f in info]: t.check(info[field], info['id']) @@ -114,16 +117,6 @@ def check_metadata(info): if k == 'bool': info[field] = info[field] == "yes" - # Special fields - if info['Archive Policy']: - if not re.match(r'^[0-9]+ versions$', info['Archive Policy']): - raise MetaDataException("Invalid archive policy '%s' in %s" - % (info['Archive Policy'], info["id"])) - versions = int(info['Archive Policy'][:-9]) - if versions < 1 or versions > 20: - raise MetaDataException("Silly number of versions '%s' for archive policy in %s" - % (versions, info["id"])) - # Formatter for descriptions. Create an instance, and call parseline() with # each line of the description source from the metadata. At the end, call # end() and then text_plain, text_wiki and text_html will contain the result. From 11f4165f374cee7e6fc6b7964bfd5fe304841d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 19 Nov 2013 15:47:53 +0100 Subject: [PATCH 3/5] Properly convert boolean strings to bools as before --- fdroidserver/metadata.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index f02079be..904dd134 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -107,15 +107,19 @@ valuetypes = { def check_metadata(info): for k, t in valuetypes.iteritems(): - for field in [f for f in t.fields if f in info]: - t.check(info[field], info['id']) - if k == 'Bool': - info[field] = info[field] == "Yes" + for field in t.fields: + if field in info: + t.check(info[field], info['id']) + if k == 'Bool': + info[field] = info[field] == "Yes" for build in info['builds']: - for attr in [a for a in t.attrs if a in build]: - t.check(build[attr], info['id']) - if k == 'bool': - info[field] = info[field] == "yes" + for attr in t.attrs: + if attr in build: + t.check(build[attr], info['id']) + if k == 'bool': + build[attr] = build[attr] == "yes" + elif k == 'bool': + build[attr] = False # Formatter for descriptions. Create an instance, and call parseline() with # each line of the description source from the metadata. At the end, call From f35d965d2a45128f3f46ee2266541bab9814ff55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 19 Nov 2013 16:04:35 +0100 Subject: [PATCH 4/5] Add inline python docs --- fdroidserver/metadata.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 904dd134..90ab9003 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -27,6 +27,14 @@ class MetaDataException(Exception): def __str__(self): return repr(self.value) +# Designates a metadata field type and checks that it matches +# +# 'name' - The long name of the field type +# 'matching' - List of possible values or regex expression +# 'sep' - Separator to use if value may be a list +# 'fields' - Metadata fields (Field:Value) of this type +# 'attrs' - Build attributes (attr=value) of this type +# class FieldType(): def __init__(self, name, matching, sep, fields, attrs): self.name = name @@ -63,6 +71,7 @@ class FieldType(): self._assert_regex(values, appid) +# Generic value types valuetypes = { 'int' : FieldType("Integer", r'^[0-9]+$', None, @@ -102,9 +111,10 @@ valuetypes = { 'antifeatures' : FieldType("Anti-Feature", [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd" ], ',', [ "AntiFeatures" ], - [ ]), + [ ]) } +# Check an app's metadata information for integrity errors def check_metadata(info): for k, t in valuetypes.iteritems(): for field in t.fields: From 7d20141957fac92a934143b41d5a1ee30a8d4959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 19 Nov 2013 16:11:58 +0100 Subject: [PATCH 5/5] Small fixes, no change in behaviour --- fdroidserver/metadata.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index 90ab9003..eb702085 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -79,7 +79,7 @@ valuetypes = { [ 'vercode' ]), 'http' : FieldType("HTTP link", - r'^http[s]?://.+$', None, + r'^http[s]?://', None, [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []), 'bitcoin' : FieldType("Bitcoin address", @@ -672,17 +672,18 @@ def write_metadata(dest, app): 'preassemble', 'bindir', 'antcommand', 'novcheck'] def write_builditem(key, value): - if key not in ['version', 'vercode', 'origlines']: - if key in valuetypes['bool'].attrs: - if not value: - return - value = 'yes' - #if options.verbose: - #print "...writing {0} : {1}".format(key, value) - outline = ' %s=' % key - outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')]) - outline += '\n' - mf.write(outline) + if key in ['version', 'vercode', 'origlines']: + return + if key in valuetypes['bool'].attrs: + if not value: + return + value = 'yes' + #if options.verbose: + #print "...writing {0} : {1}".format(key, value) + outline = ' %s=' % key + outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')]) + outline += '\n' + mf.write(outline) for key in keyorder: if key in build: