'
- self.state = self.stPARA
- elif self.state == self.stPARA:
- self.text_html += ' '
- self.text_plain += ' '
- self.addtext(line)
-
- def end(self):
- self.endcur()
-
-# Parse multiple lines of description as written in a metadata file, returning
-# a single string in plain text format.
-def description_plain(lines, linkres):
- ps = DescriptionFormatter(linkres)
- for line in lines:
- ps.parseline(line)
- ps.end()
- return ps.text_plain
-
-# Parse multiple lines of description as written in a metadata file, returning
-# a single string in wiki format. Used for the Maintainer Notes field as well,
-# because it's the same format.
-def description_wiki(lines):
- ps = DescriptionFormatter(None)
- for line in lines:
- ps.parseline(line)
- ps.end()
- return ps.text_wiki
-
-# Parse multiple lines of description as written in a metadata file, returning
-# a single string in HTML format.
-def description_html(lines,linkres):
- ps = DescriptionFormatter(linkres)
- for line in lines:
- ps.parseline(line)
- ps.end()
- return ps.text_html
-
def retrieve_string(xml_dir, string):
if not string.startswith('@string/'):
return string.replace("\\'","'")
@@ -1248,48 +629,6 @@ class VCSException(Exception):
def __str__(self):
return repr(self.value)
-class MetaDataException(Exception):
- def __init__(self, value):
- self.value = value
-
- def __str__(self):
- return repr(self.value)
-
-def parse_srclib(metafile, **kw):
-
- thisinfo = {}
- if metafile and not isinstance(metafile, file):
- metafile = open(metafile, "r")
-
- # Defaults for fields that come from metadata
- thisinfo['Repo Type'] = ''
- thisinfo['Repo'] = ''
- thisinfo['Subdir'] = None
- thisinfo['Prepare'] = None
- thisinfo['Srclibs'] = None
- thisinfo['Update Project'] = None
-
- if metafile is None:
- return thisinfo
-
- for line in metafile:
- line = line.rstrip('\r\n')
- if not line or line.startswith("#"):
- continue
-
- index = line.find(':')
- if index == -1:
- raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
- field = line[:index]
- value = line[index+1:]
-
- if field == "Subdir":
- thisinfo[field] = value.split(',')
- else:
- thisinfo[field] = value
-
- return thisinfo
-
# Get the specified source library.
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
@@ -1314,7 +653,7 @@ def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None, basepath=False,
if not os.path.exists(srclib_path):
raise BuildException('srclib ' + name + ' not found.')
- srclib = parse_srclib(srclib_path)
+ srclib = metadata.parse_srclib(srclib_path)
sdir = os.path.join(srclib_dir, name)
diff --git a/fdroidserver/import.py b/fdroidserver/import.py
index cda1765c..3bb03681 100644
--- a/fdroidserver/import.py
+++ b/fdroidserver/import.py
@@ -22,7 +22,7 @@ import os
import shutil
import urllib
from optparse import OptionParser
-import common
+import common, metadata
# Get the repo type and address from the given web page. The page is scanned
# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and
@@ -114,7 +114,7 @@ def main():
os.makedirs(tmp_dir)
# Get all apps...
- apps = common.read_metadata()
+ apps = metadata.read_metadata()
# Figure out what kind of project it is...
projecttype = None
@@ -249,7 +249,7 @@ def main():
sys.exit(1)
# Construct the metadata...
- app = common.parse_metadata(None)
+ app = metadata.parse_metadata(None)
app['id'] = package
app['Web Site'] = website
app['Source Code'] = sourcecode
@@ -281,7 +281,7 @@ def main():
f.write(repotype + ' ' + repo)
metafile = os.path.join('metadata', package + '.txt')
- common.write_metadata(metafile, app)
+ metadata.write_metadata(metafile, app)
print "Wrote " + metafile
diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py
new file mode 100644
index 00000000..eb702085
--- /dev/null
+++ b/fdroidserver/metadata.py
@@ -0,0 +1,722 @@
+# -*- coding: utf-8 -*-
+#
+# common.py - part of the FDroid server tools
+# Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2013 Daniel MartÃ
' + self.state = self.stPARA + elif self.state == self.stPARA: + self.text_html += ' ' + self.text_plain += ' ' + self.addtext(line) + + def end(self): + self.endcur() + +# Parse multiple lines of description as written in a metadata file, returning +# a single string in plain text format. +def description_plain(lines, linkres): + ps = DescriptionFormatter(linkres) + for line in lines: + ps.parseline(line) + ps.end() + return ps.text_plain + +# Parse multiple lines of description as written in a metadata file, returning +# a single string in wiki format. Used for the Maintainer Notes field as well, +# because it's the same format. +def description_wiki(lines): + ps = DescriptionFormatter(None) + for line in lines: + ps.parseline(line) + ps.end() + return ps.text_wiki + +# Parse multiple lines of description as written in a metadata file, returning +# a single string in HTML format. +def description_html(lines,linkres): + ps = DescriptionFormatter(linkres) + for line in lines: + ps.parseline(line) + ps.end() + return ps.text_html + +def parse_srclib(metafile, **kw): + + thisinfo = {} + if metafile and not isinstance(metafile, file): + metafile = open(metafile, "r") + + # Defaults for fields that come from metadata + thisinfo['Repo Type'] = '' + thisinfo['Repo'] = '' + thisinfo['Subdir'] = None + thisinfo['Prepare'] = None + thisinfo['Srclibs'] = None + thisinfo['Update Project'] = None + + if metafile is None: + return thisinfo + + for line in metafile: + line = line.rstrip('\r\n') + if not line or line.startswith("#"): + continue + + index = line.find(':') + if index == -1: + raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line) + field = line[:index] + value = line[index+1:] + + if field == "Subdir": + thisinfo[field] = value.split(',') + else: + thisinfo[field] = value + + return thisinfo + +# Read all metadata. Returns a list of 'app' objects (which are dictionaries as +# returned by the parse_metadata function. +def read_metadata(xref=True, package=None): + apps = [] + for basedir in ('metadata', 'tmp'): + if not os.path.exists(basedir): + os.makedirs(basedir) + for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))): + if package is None or metafile == os.path.join('metadata', package + '.txt'): + try: + appinfo = parse_metadata(metafile) + except Exception, e: + raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e))) + check_metadata(appinfo) + apps.append(appinfo) + + if xref: + # Parse all descriptions at load time, just to ensure cross-referencing + # errors are caught early rather than when they hit the build server. + def linkres(link): + for app in apps: + if app['id'] == link: + return ("fdroid.app:" + link, "Dummy name - don't know yet") + raise MetaDataException("Cannot resolve app id " + link) + for app in apps: + try: + description_html(app['Description'], linkres) + except Exception, e: + raise MetaDataException("Problem with description of " + app['id'] + + " - " + str(e)) + + return apps + +# Get the type expected for a given metadata field. +def metafieldtype(name): + if name in ['Description', 'Maintainer Notes']: + return 'multiline' + if name == 'Build Version': + return 'build' + if name == 'Build': + return 'buildv2' + if name == 'Use Built': + return 'obsolete' + return 'string' + +# Parse metadata for a single application. +# +# 'metafile' - the filename to read. The package id for the application comes +# from this filename. Pass None to get a blank entry. +# +# Returns a dictionary containing all the details of the application. There are +# two major kinds of information in the dictionary. Keys beginning with capital +# letters correspond directory to identically named keys in the metadata file. +# Keys beginning with lower case letters are generated in one way or another, +# and are not found verbatim in the metadata. +# +# Known keys not originating from the metadata are: +# +# 'id' - the application's package ID +# 'builds' - a list of dictionaries containing build information +# for each defined build +# 'comments' - a list of comments from the metadata file. Each is +# a tuple of the form (field, comment) where field is +# the name of the field it preceded in the metadata +# file. Where field is None, the comment goes at the +# end of the file. Alternatively, 'build:version' is +# for a comment before a particular build version. +# 'descriptionlines' - original lines of description as formatted in the +# metadata file. +# +def parse_metadata(metafile): + + def parse_buildline(lines): + value = "".join(lines) + parts = [p.replace("\\,", ",") + for p in re.split(r"(? 0: + #print "...writing comments for " + (key if key else 'EOF') + + def writefield(field, value=None): + writecomments(field) + if value is None: + value = app[field] + mf.write("%s:%s\n" % (field, value)) + + mf = open(dest, 'w') + if app['Disabled']: + writefield('Disabled') + if app['AntiFeatures']: + writefield('AntiFeatures') + writefield('Categories') + writefield('License') + writefield('Web Site') + writefield('Source Code') + writefield('Issue Tracker') + if app['Donate']: + writefield('Donate') + if app['FlattrID']: + writefield('FlattrID') + if app['Bitcoin']: + writefield('Bitcoin') + if app['Litecoin']: + writefield('Litecoin') + mf.write('\n') + if app['Name']: + writefield('Name') + if app['Auto Name']: + writefield('Auto Name') + writefield('Summary') + writefield('Description', '') + for line in app['Description']: + mf.write("%s\n" % line) + mf.write('.\n') + mf.write('\n') + if app['Requires Root']: + writefield('Requires Root', 'Yes') + mf.write('\n') + if app['Repo Type']: + writefield('Repo Type') + writefield('Repo') + mf.write('\n') + for build in app['builds']: + writecomments('build:' + build['version']) + mf.write("Build:%s,%s\n" % ( build['version'], build['vercode'])) + + # This defines the preferred order for the build items - as in the + # manual, they're roughly in order of application. + keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init', + 'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk', + 'update', 'encoding', 'forceversion', 'forcevercode', 'rm', + 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch', + 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni', + 'preassemble', 'bindir', 'antcommand', 'novcheck'] + + def write_builditem(key, value): + if key in ['version', 'vercode', 'origlines']: + return + if key in valuetypes['bool'].attrs: + if not value: + return + value = 'yes' + #if options.verbose: + #print "...writing {0} : {1}".format(key, value) + outline = ' %s=' % key + outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')]) + outline += '\n' + mf.write(outline) + + for key in keyorder: + if key in build: + write_builditem(key, build[key]) + for key, value in build.iteritems(): + if not key in keyorder: + write_builditem(key, value) + mf.write('\n') + + if 'Maintainer Notes' in app: + writefield('Maintainer Notes', '') + for line in app['Maintainer Notes']: + mf.write("%s\n" % line) + mf.write('.\n') + mf.write('\n') + + + if app['Archive Policy']: + writefield('Archive Policy') + writefield('Auto Update Mode') + writefield('Update Check Mode') + if app['Vercode Operation']: + writefield('Vercode Operation') + if 'Update Check Data' in app: + writefield('Update Check Data') + if app['Current Version']: + writefield('Current Version') + writefield('Current Version Code') + mf.write('\n') + if app['No Source Since']: + writefield('No Source Since') + mf.write('\n') + writecomments(None) + mf.close() + + diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py index 4d984916..8e824d26 100644 --- a/fdroidserver/publish.py +++ b/fdroidserver/publish.py @@ -26,7 +26,7 @@ import md5 import glob from optparse import OptionParser -import common +import common, metadata from common import BuildException config = None @@ -74,7 +74,7 @@ def main(): # and b) a sane-looking ID that would make its way into the repo. # Nonetheless, to be sure, before publishing we check that there are no # collisions, and refuse to do any publishing if that's the case... - apps = common.read_metadata() + apps = metadata.read_metadata() allaliases = [] for app in apps: m = md5.new() diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py index 0565e3d3..30dfb6e8 100644 --- a/fdroidserver/rewritemeta.py +++ b/fdroidserver/rewritemeta.py @@ -20,7 +20,7 @@ import sys import os from optparse import OptionParser -import common +import common, metadata config = None options = None @@ -40,7 +40,7 @@ def main(): config = common.read_config(options) # Get all apps... - apps = common.read_metadata(package=options.package, xref=False) + apps = metadata.read_metadata(package=options.package, xref=False) if len(apps) == 0 and options.package: print "No such package" @@ -48,7 +48,7 @@ def main(): for app in apps: print "Writing " + app['id'] - common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app) + metadata.write_metadata(os.path.join('metadata', app['id']) + '.txt', app) print "Finished." diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index a9498254..5fb7fc36 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -21,7 +21,7 @@ import sys import os import traceback from optparse import OptionParser -import common +import common, metadata from common import BuildException from common import VCSException @@ -45,7 +45,7 @@ def main(): config = common.read_config(options) # Get all apps... - apps = common.read_metadata() + apps = metadata.read_metadata() # Filter apps according to command-line options if options.package: diff --git a/fdroidserver/stats.py b/fdroidserver/stats.py index 311d835b..be62f650 100644 --- a/fdroidserver/stats.py +++ b/fdroidserver/stats.py @@ -25,7 +25,7 @@ import traceback import glob from optparse import OptionParser import paramiko -import common +import common, metadata import socket import subprocess @@ -58,7 +58,7 @@ def main(): sys.exit(1) # Get all metadata-defined apps... - metaapps = common.read_metadata(options.verbose) + metaapps = metadata.read_metadata(options.verbose) statsdir = 'stats' logsdir = os.path.join(statsdir, 'logs') diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 10e1a91d..e6203130 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -30,7 +30,7 @@ import pickle from xml.dom.minidom import Document from optparse import OptionParser import time -import common +import common, metadata from common import MetaDataException from PIL import Image @@ -75,11 +75,11 @@ def update_wiki(apps, apks): wikidata += " - [http://f-droid.org/repository/browse/?fdid=" + app['id'] + " view in repository]\n\n" wikidata += "=Description=\n" - wikidata += common.description_wiki(app['Description']) + "\n" + wikidata += metadata.description_wiki(app['Description']) + "\n" wikidata += "=Maintainer Notes=\n" if 'Maintainer Notes' in app: - wikidata += common.description_wiki(app['Maintainer Notes']) + "\n" + wikidata += metadata.description_wiki(app['Maintainer Notes']) + "\n" wikidata += "\nMetadata: [https://gitorious.org/f-droid/fdroiddata/source/master:metadata/{0}.txt current] [https://gitorious.org/f-droid/fdroiddata/history/metadata/{0}.txt history]\n".format(app['id']) # Get a list of all packages for this application... @@ -232,7 +232,7 @@ def update_wiki(apps, apks): def delete_disabled_builds(apps, apkcache, repodirs): """Delete disabled build outputs. - :param apps: list of all applications, as per common.read_metadata + :param apps: list of all applications, as per metadata.read_metadata :param apkcache: current apk cache information :param repodirs: the repo directories to process """ @@ -268,7 +268,7 @@ def resize_icon(iconpath): def resize_all_icons(repodirs): """Resize all icons that exceed the max size - :param apps: list of all applications, as per common.read_metadata + :param apps: list of all applications, as per metadata.read_metadata :param repodirs: the repo directories to process """ for repodir in repodirs: @@ -280,7 +280,7 @@ def scan_apks(apps, apkcache, repodir, knownapks): This also extracts the icons. - :param apps: list of all applications, as per common.read_metadata + :param apps: list of all applications, as per metadata.read_metadata :param apkcache: current apk cache information :param repodir: repo directory to scan :param knownapks: known apks info @@ -538,7 +538,7 @@ def make_index(apps, apks, repodir, archive, categories): return ("fdroid.app:" + link, app['Name']) raise MetaDataException("Cannot resolve app id " + link) addElement('desc', - common.description_html(app['Description'], linkres), doc, apel) + metadata.description_html(app['Description'], linkres), doc, apel) addElement('license', app['License'], doc, apel) if 'Categories' in app: appcategories = [c.strip() for c in app['Categories'].split(',')] @@ -739,7 +739,7 @@ def main(): sys.exit(0) # Get all apps... - apps = common.read_metadata() + apps = metadata.read_metadata() # Generate a list of categories... categories = []