mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-11 15:13:27 +02:00
400acdefab
By checking first, this prevents a stacktrace when the duplicate metadata file is not valid. For example, in the tests, the duplicate is just a zero length file, which was causing a stacktrace.
1394 lines
42 KiB
Python
1394 lines
42 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# metadata.py - part of the FDroid server tools
|
|
# Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
|
|
# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import glob
|
|
import cgi
|
|
import logging
|
|
import textwrap
|
|
import io
|
|
|
|
import yaml
|
|
# use libyaml if it is available
|
|
try:
|
|
from yaml import CLoader
|
|
YamlLoader = CLoader
|
|
except ImportError:
|
|
from yaml import Loader
|
|
YamlLoader = Loader
|
|
|
|
# use the C implementation when available
|
|
import xml.etree.cElementTree as ElementTree
|
|
|
|
import fdroidserver.common
|
|
|
|
srclibs = None
|
|
|
|
|
|
class MetaDataException(Exception):
|
|
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
def __str__(self):
|
|
return self.value
|
|
|
|
# To filter which ones should be written to the metadata files if
|
|
# present
|
|
app_fields = set([
|
|
'Disabled',
|
|
'AntiFeatures',
|
|
'Provides',
|
|
'Categories',
|
|
'License',
|
|
'Author Name',
|
|
'Author Email',
|
|
'Web Site',
|
|
'Source Code',
|
|
'Issue Tracker',
|
|
'Changelog',
|
|
'Donate',
|
|
'FlattrID',
|
|
'Bitcoin',
|
|
'Litecoin',
|
|
'Name',
|
|
'Auto Name',
|
|
'Summary',
|
|
'Description',
|
|
'Requires Root',
|
|
'Repo Type',
|
|
'Repo',
|
|
'Binaries',
|
|
'Maintainer Notes',
|
|
'Archive Policy',
|
|
'Auto Update Mode',
|
|
'Update Check Mode',
|
|
'Update Check Ignore',
|
|
'Vercode Operation',
|
|
'Update Check Name',
|
|
'Update Check Data',
|
|
'Current Version',
|
|
'Current Version Code',
|
|
'No Source Since',
|
|
|
|
'comments', # For formats that don't do inline comments
|
|
'builds', # For formats that do builds as a list
|
|
])
|
|
|
|
|
|
class App():
|
|
|
|
def __init__(self):
|
|
self.Disabled = None
|
|
self.AntiFeatures = []
|
|
self.Provides = None
|
|
self.Categories = ['None']
|
|
self.License = 'Unknown'
|
|
self.AuthorName = None
|
|
self.AuthorEmail = None
|
|
self.WebSite = ''
|
|
self.SourceCode = ''
|
|
self.IssueTracker = ''
|
|
self.Changelog = ''
|
|
self.Donate = None
|
|
self.FlattrID = None
|
|
self.Bitcoin = None
|
|
self.Litecoin = None
|
|
self.Name = None
|
|
self.AutoName = ''
|
|
self.Summary = ''
|
|
self.Description = ''
|
|
self.RequiresRoot = False
|
|
self.RepoType = ''
|
|
self.Repo = ''
|
|
self.Binaries = None
|
|
self.MaintainerNotes = ''
|
|
self.ArchivePolicy = None
|
|
self.AutoUpdateMode = 'None'
|
|
self.UpdateCheckMode = 'None'
|
|
self.UpdateCheckIgnore = None
|
|
self.VercodeOperation = None
|
|
self.UpdateCheckName = None
|
|
self.UpdateCheckData = None
|
|
self.CurrentVersion = ''
|
|
self.CurrentVersionCode = None
|
|
self.NoSourceSince = ''
|
|
|
|
self.id = None
|
|
self.metadatapath = None
|
|
self.builds = []
|
|
self.comments = {}
|
|
self.added = None
|
|
self.lastupdated = None
|
|
self._modified = set()
|
|
|
|
# Translates human-readable field names to attribute names, e.g.
|
|
# 'Auto Name' to 'AutoName'
|
|
@classmethod
|
|
def field_to_attr(cls, f):
|
|
return f.replace(' ', '')
|
|
|
|
# Translates attribute names to human-readable field names, e.g.
|
|
# 'AutoName' to 'Auto Name'
|
|
@classmethod
|
|
def attr_to_field(cls, k):
|
|
if k in app_fields:
|
|
return k
|
|
f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
|
|
return f
|
|
|
|
# Constructs an old-fashioned dict with the human-readable field
|
|
# names. Should only be used for tests.
|
|
def field_dict(self):
|
|
d = {}
|
|
for k, v in self.__dict__.items():
|
|
if k == 'builds':
|
|
d['builds'] = []
|
|
for build in v:
|
|
b = {k: v for k, v in build.__dict__.items() if not k.startswith('_')}
|
|
d['builds'].append(b)
|
|
elif not k.startswith('_'):
|
|
f = App.attr_to_field(k)
|
|
d[f] = v
|
|
return d
|
|
|
|
# Gets the value associated to a field name, e.g. 'Auto Name'
|
|
def get_field(self, f):
|
|
if f not in app_fields:
|
|
raise MetaDataException('Unrecognised app field: ' + f)
|
|
k = App.field_to_attr(f)
|
|
return getattr(self, k)
|
|
|
|
# Sets the value associated to a field name, e.g. 'Auto Name'
|
|
def set_field(self, f, v):
|
|
if f not in app_fields:
|
|
raise MetaDataException('Unrecognised app field: ' + f)
|
|
k = App.field_to_attr(f)
|
|
self.__dict__[k] = v
|
|
self._modified.add(k)
|
|
|
|
# Appends to the value associated to a field name, e.g. 'Auto Name'
|
|
def append_field(self, f, v):
|
|
if f not in app_fields:
|
|
raise MetaDataException('Unrecognised app field: ' + f)
|
|
k = App.field_to_attr(f)
|
|
if k not in self.__dict__:
|
|
self.__dict__[k] = [v]
|
|
else:
|
|
self.__dict__[k].append(v)
|
|
|
|
# Like dict.update(), but using human-readable field names
|
|
def update_fields(self, d):
|
|
for f, v in d.items():
|
|
if f == 'builds':
|
|
for b in v:
|
|
build = Build()
|
|
build.update_flags(b)
|
|
self.builds.append(build)
|
|
else:
|
|
self.set_field(f, v)
|
|
|
|
TYPE_UNKNOWN = 0
|
|
TYPE_OBSOLETE = 1
|
|
TYPE_STRING = 2
|
|
TYPE_BOOL = 3
|
|
TYPE_LIST = 4
|
|
TYPE_SCRIPT = 5
|
|
TYPE_MULTILINE = 6
|
|
TYPE_BUILD = 7
|
|
TYPE_BUILD_V2 = 8
|
|
|
|
fieldtypes = {
|
|
'Description': TYPE_MULTILINE,
|
|
'Maintainer Notes': TYPE_MULTILINE,
|
|
'Categories': TYPE_LIST,
|
|
'AntiFeatures': TYPE_LIST,
|
|
'Build Version': TYPE_BUILD,
|
|
'Build': TYPE_BUILD_V2,
|
|
'Use Built': TYPE_OBSOLETE,
|
|
}
|
|
|
|
|
|
def fieldtype(name):
|
|
if name in fieldtypes:
|
|
return fieldtypes[name]
|
|
return TYPE_STRING
|
|
|
|
|
|
# In the order in which they are laid out on files
|
|
build_flags_order = [
|
|
'disable',
|
|
'commit',
|
|
'subdir',
|
|
'submodules',
|
|
'init',
|
|
'patch',
|
|
'gradle',
|
|
'maven',
|
|
'kivy',
|
|
'output',
|
|
'srclibs',
|
|
'oldsdkloc',
|
|
'encoding',
|
|
'forceversion',
|
|
'forcevercode',
|
|
'rm',
|
|
'extlibs',
|
|
'prebuild',
|
|
'update',
|
|
'target',
|
|
'scanignore',
|
|
'scandelete',
|
|
'build',
|
|
'buildjni',
|
|
'ndk',
|
|
'preassemble',
|
|
'gradleprops',
|
|
'antcommands',
|
|
'novcheck',
|
|
]
|
|
|
|
|
|
build_flags = set(build_flags_order + ['version', 'vercode'])
|
|
|
|
|
|
class Build():
|
|
|
|
def __init__(self):
|
|
self.disable = False
|
|
self.commit = None
|
|
self.subdir = None
|
|
self.submodules = False
|
|
self.init = ''
|
|
self.patch = []
|
|
self.gradle = []
|
|
self.maven = False
|
|
self.kivy = False
|
|
self.output = None
|
|
self.srclibs = []
|
|
self.oldsdkloc = False
|
|
self.encoding = None
|
|
self.forceversion = False
|
|
self.forcevercode = False
|
|
self.rm = []
|
|
self.extlibs = []
|
|
self.prebuild = ''
|
|
self.update = []
|
|
self.target = None
|
|
self.scanignore = []
|
|
self.scandelete = []
|
|
self.build = ''
|
|
self.buildjni = []
|
|
self.ndk = None
|
|
self.preassemble = []
|
|
self.gradleprops = []
|
|
self.antcommands = []
|
|
self.novcheck = False
|
|
|
|
self._modified = set()
|
|
|
|
def get_flag(self, f):
|
|
if f not in build_flags:
|
|
raise MetaDataException('Unrecognised build flag: ' + f)
|
|
return getattr(self, f)
|
|
|
|
def set_flag(self, f, v):
|
|
if f == 'versionName':
|
|
f = 'version'
|
|
if f == 'versionCode':
|
|
f = 'vercode'
|
|
if f not in build_flags:
|
|
raise MetaDataException('Unrecognised build flag: ' + f)
|
|
self.__dict__[f] = v
|
|
self._modified.add(f)
|
|
|
|
def append_flag(self, f, v):
|
|
if f not in build_flags:
|
|
raise MetaDataException('Unrecognised build flag: ' + f)
|
|
if f not in self.__dict__:
|
|
self.__dict__[f] = [v]
|
|
else:
|
|
self.__dict__[f].append(v)
|
|
|
|
def build_method(self):
|
|
for f in ['maven', 'gradle', 'kivy']:
|
|
if self.get_flag(f):
|
|
return f
|
|
if self.output:
|
|
return 'raw'
|
|
return 'ant'
|
|
|
|
# like build_method, but prioritize output=
|
|
def output_method(self):
|
|
if self.output:
|
|
return 'raw'
|
|
for f in ['maven', 'gradle', 'kivy']:
|
|
if self.get_flag(f):
|
|
return f
|
|
return 'ant'
|
|
|
|
def ndk_path(self):
|
|
version = self.ndk
|
|
if not version:
|
|
version = 'r10e' # falls back to latest
|
|
paths = fdroidserver.common.config['ndk_paths']
|
|
if version not in paths:
|
|
return ''
|
|
return paths[version]
|
|
|
|
def update_flags(self, d):
|
|
for f, v in d.items():
|
|
self.set_flag(f, v)
|
|
|
|
flagtypes = {
|
|
'extlibs': TYPE_LIST,
|
|
'srclibs': TYPE_LIST,
|
|
'patch': TYPE_LIST,
|
|
'rm': TYPE_LIST,
|
|
'buildjni': TYPE_LIST,
|
|
'preassemble': TYPE_LIST,
|
|
'update': TYPE_LIST,
|
|
'scanignore': TYPE_LIST,
|
|
'scandelete': TYPE_LIST,
|
|
'gradle': TYPE_LIST,
|
|
'antcommands': TYPE_LIST,
|
|
'gradleprops': TYPE_LIST,
|
|
'init': TYPE_SCRIPT,
|
|
'prebuild': TYPE_SCRIPT,
|
|
'build': TYPE_SCRIPT,
|
|
'submodules': TYPE_BOOL,
|
|
'oldsdkloc': TYPE_BOOL,
|
|
'forceversion': TYPE_BOOL,
|
|
'forcevercode': TYPE_BOOL,
|
|
'novcheck': TYPE_BOOL,
|
|
}
|
|
|
|
|
|
def flagtype(name):
|
|
if name in flagtypes:
|
|
return flagtypes[name]
|
|
return TYPE_STRING
|
|
|
|
|
|
# Designates a metadata field type and checks that it matches
|
|
#
|
|
# 'name' - The long name of the field type
|
|
# 'matching' - List of possible values or regex expression
|
|
# 'sep' - Separator to use if value may be a list
|
|
# 'fields' - Metadata fields (Field:Value) of this type
|
|
# 'flags' - Build flags (flag=value) of this type
|
|
#
|
|
class FieldValidator():
|
|
|
|
def __init__(self, name, matching, fields, flags):
|
|
self.name = name
|
|
self.matching = matching
|
|
self.compiled = re.compile(matching)
|
|
self.fields = fields
|
|
self.flags = flags
|
|
|
|
def check(self, v, appid):
|
|
if not v:
|
|
return
|
|
if type(v) == list:
|
|
values = v
|
|
else:
|
|
values = [v]
|
|
for v in values:
|
|
if not self.compiled.match(v):
|
|
raise MetaDataException("'%s' is not a valid %s in %s. Regex pattern: %s"
|
|
% (v, self.name, appid, self.matching))
|
|
|
|
# Generic value types
|
|
valuetypes = {
|
|
FieldValidator("Integer",
|
|
r'^[1-9][0-9]*$',
|
|
[],
|
|
['vercode']),
|
|
|
|
FieldValidator("Hexadecimal",
|
|
r'^[0-9a-f]+$',
|
|
['FlattrID'],
|
|
[]),
|
|
|
|
FieldValidator("HTTP link",
|
|
r'^http[s]?://',
|
|
["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"], []),
|
|
|
|
FieldValidator("Email",
|
|
r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
|
|
["AuthorEmail"], []),
|
|
|
|
FieldValidator("Bitcoin address",
|
|
r'^[a-zA-Z0-9]{27,34}$',
|
|
["Bitcoin"],
|
|
[]),
|
|
|
|
FieldValidator("Litecoin address",
|
|
r'^L[a-zA-Z0-9]{33}$',
|
|
["Litecoin"],
|
|
[]),
|
|
|
|
FieldValidator("Repo Type",
|
|
r'^(git|git-svn|svn|hg|bzr|srclib)$',
|
|
["RepoType"],
|
|
[]),
|
|
|
|
FieldValidator("Binaries",
|
|
r'^http[s]?://',
|
|
["Binaries"],
|
|
[]),
|
|
|
|
FieldValidator("Archive Policy",
|
|
r'^[0-9]+ versions$',
|
|
["ArchivePolicy"],
|
|
[]),
|
|
|
|
FieldValidator("Anti-Feature",
|
|
r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets)$',
|
|
["AntiFeatures"],
|
|
[]),
|
|
|
|
FieldValidator("Auto Update Mode",
|
|
r"^(Version .+|None)$",
|
|
["AutoUpdateMode"],
|
|
[]),
|
|
|
|
FieldValidator("Update Check Mode",
|
|
r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
|
|
["UpdateCheckMode"],
|
|
[])
|
|
}
|
|
|
|
|
|
# Check an app's metadata information for integrity errors
|
|
def check_metadata(app):
|
|
for v in valuetypes:
|
|
for k in v.fields:
|
|
if k not in app._modified:
|
|
continue
|
|
v.check(app.__dict__[k], app.id)
|
|
for build in app.builds:
|
|
for k in v.flags:
|
|
if k not in build._modified:
|
|
continue
|
|
v.check(build.__dict__[k], app.id)
|
|
|
|
|
|
# Formatter for descriptions. Create an instance, and call parseline() with
|
|
# each line of the description source from the metadata. At the end, call
|
|
# end() and then text_txt and text_html will contain the result.
|
|
class DescriptionFormatter:
|
|
|
|
stNONE = 0
|
|
stPARA = 1
|
|
stUL = 2
|
|
stOL = 3
|
|
|
|
def __init__(self, linkres):
|
|
self.bold = False
|
|
self.ital = False
|
|
self.state = self.stNONE
|
|
self.laststate = self.stNONE
|
|
self.text_html = ''
|
|
self.text_txt = ''
|
|
self.html = io.StringIO()
|
|
self.text = io.StringIO()
|
|
self.para_lines = []
|
|
self.linkResolver = None
|
|
self.linkResolver = linkres
|
|
|
|
def endcur(self, notstates=None):
|
|
if notstates and self.state in notstates:
|
|
return
|
|
if self.state == self.stPARA:
|
|
self.endpara()
|
|
elif self.state == self.stUL:
|
|
self.endul()
|
|
elif self.state == self.stOL:
|
|
self.endol()
|
|
|
|
def endpara(self):
|
|
self.laststate = self.state
|
|
self.state = self.stNONE
|
|
whole_para = ' '.join(self.para_lines)
|
|
self.addtext(whole_para)
|
|
wrapped = textwrap.fill(whole_para, 80,
|
|
break_long_words=False,
|
|
break_on_hyphens=False)
|
|
self.text.write(wrapped)
|
|
self.html.write('</p>')
|
|
del self.para_lines[:]
|
|
|
|
def endul(self):
|
|
self.html.write('</ul>')
|
|
self.laststate = self.state
|
|
self.state = self.stNONE
|
|
|
|
def endol(self):
|
|
self.html.write('</ol>')
|
|
self.laststate = self.state
|
|
self.state = self.stNONE
|
|
|
|
def formatted(self, txt, html):
|
|
res = ''
|
|
if html:
|
|
txt = cgi.escape(txt)
|
|
while True:
|
|
index = txt.find("''")
|
|
if index == -1:
|
|
return res + txt
|
|
res += txt[:index]
|
|
txt = txt[index:]
|
|
if txt.startswith("'''"):
|
|
if html:
|
|
if self.bold:
|
|
res += '</b>'
|
|
else:
|
|
res += '<b>'
|
|
self.bold = not self.bold
|
|
txt = txt[3:]
|
|
else:
|
|
if html:
|
|
if self.ital:
|
|
res += '</i>'
|
|
else:
|
|
res += '<i>'
|
|
self.ital = not self.ital
|
|
txt = txt[2:]
|
|
|
|
def linkify(self, txt):
|
|
res_plain = ''
|
|
res_html = ''
|
|
while True:
|
|
index = txt.find("[")
|
|
if index == -1:
|
|
return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True))
|
|
res_plain += self.formatted(txt[:index], False)
|
|
res_html += self.formatted(txt[:index], True)
|
|
txt = txt[index:]
|
|
if txt.startswith("[["):
|
|
index = txt.find("]]")
|
|
if index == -1:
|
|
raise MetaDataException("Unterminated ]]")
|
|
url = txt[2:index]
|
|
if self.linkResolver:
|
|
url, urltext = self.linkResolver(url)
|
|
else:
|
|
urltext = url
|
|
res_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
|
|
res_plain += urltext
|
|
txt = txt[index + 2:]
|
|
else:
|
|
index = txt.find("]")
|
|
if index == -1:
|
|
raise MetaDataException("Unterminated ]")
|
|
url = txt[1:index]
|
|
index2 = url.find(' ')
|
|
if index2 == -1:
|
|
urltxt = url
|
|
else:
|
|
urltxt = url[index2 + 1:]
|
|
url = url[:index2]
|
|
if url == urltxt:
|
|
raise MetaDataException("Url title is just the URL - use [url]")
|
|
res_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
|
|
res_plain += urltxt
|
|
if urltxt != url:
|
|
res_plain += ' (' + url + ')'
|
|
txt = txt[index + 1:]
|
|
|
|
def addtext(self, txt):
|
|
p, h = self.linkify(txt)
|
|
self.html.write(h)
|
|
|
|
def parseline(self, line):
|
|
if not line:
|
|
self.endcur()
|
|
elif line.startswith('* '):
|
|
self.endcur([self.stUL])
|
|
if self.state != self.stUL:
|
|
self.html.write('<ul>')
|
|
self.state = self.stUL
|
|
if self.laststate != self.stNONE:
|
|
self.text.write('\n\n')
|
|
else:
|
|
self.text.write('\n')
|
|
self.text.write(line)
|
|
self.html.write('<li>')
|
|
self.addtext(line[1:])
|
|
self.html.write('</li>')
|
|
elif line.startswith('# '):
|
|
self.endcur([self.stOL])
|
|
if self.state != self.stOL:
|
|
self.html.write('<ol>')
|
|
self.state = self.stOL
|
|
if self.laststate != self.stNONE:
|
|
self.text.write('\n\n')
|
|
else:
|
|
self.text.write('\n')
|
|
self.text.write(line)
|
|
self.html.write('<li>')
|
|
self.addtext(line[1:])
|
|
self.html.write('</li>')
|
|
else:
|
|
self.para_lines.append(line)
|
|
self.endcur([self.stPARA])
|
|
if self.state == self.stNONE:
|
|
self.state = self.stPARA
|
|
if self.laststate != self.stNONE:
|
|
self.text.write('\n\n')
|
|
self.html.write('<p>')
|
|
|
|
def end(self):
|
|
self.endcur()
|
|
self.text_txt = self.text.getvalue()
|
|
self.text_html = self.html.getvalue()
|
|
self.text.close()
|
|
self.html.close()
|
|
|
|
|
|
# Parse multiple lines of description as written in a metadata file, returning
|
|
# a single string in text format and wrapped to 80 columns.
|
|
def description_txt(s):
|
|
ps = DescriptionFormatter(None)
|
|
for line in s.splitlines():
|
|
ps.parseline(line)
|
|
ps.end()
|
|
return ps.text_txt
|
|
|
|
|
|
# Parse multiple lines of description as written in a metadata file, returning
|
|
# a single string in wiki format. Used for the Maintainer Notes field as well,
|
|
# because it's the same format.
|
|
def description_wiki(s):
|
|
return s
|
|
|
|
|
|
# Parse multiple lines of description as written in a metadata file, returning
|
|
# a single string in HTML format.
|
|
def description_html(s, linkres):
|
|
ps = DescriptionFormatter(linkres)
|
|
for line in s.splitlines():
|
|
ps.parseline(line)
|
|
ps.end()
|
|
return ps.text_html
|
|
|
|
|
|
def parse_srclib(metadatapath):
|
|
|
|
thisinfo = {}
|
|
|
|
# Defaults for fields that come from metadata
|
|
thisinfo['Repo Type'] = ''
|
|
thisinfo['Repo'] = ''
|
|
thisinfo['Subdir'] = None
|
|
thisinfo['Prepare'] = None
|
|
|
|
if not os.path.exists(metadatapath):
|
|
return thisinfo
|
|
|
|
metafile = open(metadatapath, "r", encoding='utf-8')
|
|
|
|
n = 0
|
|
for line in metafile:
|
|
n += 1
|
|
line = line.rstrip('\r\n')
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
|
|
try:
|
|
f, v = line.split(':', 1)
|
|
except ValueError:
|
|
raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
|
|
|
|
if f == "Subdir":
|
|
thisinfo[f] = v.split(',')
|
|
else:
|
|
thisinfo[f] = v
|
|
|
|
metafile.close()
|
|
|
|
return thisinfo
|
|
|
|
|
|
def read_srclibs():
|
|
"""Read all srclib metadata.
|
|
|
|
The information read will be accessible as metadata.srclibs, which is a
|
|
dictionary, keyed on srclib name, with the values each being a dictionary
|
|
in the same format as that returned by the parse_srclib function.
|
|
|
|
A MetaDataException is raised if there are any problems with the srclib
|
|
metadata.
|
|
"""
|
|
global srclibs
|
|
|
|
# They were already loaded
|
|
if srclibs is not None:
|
|
return
|
|
|
|
srclibs = {}
|
|
|
|
srcdir = 'srclibs'
|
|
if not os.path.exists(srcdir):
|
|
os.makedirs(srcdir)
|
|
|
|
for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
|
|
srclibname = os.path.basename(metadatapath[:-4])
|
|
srclibs[srclibname] = parse_srclib(metadatapath)
|
|
|
|
|
|
# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
|
|
# returned by the parse_txt_metadata function.
|
|
def read_metadata(xref=True):
|
|
|
|
# Always read the srclibs before the apps, since they can use a srlib as
|
|
# their source repository.
|
|
read_srclibs()
|
|
|
|
apps = {}
|
|
|
|
for basedir in ('metadata', 'tmp'):
|
|
if not os.path.exists(basedir):
|
|
os.makedirs(basedir)
|
|
|
|
# If there are multiple metadata files for a single appid, then the first
|
|
# file that is parsed wins over all the others, and the rest throw an
|
|
# exception. So the original .txt format is parsed first, at least until
|
|
# newer formats stabilize.
|
|
|
|
for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt'))
|
|
+ glob.glob(os.path.join('metadata', '*.json'))
|
|
+ glob.glob(os.path.join('metadata', '*.xml'))
|
|
+ glob.glob(os.path.join('metadata', '*.yml'))
|
|
+ glob.glob('.fdroid.json')
|
|
+ glob.glob('.fdroid.xml')
|
|
+ glob.glob('.fdroid.yml')):
|
|
packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
|
|
if packageName in apps:
|
|
raise MetaDataException("Found multiple metadata files for " + packageName)
|
|
app = parse_metadata(metadatapath)
|
|
check_metadata(app)
|
|
apps[app.id] = app
|
|
|
|
if xref:
|
|
# Parse all descriptions at load time, just to ensure cross-referencing
|
|
# errors are caught early rather than when they hit the build server.
|
|
def linkres(appid):
|
|
if appid in apps:
|
|
return ("fdroid.app:" + appid, "Dummy name - don't know yet")
|
|
raise MetaDataException("Cannot resolve app id " + appid)
|
|
|
|
for appid, app in apps.items():
|
|
try:
|
|
description_html(app.Description, linkres)
|
|
except MetaDataException as e:
|
|
raise MetaDataException("Problem with description of " + appid +
|
|
" - " + str(e))
|
|
|
|
return apps
|
|
|
|
# Port legacy ';' separators
|
|
list_sep = re.compile(r'[,;]')
|
|
|
|
|
|
def split_list_values(s):
|
|
res = []
|
|
for v in re.split(list_sep, s):
|
|
if not v:
|
|
continue
|
|
v = v.strip()
|
|
if not v:
|
|
continue
|
|
res.append(v)
|
|
return res
|
|
|
|
|
|
def get_default_app_info(metadatapath=None):
|
|
if metadatapath is None:
|
|
appid = None
|
|
else:
|
|
appid, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
|
|
|
|
if appid == '.fdroid': # we have local metadata in the app's source
|
|
if os.path.exists('AndroidManifest.xml'):
|
|
manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
|
|
else:
|
|
pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
|
|
for root, dirs, files in os.walk(os.getcwd()):
|
|
if 'build.gradle' in files:
|
|
p = os.path.join(root, 'build.gradle')
|
|
with open(p, 'rb') as f:
|
|
data = f.read()
|
|
m = pattern.search(data)
|
|
if m:
|
|
logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml'))
|
|
manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
|
|
break
|
|
if manifestroot is None:
|
|
raise MetaDataException("Cannot find a packageName for {0}!".format(metadatapath))
|
|
appid = manifestroot.attrib['package']
|
|
|
|
app = App()
|
|
app.metadatapath = metadatapath
|
|
if appid is not None:
|
|
app.id = appid
|
|
|
|
return app
|
|
|
|
|
|
def sorted_builds(builds):
|
|
return sorted(builds, key=lambda build: int(build.vercode))
|
|
|
|
|
|
esc_newlines = re.compile(r'\\( |\n)')
|
|
|
|
|
|
# This function uses __dict__ to be faster
|
|
def post_metadata_parse(app):
|
|
|
|
for k in app._modified:
|
|
v = app.__dict__[k]
|
|
if type(v) in (float, int):
|
|
app.__dict__[k] = str(v)
|
|
|
|
for build in app.builds:
|
|
for k in build._modified:
|
|
v = build.__dict__[k]
|
|
if type(v) in (float, int):
|
|
build.__dict__[k] = str(v)
|
|
continue
|
|
ftype = flagtype(k)
|
|
|
|
if ftype == TYPE_SCRIPT:
|
|
build.__dict__[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
|
|
elif ftype == TYPE_BOOL:
|
|
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
|
|
if isinstance(v, str):
|
|
build.__dict__[k] = _decode_bool(v)
|
|
elif ftype == TYPE_STRING:
|
|
if isinstance(v, bool) and v:
|
|
build.__dict__[k] = 'yes'
|
|
|
|
if not app.Description:
|
|
app.Description = 'No description available'
|
|
|
|
app.builds = sorted_builds(app.builds)
|
|
|
|
|
|
# Parse metadata for a single application.
|
|
#
|
|
# 'metadatapath' - the filename to read. The package id for the application comes
|
|
# from this filename. Pass None to get a blank entry.
|
|
#
|
|
# Returns a dictionary containing all the details of the application. There are
|
|
# two major kinds of information in the dictionary. Keys beginning with capital
|
|
# letters correspond directory to identically named keys in the metadata file.
|
|
# Keys beginning with lower case letters are generated in one way or another,
|
|
# and are not found verbatim in the metadata.
|
|
#
|
|
# Known keys not originating from the metadata are:
|
|
#
|
|
# 'builds' - a list of dictionaries containing build information
|
|
# for each defined build
|
|
# 'comments' - a list of comments from the metadata file. Each is
|
|
# a list of the form [field, comment] where field is
|
|
# the name of the field it preceded in the metadata
|
|
# file. Where field is None, the comment goes at the
|
|
# end of the file. Alternatively, 'build:version' is
|
|
# for a comment before a particular build version.
|
|
# 'descriptionlines' - original lines of description as formatted in the
|
|
# metadata file.
|
|
#
|
|
|
|
|
|
bool_true = re.compile(r'([Yy]es|[Tt]rue)')
|
|
bool_false = re.compile(r'([Nn]o|[Ff]alse)')
|
|
|
|
|
|
def _decode_bool(s):
|
|
if bool_true.match(s):
|
|
return True
|
|
if bool_false.match(s):
|
|
return False
|
|
raise MetaDataException("Invalid bool '%s'" % s)
|
|
|
|
|
|
def parse_metadata(metadatapath):
|
|
_, ext = fdroidserver.common.get_extension(metadatapath)
|
|
accepted = fdroidserver.common.config['accepted_formats']
|
|
if ext not in accepted:
|
|
raise MetaDataException('"%s" is not an accepted format, convert to: %s' % (
|
|
metadatapath, ', '.join(accepted)))
|
|
|
|
app = App()
|
|
app.metadatapath = metadatapath
|
|
app.id, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
|
|
|
|
with open(metadatapath, 'r', encoding='utf-8') as mf:
|
|
if ext == 'txt':
|
|
parse_txt_metadata(mf, app)
|
|
elif ext == 'json':
|
|
parse_json_metadata(mf, app)
|
|
elif ext == 'xml':
|
|
parse_xml_metadata(mf, app)
|
|
elif ext == 'yml':
|
|
parse_yaml_metadata(mf, app)
|
|
else:
|
|
raise MetaDataException('Unknown metadata format: %s' % metadatapath)
|
|
|
|
post_metadata_parse(app)
|
|
return app
|
|
|
|
|
|
def parse_json_metadata(mf, app):
|
|
|
|
# fdroid metadata is only strings and booleans, no floats or ints.
|
|
# TODO create schema using https://pypi.python.org/pypi/jsonschema
|
|
jsoninfo = json.load(mf, parse_int=lambda s: s,
|
|
parse_float=lambda s: s)
|
|
app.update_fields(jsoninfo)
|
|
for f in ['Description', 'Maintainer Notes']:
|
|
v = app.get_field(f)
|
|
app.set_field(f, '\n'.join(v))
|
|
return app
|
|
|
|
|
|
def parse_xml_metadata(mf, app):
|
|
|
|
tree = ElementTree.ElementTree(file=mf)
|
|
root = tree.getroot()
|
|
|
|
if root.tag != 'resources':
|
|
raise MetaDataException('resources file does not have root element <resources/>')
|
|
|
|
for child in root:
|
|
if child.tag != 'builds':
|
|
# builds does not have name="" attrib
|
|
name = child.attrib['name']
|
|
|
|
if child.tag == 'string':
|
|
app.set_field(name, child.text)
|
|
elif child.tag == 'string-array':
|
|
for item in child:
|
|
app.append_field(name, item.text)
|
|
elif child.tag == 'builds':
|
|
for b in child:
|
|
build = Build()
|
|
for key in b:
|
|
build.set_flag(key.tag, key.text)
|
|
app.builds.append(build)
|
|
|
|
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
|
|
if not isinstance(app.RequiresRoot, bool):
|
|
app.RequiresRoot = app.RequiresRoot == 'true'
|
|
|
|
return app
|
|
|
|
|
|
def parse_yaml_metadata(mf, app):
|
|
|
|
yamlinfo = yaml.load(mf, Loader=YamlLoader)
|
|
app.update_fields(yamlinfo)
|
|
return app
|
|
|
|
|
|
build_line_sep = re.compile(r'(?<!\\),')
|
|
build_cont = re.compile(r'^[ \t]')
|
|
|
|
|
|
def parse_txt_metadata(mf, app):
|
|
|
|
linedesc = None
|
|
|
|
def add_buildflag(p, build):
|
|
if not p.strip():
|
|
raise MetaDataException("Empty build flag at {1}"
|
|
.format(buildlines[0], linedesc))
|
|
bv = p.split('=', 1)
|
|
if len(bv) != 2:
|
|
raise MetaDataException("Invalid build flag at {0} in {1}"
|
|
.format(buildlines[0], linedesc))
|
|
|
|
pk, pv = bv
|
|
pk = pk.lstrip()
|
|
t = flagtype(pk)
|
|
if t == TYPE_LIST:
|
|
pv = split_list_values(pv)
|
|
build.set_flag(pk, pv)
|
|
elif t == TYPE_STRING or t == TYPE_SCRIPT:
|
|
build.set_flag(pk, pv)
|
|
elif t == TYPE_BOOL:
|
|
build.set_flag(pk, _decode_bool(pv))
|
|
|
|
def parse_buildline(lines):
|
|
v = "".join(lines)
|
|
parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
|
|
if len(parts) < 3:
|
|
raise MetaDataException("Invalid build format: " + v + " in " + mf.name)
|
|
build = Build()
|
|
build.version = parts[0]
|
|
build.vercode = parts[1]
|
|
if parts[2].startswith('!'):
|
|
# For backwards compatibility, handle old-style disabling,
|
|
# including attempting to extract the commit from the message
|
|
build.disable = parts[2][1:]
|
|
commit = 'unknown - see disabled'
|
|
index = parts[2].rfind('at ')
|
|
if index != -1:
|
|
commit = parts[2][index + 3:]
|
|
if commit.endswith(')'):
|
|
commit = commit[:-1]
|
|
build.commit = commit
|
|
else:
|
|
build.commit = parts[2]
|
|
for p in parts[3:]:
|
|
add_buildflag(p, build)
|
|
|
|
return build
|
|
|
|
def add_comments(key):
|
|
if not curcomments:
|
|
return
|
|
app.comments[key] = list(curcomments)
|
|
del curcomments[:]
|
|
|
|
mode = 0
|
|
buildlines = []
|
|
multiline_lines = []
|
|
curcomments = []
|
|
build = None
|
|
vc_seen = set()
|
|
|
|
c = 0
|
|
for line in mf:
|
|
c += 1
|
|
linedesc = "%s:%d" % (mf.name, c)
|
|
line = line.rstrip('\r\n')
|
|
if mode == 3:
|
|
if build_cont.match(line):
|
|
if line.endswith('\\'):
|
|
buildlines.append(line[:-1].lstrip())
|
|
else:
|
|
buildlines.append(line.lstrip())
|
|
bl = ''.join(buildlines)
|
|
add_buildflag(bl, build)
|
|
del buildlines[:]
|
|
else:
|
|
if not build.commit and not build.disable:
|
|
raise MetaDataException("No commit specified for {0} in {1}"
|
|
.format(build.version, linedesc))
|
|
|
|
app.builds.append(build)
|
|
add_comments('build:' + build.vercode)
|
|
mode = 0
|
|
|
|
if mode == 0:
|
|
if not line:
|
|
continue
|
|
if line.startswith("#"):
|
|
curcomments.append(line[1:].strip())
|
|
continue
|
|
try:
|
|
f, v = line.split(':', 1)
|
|
except ValueError:
|
|
raise MetaDataException("Invalid metadata in " + linedesc)
|
|
|
|
# Translate obsolete fields...
|
|
if f == 'Market Version':
|
|
f = 'Current Version'
|
|
if f == 'Market Version Code':
|
|
f = 'Current Version Code'
|
|
|
|
ftype = fieldtype(f)
|
|
if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
|
|
add_comments(f)
|
|
if ftype == TYPE_MULTILINE:
|
|
mode = 1
|
|
if v:
|
|
raise MetaDataException("Unexpected text on same line as " + f + " in " + linedesc)
|
|
elif ftype == TYPE_STRING:
|
|
app.set_field(f, v)
|
|
elif ftype == TYPE_LIST:
|
|
app.set_field(f, split_list_values(v))
|
|
elif ftype == TYPE_BUILD:
|
|
if v.endswith("\\"):
|
|
mode = 2
|
|
del buildlines[:]
|
|
buildlines.append(v[:-1])
|
|
else:
|
|
build = parse_buildline([v])
|
|
app.builds.append(build)
|
|
add_comments('build:' + app.builds[-1].vercode)
|
|
elif ftype == TYPE_BUILD_V2:
|
|
vv = v.split(',')
|
|
if len(vv) != 2:
|
|
raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
|
|
.format(v, linedesc))
|
|
build = Build()
|
|
build.version = vv[0]
|
|
build.vercode = vv[1]
|
|
if build.vercode in vc_seen:
|
|
raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
|
|
build.vercode, linedesc))
|
|
vc_seen.add(build.vercode)
|
|
del buildlines[:]
|
|
mode = 3
|
|
elif ftype == TYPE_OBSOLETE:
|
|
pass # Just throw it away!
|
|
else:
|
|
raise MetaDataException("Unrecognised field '" + f + "' in " + linedesc)
|
|
elif mode == 1: # Multiline field
|
|
if line == '.':
|
|
mode = 0
|
|
app.set_field(f, '\n'.join(multiline_lines))
|
|
del multiline_lines[:]
|
|
else:
|
|
multiline_lines.append(line)
|
|
elif mode == 2: # Line continuation mode in Build Version
|
|
if line.endswith("\\"):
|
|
buildlines.append(line[:-1])
|
|
else:
|
|
buildlines.append(line)
|
|
build = parse_buildline(buildlines)
|
|
app.builds.append(build)
|
|
add_comments('build:' + app.builds[-1].vercode)
|
|
mode = 0
|
|
add_comments(None)
|
|
|
|
# Mode at end of file should always be 0
|
|
if mode == 1:
|
|
raise MetaDataException(f + " not terminated in " + mf.name)
|
|
if mode == 2:
|
|
raise MetaDataException("Unterminated continuation in " + mf.name)
|
|
if mode == 3:
|
|
raise MetaDataException("Unterminated build in " + mf.name)
|
|
|
|
return app
|
|
|
|
|
|
def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
|
|
|
|
def w_comments(key):
|
|
if key not in app.comments:
|
|
return
|
|
for line in app.comments[key]:
|
|
w_comment(line)
|
|
|
|
def w_field_always(f, v=None):
|
|
if v is None:
|
|
v = app.get_field(f)
|
|
w_comments(f)
|
|
w_field(f, v)
|
|
|
|
def w_field_nonempty(f, v=None):
|
|
if v is None:
|
|
v = app.get_field(f)
|
|
w_comments(f)
|
|
if v:
|
|
w_field(f, v)
|
|
|
|
w_field_nonempty('Disabled')
|
|
w_field_nonempty('AntiFeatures')
|
|
w_field_nonempty('Provides')
|
|
w_field_always('Categories')
|
|
w_field_always('License')
|
|
w_field_nonempty('Author Name')
|
|
w_field_nonempty('Author Email')
|
|
w_field_always('Web Site')
|
|
w_field_always('Source Code')
|
|
w_field_always('Issue Tracker')
|
|
w_field_nonempty('Changelog')
|
|
w_field_nonempty('Donate')
|
|
w_field_nonempty('FlattrID')
|
|
w_field_nonempty('Bitcoin')
|
|
w_field_nonempty('Litecoin')
|
|
mf.write('\n')
|
|
w_field_nonempty('Name')
|
|
w_field_nonempty('Auto Name')
|
|
w_field_always('Summary')
|
|
w_field_always('Description', description_txt(app.Description))
|
|
mf.write('\n')
|
|
if app.RequiresRoot:
|
|
w_field_always('Requires Root', 'yes')
|
|
mf.write('\n')
|
|
if app.RepoType:
|
|
w_field_always('Repo Type')
|
|
w_field_always('Repo')
|
|
if app.Binaries:
|
|
w_field_always('Binaries')
|
|
mf.write('\n')
|
|
|
|
for build in app.builds:
|
|
|
|
if build.version == "Ignore":
|
|
continue
|
|
|
|
w_comments('build:' + build.vercode)
|
|
w_build(build)
|
|
mf.write('\n')
|
|
|
|
if app.MaintainerNotes:
|
|
w_field_always('Maintainer Notes', app.MaintainerNotes)
|
|
mf.write('\n')
|
|
|
|
w_field_nonempty('Archive Policy')
|
|
w_field_always('Auto Update Mode')
|
|
w_field_always('Update Check Mode')
|
|
w_field_nonempty('Update Check Ignore')
|
|
w_field_nonempty('Vercode Operation')
|
|
w_field_nonempty('Update Check Name')
|
|
w_field_nonempty('Update Check Data')
|
|
if app.CurrentVersion:
|
|
w_field_always('Current Version')
|
|
w_field_always('Current Version Code')
|
|
if app.NoSourceSince:
|
|
mf.write('\n')
|
|
w_field_always('No Source Since')
|
|
w_comments(None)
|
|
|
|
|
|
# Write a metadata file in txt format.
|
|
#
|
|
# 'mf' - Writer interface (file, StringIO, ...)
|
|
# 'app' - The app data
|
|
def write_txt(mf, app):
|
|
|
|
def w_comment(line):
|
|
mf.write("# %s\n" % line)
|
|
|
|
def w_field(f, v):
|
|
t = fieldtype(f)
|
|
if t == TYPE_LIST:
|
|
v = ','.join(v)
|
|
elif t == TYPE_MULTILINE:
|
|
v = '\n' + v + '\n.'
|
|
mf.write("%s:%s\n" % (f, v))
|
|
|
|
def w_build(build):
|
|
mf.write("Build:%s,%s\n" % (build.version, build.vercode))
|
|
|
|
for f in build_flags_order:
|
|
v = build.get_flag(f)
|
|
if not v:
|
|
continue
|
|
|
|
t = flagtype(f)
|
|
mf.write(' %s=' % f)
|
|
if t == TYPE_STRING:
|
|
mf.write(v)
|
|
elif t == TYPE_BOOL:
|
|
mf.write('yes')
|
|
elif t == TYPE_SCRIPT:
|
|
first = True
|
|
for s in v.split(' && '):
|
|
if first:
|
|
first = False
|
|
else:
|
|
mf.write(' && \\\n ')
|
|
mf.write(s)
|
|
elif t == TYPE_LIST:
|
|
mf.write(','.join(v))
|
|
|
|
mf.write('\n')
|
|
|
|
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
|
|
|
|
|
|
def write_yaml(mf, app):
|
|
|
|
def w_comment(line):
|
|
mf.write("# %s\n" % line)
|
|
|
|
def escape(v):
|
|
if not v:
|
|
return ''
|
|
if any(c in v for c in [': ', '%', '@', '*']):
|
|
return "'" + v.replace("'", "''") + "'"
|
|
return v
|
|
|
|
def w_field(f, v, prefix='', t=None):
|
|
if t is None:
|
|
t = fieldtype(f)
|
|
v = ''
|
|
if t == TYPE_LIST:
|
|
v = '\n'
|
|
for e in v:
|
|
v += prefix + ' - ' + escape(e) + '\n'
|
|
elif t == TYPE_MULTILINE:
|
|
v = ' |\n'
|
|
for l in v.splitlines():
|
|
if l:
|
|
v += prefix + ' ' + l + '\n'
|
|
else:
|
|
v += '\n'
|
|
elif t == TYPE_BOOL:
|
|
v = ' yes\n'
|
|
elif t == TYPE_SCRIPT:
|
|
cmds = [s + '&& \\' for s in v.split('&& ')]
|
|
if len(cmds) > 0:
|
|
cmds[-1] = cmds[-1][:-len('&& \\')]
|
|
w_field(f, cmds, prefix, 'multiline')
|
|
return
|
|
else:
|
|
v = ' ' + escape(v) + '\n'
|
|
|
|
mf.write(prefix)
|
|
mf.write(f)
|
|
mf.write(":")
|
|
mf.write(v)
|
|
|
|
global first_build
|
|
first_build = True
|
|
|
|
def w_build(build):
|
|
global first_build
|
|
if first_build:
|
|
mf.write("builds:\n")
|
|
first_build = False
|
|
|
|
w_field('versionName', build.version, ' - ', TYPE_STRING)
|
|
w_field('versionCode', build.vercode, ' ', TYPE_STRING)
|
|
for f in build_flags_order:
|
|
v = build.get_flag(f)
|
|
if not v:
|
|
continue
|
|
|
|
w_field(f, v, ' ', flagtype(f))
|
|
|
|
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
|
|
|
|
|
|
def write_metadata(metadatapath, app):
|
|
_, ext = fdroidserver.common.get_extension(metadatapath)
|
|
accepted = fdroidserver.common.config['accepted_formats']
|
|
if ext not in accepted:
|
|
raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % (
|
|
metadatapath, ', '.join(accepted)))
|
|
|
|
with open(metadatapath, 'w', encoding='utf8') as mf:
|
|
if ext == 'txt':
|
|
return write_txt(mf, app)
|
|
elif ext == 'yml':
|
|
return write_yaml(mf, app)
|
|
raise MetaDataException('Unknown metadata format: %s' % metadatapath)
|