kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-04 22:40:12 +01:00
Code Activity

support app metadata in XML format

Browse Source 
While the current text metadata format is good for human readability and
editability, it is difficult to produce and parse using code.  XML is a
widespread standard format for easy automatic parsing and creating, while
having decent human readability.

The .pickle for testing is a lightly edited version of the real metadata
for net.osmand.plus:

* comments were removed
* "NonFreeNet" was added as an AntiFeature
This commit is contained in:
Hans-Christoph Steiner 2015-07-22 18:40:31 -07:00
parent 2831b3e93f
commit ab145de6bc
4 changed files with 766 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
fdroidserver/metadata.py
View File

@ -20,10 +20,14 @@
import json
import os
import re
import sys
import glob
import cgi
import logging
# use the C implementation when available
import xml.etree.cElementTree as ElementTree
from collections import OrderedDict
import common
@ -79,6 +83,8 @@ app_defaults = OrderedDict([
# In the order in which they are laid out on files
# Sorted by their action and their place in the build timeline
# These variables can have varying datatypes. For example, anything with
# flagtype(v) == 'list' is inited as False, then set as a list of strings.
flag_defaults = OrderedDict([
('disable', False),
('commit', None),
@ -494,6 +500,11 @@ def read_metadata(xref=True):
check_metadata(appinfo)
apps[appid] = appinfo
for metafile in sorted(glob.glob(os.path.join('metadata', '*.xml'))):
appid, appinfo = parse_xml_metadata(metafile)
check_metadata(appinfo)
apps[appid] = appinfo
if xref:
# Parse all descriptions at load time, just to ensure cross-referencing
# errors are caught early rather than when they hit the build server.
@ -579,6 +590,20 @@ def get_default_app_info_list():
def post_metadata_parse(thisinfo):
for build in thisinfo['builds']:
for k, v in build.iteritems():
if k == 'versionCode':
build['vercode'] = str(v)
del build['versionCode']
elif k == 'versionName':
build['version'] = str(v)
del build['versionName']
elif flagtype(k) == 'bool':
if v == 'no':
build[k] = False
else:
build[k] = True
if not thisinfo['Description']:
thisinfo['Description'].append('No description available')
@ -682,12 +707,6 @@ def parse_json_metadata(metafile):
build[k] = ['yes']
else:
build[k] = ['no']
elif k == 'versionCode':
build['vercode'] = v
del build['versionCode']
elif k == 'versionName':
build['version'] = v
del build['versionName']
# TODO create schema using https://pypi.python.org/pypi/jsonschema
post_metadata_parse(thisinfo)
@ -695,6 +714,75 @@ def parse_json_metadata(metafile):
return (appid, thisinfo)
def parse_xml_metadata(metafile):
appid = os.path.basename(metafile)[0:-4] # strip path and .xml
thisinfo = get_default_app_info_list()
thisinfo['id'] = appid
tree = ElementTree.ElementTree(file=metafile)
root = tree.getroot()
if root.tag != 'resources':
logging.critical(metafile + ' does not have root as <resources></resources>!')
sys.exit(1)
supported_metadata = app_defaults.keys()
for child in root:
if child.tag != 'builds':
# builds does not have name="" attrib
name = child.attrib['name']
if name not in supported_metadata:
raise MetaDataException("Unrecognised metadata: <"
+ child.tag + ' name="' + name + '">'
+ child.text
+ "</" + child.tag + '>')
if child.tag == 'string':
thisinfo[name] = child.text
elif child.tag == 'string-array':
items = []
for item in child:
items.append(item.text)
thisinfo[name] = items
elif child.tag == 'builds':
builds = []
for build in child:
builddict = dict()
for key in build:
builddict[key.tag] = key.text
builds.append(builddict)
thisinfo['builds'] = builds
# convert to the odd internal format
for k in ('Description', 'Maintainer Notes'):
if isinstance(thisinfo[k], basestring):
text = thisinfo[k].rstrip().lstrip()
thisinfo[k] = text.split('\n')
supported_flags = flag_defaults.keys() + ['versionCode', 'versionName']
for build in thisinfo['builds']:
for k, v in build.iteritems():
if k not in supported_flags:
raise MetaDataException("Unrecognised build flag: {0}={1}"
.format(k, v))
keyflagtype = flagtype(k)
if keyflagtype == 'bool':
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
if isinstance(v, basestring):
if v == 'true':
build[k] = True
else:
build[k] = False
elif keyflagtype == 'list':
if isinstance(v, basestring):
build[k] = [v]
post_metadata_parse(thisinfo)
return (appid, thisinfo)
def parse_txt_metadata(metafile):
appid = None

2
tests/metadata.TestCase
View File

@ -37,7 +37,7 @@ class MetadataTest(unittest.TestCase):
fdroidserver.common.config = config
apps = fdroidserver.metadata.read_metadata(xref=True)
for appid in ('org.smssecure.smssecure', 'org.adaway'):
for appid in ('org.smssecure.smssecure', 'org.adaway', 'net.osmand.plus'):
frompickle = pickle.load(open(os.path.join('metadata', appid + '.pickle')))
self.assertTrue(appid in apps.keys())
self.assertEquals(apps[appid], frompickle)

491
tests/metadata/net.osmand.plus.pickle Normal file
View File

@ -0,0 +1,491 @@
(dp0
S'Update Check Data'
p1
NsS'Bitcoin'
p2
NsS'AntiFeatures'
p3
(lp4
S'Tracking'
p5
aS'NonFreeNet'
p6
asS'Web Site'
p7
S'http://osmand.net'
p8
sS'Auto Update Mode'
p9
S'None'
p10
sS'Provides'
p11
NsS'Issue Tracker'
p12
S'https://github.com/osmandapp/Osmand/issues'
p13
sS'Donate'
p14
S'https://code.google.com/p/osmand/#Please_support_the_project'
p15
sS'Repo Type'
p16
S'git'
p17
sS'Description'
p18
(lp19
S"Osmand~'s features can be extended by enabling the plugins via the settings,"
p20
aS'which include online maps from many sources, tracking, OpenStreetMap (OSM) editing and'
p21
aS'accessibility enhancements.'
p22
aS''
p23
aS'Map data of both vector and raster types can be stored on the phone memory'
p24
aS'card for offline usage, and navigation by default uses offline methods. Map'
p25
aS'data packages for many territories can be downloaded from within the app and'
p26
aS'there is a desktop program available on the website as well for creating your'
p27
aS'own.'
p28
ag23
aS'Anti-Features: Tracking - It will send your device and application specs to an'
p29
aS'Analytics server upon downloading the list of maps you can download.'
p30
ag23
aS'[https://osmandapp.github.io/changes.html Changelog]'
p31
asS'Requires Root'
p32
I00
sS'comments'
p33
(lp34
sS'id'
p35
S'net.osmand.plus'
p36
sS'Repo'
p37
S'https://github.com/mvdan/OsmAnd-submodules'
p38
sS'No Source Since'
p39
g23
sS'Auto Name'
p40
g23
sS'Categories'
p41
(lp42
S'Navigation'
p43
asS'Source Code'
p44
S'https://github.com/osmandapp/Osmand'
p45
sS'Litecoin'
p46
NsS'Update Check Ignore'
p47
NsS'Name'
p48
S'OsmAnd~'
p49
sS'License'
p50
S'GPLv3'
p51
sS'Changelog'
p52
g23
sS'Update Check Mode'
p53
S'None'
p54
sS'Summary'
p55
S'Offline/online maps and navigation'
p56
sS'Dogecoin'
p57
NsS'Maintainer Notes'
p58
(lp59
S'No UCMs apply because git never contains actual releases, only pre-releses.'
p60
ag23
aS'The build instructions have been moved to a script in the root of the repo,'
p61
aS"'build'. This way it can be updated along with the submodules."
p62
asS'Current Version Code'
p63
S'197'
p64
sS'Binaries'
p65
NsS'Archive Policy'
p66
NsS'builds'
p67
(lp68
(dp69
S'submodules'
p70
I01
sS'vercode'
p71
S'182'
p72
sS'forceversion'
p73
I00
sS'oldsdkloc'
p74
I00
sS'kivy'
p75
I00
sS'gradle'
p76
I00
sS'scanignore'
p77
(lp78
sS'srclibs'
p79
(lp80
sS'encoding'
p81
NsS'extlibs'
p82
(lp83
sS'init'
p84
g23
sS'version'
p85
S'1.8.2'
p86
sS'build'
p87
S'./old-ndk-build.sh && ant -Dsdk.dir="$ANDROID_SDK" -Dndk.dir="$ANDROID_NDK" -DBLACKBERRY_BUILD=false -DBUILD_SUFFIX= -DAPK_NUMBER_VERSION=182 "-DFEATURES=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" -DCLEAN_CPP=false -DPACKAGE_TO_BUILT=net.osmand.plus -DAPK_VERSION=1.8.2 -Dnet.osmand.plus= -Dbuild.version=1.8.2 -Dbuild.version.code=182 -Dnativeoff=false "-DversionFeatures=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" clean release'
p88
sS'rm'
p89
(lp90
sS'type'
p91
S'raw'
p92
sS'subdir'
p93
S'android/OsmAnd'
p94
sS'ndk_path'
p95
g23
sS'forcevercode'
p96
I00
sS'preassemble'
p97
(lp98
sS'update'
p99
(lp100
S'auto'
p101
asS'maven'
p102
I00
sS'disable'
p103
I00
sS'commit'
p104
S'76ada6c8a08afe69acb755503373ac36328ef665'
p105
sS'scandelete'
p106
(lp107
sS'buildjni'
p108
(lp109
S'no'
p110
asS'ndk'
p111
S'r10e'
p112
sS'target'
p113
NsS'antcommands'
p114
NsS'patch'
p115
(lp116
sS'prebuild'
p117
S'sed -i \'s/"OsmAnd+"/"OsmAnd~"/g\' build.xml'
p118
sS'novcheck'
p119
I00
sS'output'
p120
S'bin/OsmAnd-release-unsigned.apk'
p121
sa(dp122
S'submodules'
p123
I01
sg71
S'183'
p124
sg73
I00
sg74
I00
sg75
I00
sg76
I00
sg77
g78
sg79
g80
sg81
Nsg82
g83
sg84
g23
sg85
S'1.8.3'
p125
sS'subdir'
p126
S'android/OsmAnd'
p127
sg89
g90
sg91
g92
sS'build'
p128
S'../../build'
p129
sg95
g23
sg96
I00
sg97
g98
sg99
g100
sg102
I00
sg103
I00
sS'commit'
p130
S'1.8.3'
p131
sg106
g107
sS'buildjni'
p132
(lp133
S'no'
p134
asg111
g112
sg113
Nsg114
Nsg115
g116
sS'prebuild'
p135
g23
sg119
I00
sS'output'
p136
S'bin/OsmAnd-release-unsigned.apk'
p137
sa(dp138
S'submodules'
p139
I01
sg71
S'196'
p140
sg73
I00
sg74
I00
sg75
I00
sg76
I00
sg77
g78
sg79
g80
sg81
Nsg82
g83
sg84
g23
sg85
S'1.9.4'
p141
sS'subdir'
p142
S'android/OsmAnd'
p143
sg89
g90
sg91
g92
sS'build'
p144
S'../../build'
p145
sg95
g23
sg96
I00
sg97
g98
sg99
g100
sg102
I00
sg103
I00
sS'commit'
p146
S'1.9.4'
p147
sg106
g107
sS'buildjni'
p148
(lp149
S'no'
p150
asS'ndk'
p151
S'r10d'
p152
sg113
Nsg114
Nsg115
g116
sg135
g23
sg119
I00
sS'output'
p153
S'bin/OsmAnd-release-unsigned.apk'
p154
sa(dp155
S'submodules'
p156
I01
sg71
S'197'
p157
sg73
I00
sg74
I00
sg75
I00
sg76
I00
sg77
g78
sg79
g80
sg81
Nsg82
g83
sg84
g23
sg85
S'1.9.5'
p158
sS'subdir'
p159
S'android/OsmAnd'
p160
sg89
g90
sg91
g92
sS'build'
p161
S'../../build'
p162
sg95
g23
sg96
I00
sg97
g98
sg99
g100
sg102
I00
sg103
I00
sS'commit'
p163
S'1.9.5'
p164
sg106
g107
sS'buildjni'
p165
(lp166
S'no'
p167
asS'ndk'
p168
S'r10d'
p169
sg113
Nsg114
Nsg115
g116
sg135
g23
sg119
I00
sS'output'
p170
S'bin/OsmAnd-release-unsigned.apk'
p171
sasS'FlattrID'
p172
NsS'Disabled'
p173
NsS'Update Check Name'
p174
NsS'Vercode Operation'
p175
NsS'Current Version'
p176
S'1.9.5'
p177
s.

180
tests/metadata/net.osmand.plus.xml Normal file
View File

@ -0,0 +1,180 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>
<string-array name="AntiFeatures">
<item>Tracking</item>
<item>NonFreeNet</item>
</string-array>
<string-array name="Categories">
<item>Navigation</item>
</string-array>
<string name="License">GPLv3</string>
<string name="Web Site">http://osmand.net</string>
<string name="Source Code">https://github.com/osmandapp/Osmand</string>
<string name="Issue Tracker">https://github.com/osmandapp/Osmand/issues</string>
<string name="Donate">https://code.google.com/p/osmand/#Please_support_the_project</string>
<string name="Name">OsmAnd~</string>
<string name="Summary">Offline/online maps and navigation</string>
<string name="Description">Osmand~'s features can be extended by enabling the plugins via the settings,
which include online maps from many sources, tracking, OpenStreetMap (OSM) editing and
accessibility enhancements.
Map data of both vector and raster types can be stored on the phone memory
card for offline usage, and navigation by default uses offline methods. Map
data packages for many territories can be downloaded from within the app and
there is a desktop program available on the website as well for creating your
own.
Anti-Features: Tracking - It will send your device and application specs to an
Analytics server upon downloading the list of maps you can download.
[https://osmandapp.github.io/changes.html Changelog]
</string>
<string name="Repo Type">git</string>
<string name="Repo">https://github.com/mvdan/OsmAnd-submodules</string>
<!-- <string name="Repo">https://github.com/osmandapp/Osmand</string/ -->
<!--
# Old builds with the old repo
#Build:0.6.5,34
# commit=v0.6.5
# subdir=OsmAnd
# encoding=utf-8
# prebuild=mkdir assets && \
# mkdir raw
#
#Build:0.6.6,36
# commit=v0.6.6_2
# subdir=OsmAnd
# encoding=utf-8
# prebuild=mkdir raw
#
#Build:0.6.7,37
# commit=v0.6.7
# subdir=OsmAnd
# encoding=utf-8
# patch=code37.patch
# prebuild=mkdir raw
#
#Build:0.6.8,39
# commit=v0.6.8
# subdir=OsmAnd
# encoding=utf-8
# prebuild=mkdir raw
#
#Build:0.6.8',41
# disable=No corresponding source for whatever this is
# commit=unknown - see disabled
#
#Build:0.6.9,42
# commit=v0.6.9
# subdir=OsmAnd
# encoding=utf-8
# prebuild=mkdir raw
#
#Build:0.6.9',43
# disable=No corresponding source for whatever this is
# commit=unknown - see disabled
#
#Build:0.8.1,65
# commit=d62472532d8
# subdir=OsmAnd
# target=android-8
# init=rm -f build.xml
# encoding=utf-8
# forceversion=yes
# prebuild=cd ../DataExtractionOSM && \
# ant compile build && \
# cd ../OsmAnd/ && \
# cp ../DataExtractionOSM/build/OsmAndMapCreator.jar libs/ && \
# zip -d libs/OsmAndMapCreator.jar net/osmand/LogUtil.class && \
# cp -r ../DataExtractionOSM/build/lib/ libs/
# buildjni=no
#
#Build:0.8.2,71
# commit=50a4733475cd
# subdir=OsmAnd
# submodules=yes
# target=android-8
# init=rm -f build.xml
# encoding=utf-8
# forceversion=yes
# forcevercode=yes
# prebuild=cd ../DataExtractionOSM && \
# ant compile build && \
# cd ../OsmAnd/ && \
# sed -i 's/app_version">[^<]*/app_version">0.8.2-fdroid/' res/values/no_translate.xml && \
# cp ../DataExtractionOSM/build/OsmAndMapCreator.jar libs/ && \
# zip -d libs/OsmAndMapCreator.jar net/osmand/LogUtil.class && \
# cp -r ../DataExtractionOSM/build/lib/ libs/
# buildjni=yes
-->
<builds>
<build>
<versionCode>182</versionCode>
<versionName>1.8.2</versionName>
<commit>76ada6c8a08afe69acb755503373ac36328ef665</commit>
<subdir>android/OsmAnd</subdir>
<submodules>true</submodules>
<output>bin/OsmAnd-release-unsigned.apk</output>
<prebuild>sed -i 's/"OsmAnd+"/"OsmAnd~"/g' build.xml</prebuild>
<build>./old-ndk-build.sh &amp;&amp; ant -Dsdk.dir="$ANDROID_SDK" -Dndk.dir="$ANDROID_NDK" -DBLACKBERRY_BUILD=false -DBUILD_SUFFIX= -DAPK_NUMBER_VERSION=182 "-DFEATURES=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" -DCLEAN_CPP=false -DPACKAGE_TO_BUILT=net.osmand.plus -DAPK_VERSION=1.8.2 -Dnet.osmand.plus= -Dbuild.version=1.8.2 -Dbuild.version.code=182 -Dnativeoff=false "-DversionFeatures=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" clean release</build>
<buildjni>no</buildjni>
</build>
<build>
<versionName>1.8.3</versionName>
<versionCode>183</versionCode>
<commit>1.8.3</commit>
<subdir>android/OsmAnd</subdir>
<submodules>true</submodules>
<output>bin/OsmAnd-release-unsigned.apk</output>
<build>../../build</build>
<buildjni>no</buildjni>
</build>
<build>
<versionName>1.9.4</versionName>
<versionCode>196</versionCode>
<commit>1.9.4</commit>
<subdir>android/OsmAnd</subdir>
<submodules>true</submodules>
<output>bin/OsmAnd-release-unsigned.apk</output>
<build>../../build</build>
<buildjni>no</buildjni>
<ndk>r10d</ndk>
</build>
<build>
<versionName>1.9.5</versionName>
<versionCode>197</versionCode>
<commit>1.9.5</commit>
<subdir>android/OsmAnd</subdir>
<submodules>true</submodules>
<output>bin/OsmAnd-release-unsigned.apk</output>
<build>../../build</build>
<buildjni>no</buildjni>
<ndk>r10d</ndk>
</build>
</builds>
<string name="Maintainer Notes">
No UCMs apply because git never contains actual releases, only pre-releses.
The build instructions have been moved to a script in the root of the repo,
'build'. This way it can be updated along with the submodules.
</string>
<string name="Auto Update Mode">None</string>
<string name="Update Check Mode">None</string>
<string name="Current Version">1.9.5</string>
<string name="Current Version Code">197</string>
</resources>