kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-10-05 18:50:09 +02:00
Code Activity

Merge branch 'random-bag-of-fies' into 'master'

Browse Source 
random bag of fixes

Closes #719

See merge request fdroid/fdroidserver!710
This commit is contained in:
Hans-Christoph Steiner 2020-02-05 19:12:19 +00:00
commit 140bd29c94
13 changed files with 90 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
.gitlab-ci.yml
View File

@ -1,8 +1,12 @@
variables:
pip: pip3 --timeout 100 --retries 10
test:
image: registry.gitlab.com/fdroid/ci-images-server:latest
script:
- pip3 install -e .
- $pip install -e .
- cd tests
- ./complete-ci-tests
@ -104,8 +108,8 @@ ubuntu_xenial_pip:
- rm -rf env
- pyvenv env
- . env/bin/activate
- pip3 install --upgrade babel pip setuptools
- pip3 install -e .
- $pip install --upgrade babel pip setuptools
- $pip install -e .
- ./setup.py compile_catalog
- ./tests/run-tests
@ -135,13 +139,12 @@ lint_format_safety_bandit_checks:
script:
- apk add --no-cache bash dash ca-certificates python3
- python3 -m ensurepip
- pip3 install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety
- $pip install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety
- export EXITVALUE=0
- ./hooks/pre-commit || export EXITVALUE=1
- bandit
-ii
-s B110,B310,B322,B404,B408,B410,B603,B607
-x fdroidserver/dscanner.py,docker/install_agent.py,docker/drozer.py
-s B110,B322,B404,B408,B410,B603,B607
-r $CI_PROJECT_DIR fdroid
|| export EXITVALUE=1
- safety check --full-report || export EXITVALUE=1
@ -185,7 +188,7 @@ fedora_latest:
which
- ./setup.py compile_catalog sdist
- useradd -m -c "test account" --password "fakepassword" testuser
- su testuser --login --command "cd `pwd`; pip3 install --user dist/fdroidserver-*.tar.gz"
- su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz"
- test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo
- wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.5-linux.zip
- unzip -q tools.zip
@ -223,5 +226,5 @@ gradle:
test -z "$CHANGED" && exit;
fi
- python3 -m ensurepip
- pip3 install beautifulsoup4 requests
- $pip install beautifulsoup4 requests
- ./tests/gradle-release-checksums.py

4
completion/bash-completion
View File

@ -73,7 +73,7 @@ __vercode() {
line="${line#*,}"
printf "${line%%,*} "
fi
done < "metadata/${p}.txt" )" -- $cur ) )
done < "metadata/${p}.yml" )" -- $cur ) )
}
__complete_options() {
@ -253,7 +253,7 @@ __complete_btlog() {
__complete_mirror() {
opts="-v"
lopts="--archive --output-dir"
lopts="--all --archive --build-logs --pgp-signatures --src-tarballs --output-dir"
__complete_options
}

6
examples/template.yml
View File

@ -2,7 +2,7 @@ AuthorName: .
WebSite: ''
Bitcoin: null
Litecoin: null
Donation: null
Donate: null
License: Unknown
Categories:
@ -17,5 +17,5 @@ Summary: .
Description: |
.
Archive Policy: 2 versions
Requires Root: No
ArchivePolicy: 2 versions
RequiresRoot: No

40
fdroidserver/checkupdates.py
View File

@ -64,7 +64,7 @@ def check_http(app):
if len(urlcode) > 0:
logging.debug("...requesting {0}".format(urlcode))
req = urllib.request.Request(urlcode, None)
resp = urllib.request.urlopen(req, None, 20)
resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above
page = resp.read().decode('utf-8')
m = re.search(codeex, page)
@ -77,7 +77,7 @@ def check_http(app):
if urlver != '.':
logging.debug("...requesting {0}".format(urlver))
req = urllib.request.Request(urlver, None)
resp = urllib.request.urlopen(req, None, 20)
resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above
page = resp.read().decode('utf-8')
m = re.search(verex, page)
@ -295,7 +295,7 @@ def check_gplay(app):
headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
req = urllib.request.Request(url, None, headers)
try:
resp = urllib.request.urlopen(req, None, 20)
resp = urllib.request.urlopen(req, None, 20) # nosec B310 URL base is hardcoded above
page = resp.read().decode()
except urllib.error.HTTPError as e:
return (None, str(e.code))
@ -358,6 +358,18 @@ def possible_subdirs(app):
yield subdir
def _getappname(app):
if app.Name:
return app.Name
if app.AutoName:
return app.AutoName
return app.id
def _getcvname(app):
return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
def fetch_autoname(app, tag):
if not app.RepoType or app.UpdateCheckMode in ('None', 'Static') \
@ -393,7 +405,7 @@ def fetch_autoname(app, tag):
if new_name != app.AutoName:
app.AutoName = new_name
if not commitmsg:
commitmsg = "Set autoname of {0}".format(common.getappname(app))
commitmsg = "Set autoname of {0}".format(_getappname(app))
else:
logging.debug("...couldn't get autoname")
@ -472,8 +484,8 @@ def checkupdates_app(app):
commitmsg = fetch_autoname(app, tag)
if updating:
name = common.getappname(app)
ver = common.getcvname(app)
name = _getappname(app)
ver = _getcvname(app)
logging.info('...updating to version %s' % ver)
commitmsg = 'Update CV of %s to %s' % (name, ver)
@ -513,8 +525,8 @@ def checkupdates_app(app):
commit = commit.replace('%c', newbuild.versionCode)
newbuild.commit = commit
app.builds.append(newbuild)
name = common.getappname(app)
ver = common.getcvname(app)
name = _getappname(app)
ver = _getcvname(app)
commitmsg = "Update %s to %s" % (name, ver)
else:
logging.warning('Invalid auto update mode "' + mode + '" on ' + app.id)
@ -610,24 +622,24 @@ def main():
version, reason = check_gplay(app)
if version is None:
if reason == '404':
logging.info("{0} is not in the Play Store".format(common.getappname(app)))
logging.info("{0} is not in the Play Store".format(_getappname(app)))
else:
logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
logging.info("{0} encountered a problem: {1}".format(_getappname(app), reason))
if version is not None:
stored = app.CurrentVersion
if not stored:
logging.info("{0} has no Current Version but has version {1} on the Play Store"
.format(common.getappname(app), version))
.format(_getappname(app), version))
elif LooseVersion(stored) < LooseVersion(version):
logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
.format(common.getappname(app), version, stored))
.format(_getappname(app), version, stored))
else:
if stored != version:
logging.info("{0} has version {1} on the Play Store, which differs from {2}"
.format(common.getappname(app), version, stored))
.format(_getappname(app), version, stored))
else:
logging.info("{0} has the same version {1} on the Play Store"
.format(common.getappname(app), version))
.format(_getappname(app), version))
update_wiki(gplaylog, None)
return

12
fdroidserver/common.py
View File

@ -664,18 +664,6 @@ def getsrcname(app, build):
return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
def getappname(app):
if app.Name:
return app.Name
if app.AutoName:
return app.AutoName
return app.id
def getcvname(app):
return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
def get_build_dir(app):
'''get the dir that this app will be built in'''

5
fdroidserver/import.py
View File

@ -40,8 +40,9 @@ SETTINGS_GRADLE = re.compile(r'''include\s+['"]:([^'"]*)['"]''')
# when one of these is found it's assumed that's the information we want.
# Returns repotype, address, or None, reason
def getrepofrompage(url):
req = urllib.request.urlopen(url)
if not url.startswith('http'):
return (None, _('{url} does not start with "http"!'.format(url=url)))
req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
if req.getcode() != 200:
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
page = req.read().decode(req.headers.get_content_charset())

13
fdroidserver/mirror.py
View File

@ -46,16 +46,26 @@ def main():
parser.add_argument("url", nargs='?',
help=_('Base URL to mirror, can include the index signing key '
+ 'using the query string: ?fingerprint='))
parser.add_argument("--all", action='store_true', default=False,
help=_("Mirror the full repo and archive, all file types."))
parser.add_argument("--archive", action='store_true', default=False,
help=_("Also mirror the full archive section"))
parser.add_argument("--build-logs", action='store_true', default=False,
help=_("Include the build logs in the mirror"))
parser.add_argument("--pgp-signatures", action='store_true', default=False,
help=_("Include the PGP signature .asc files in the mirror"))
parser.add_argument("--src-tarballs", action='store_true', default=False,
help=_("Include the source tarballs in the mirror"))
parser.add_argument("--output-dir", default=None,
help=_("The directory to write the mirror to"))
options = parser.parse_args()
if options.all:
options.archive = True
options.build_logs = True
options.pgp_signatures = True
options.src_tarballs = True
if options.url is None:
logging.error(_('A URL is required as an argument!') + '\n')
parser.print_help()
@ -152,7 +162,8 @@ def main():
if not os.path.exists(f) \
or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
urls.append(_append_to_url_path(section, f))
urls.append(_append_to_url_path(section, f + '.asc'))
if options.pgp_signatures:
urls.append(_append_to_url_path(section, f + '.asc'))
if options.build_logs and f.endswith('.apk'):
urls.append(_append_to_url_path(section, f[:-4] + '.log.gz'))

5
fdroidserver/update.py
View File

@ -714,7 +714,10 @@ def _set_localized_text_entry(app, locale, key, f):
with open(f, errors='replace') as fp:
text = fp.read()[:limit]
if len(text) > 0:
localized[key] = text
if key in ('name', 'summary', 'video'): # hardcoded as a single line
localized[key] = text.strip('\n')
else:
localized[key] = text
def _set_author_entry(app, key, f):

8
makebuildserver
View File

@ -362,8 +362,8 @@ CACHE_FILES = [
'1f3067073041bc44554d0efe5d402a33bc3d3c93cc39ab684f308586d732a80d'),
('https://services.gradle.org/distributions/gradle-6.0.1-bin.zip',
'd364b7098b9f2e58579a3603dc0a12a1991353ac58ed339316e6762b21efba44'),
('https://services.gradle.org/distributions/gradle-6.1-bin.zip',
'd0c43d14e1c70a48b82442f435d06186351a2d290d72afd5b8866f15e6d7038a'),
('https://services.gradle.org/distributions/gradle-6.1.1-bin.zip',
'9d94e6e4a28ad328072ef6e56bce79a810494ae756751fdcedffdeaf27c093b1'),
('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
@ -531,6 +531,10 @@ def main():
open(logfilename, 'a').close() # create blank file
log_cm = vagrant.make_file_cm(logfilename)
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
# https://phoenhex.re/2018-03-25/not-a-vagrant-bug
os_env = os.environ.copy()
os_env['VAGRANT_DISABLE_VBOXSYMLINKCREATE'] = '1'
v.env = os_env
if options.verbosity >= 2:
tail = fdroidserver.tail.Tail(logfilename)

18
tests/checkupdates.TestCase
View File

@ -19,10 +19,11 @@ if localmodule not in sys.path:
import fdroidserver.checkupdates
import fdroidserver.metadata
from fdroidserver.exception import FDroidException
class CommonTest(unittest.TestCase):
'''fdroidserver/common.py'''
class CheckupdatesTest(unittest.TestCase):
'''fdroidserver/checkupdates.py'''
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
@ -123,6 +124,17 @@ class CommonTest(unittest.TestCase):
self.assertEqual(vername, '1.1.9')
self.assertEqual(vercode, '10109')
def test_check_http_blocks_unknown_schemes(self):
app = fdroidserver.metadata.App()
for scheme in ('file', 'ssh', 'http', ';pwn'):
app.id = scheme
faked = scheme + '://fake.url/for/testing/scheme'
app.UpdateCheckData = faked + '|ignored|' + faked + '|ignored'
app.metadatapath = 'metadata/' + app.id + '.yml'
vername, vercode = fdroidserver.checkupdates.check_http(app)
self.assertIsNone(vername)
self.assertTrue(FDroidException.__name__ in vercode)
def test_check_http_ignore(self):
fdroidserver.checkupdates.options = mock.Mock()
@ -151,5 +163,5 @@ if __name__ == "__main__":
(fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
newSuite = unittest.TestSuite()
newSuite.addTest(unittest.makeSuite(CommonTest))
newSuite.addTest(unittest.makeSuite(CheckupdatesTest))
unittest.main(failfast=False)

2
tests/config.py
View File

@ -26,8 +26,8 @@ keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="
keydname = "CN=sova, OU=F-Droid"
mirrors = (
'https://foo.bar/fdroid',
'http://foobarfoobarfoobar.onion/fdroid',
'https://foo.bar/fdroid',
)
update_stats = True

6
tests/repo/index-v1.json
View File

@ -198,9 +198,9 @@
"description": "full description\n",
"featureGraphic": "featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png",
"icon": "icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png",
"name": "title\n",
"summary": "short description\n",
"video": "video\n"
"name": "title",
"summary": "short description",
"video": "video"
}
}
}

8
tests/update.TestCase
View File

@ -114,9 +114,9 @@ class UpdateTest(unittest.TestCase):
if packageName == 'info.guardianproject.urzip':
self.assertEqual(7, len(app['localized']['en-US']))
self.assertEqual('full description\n', app['localized']['en-US']['description'])
self.assertEqual('title\n', app['localized']['en-US']['name'])
self.assertEqual('short description\n', app['localized']['en-US']['summary'])
self.assertEqual('video\n', app['localized']['en-US']['video'])
self.assertEqual('title', app['localized']['en-US']['name'])
self.assertEqual('short description', app['localized']['en-US']['summary'])
self.assertEqual('video', app['localized']['en-US']['video'])
self.assertEqual('icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png',
app['localized']['en-US']['icon'])
self.assertEqual('featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png',
@ -140,7 +140,7 @@ class UpdateTest(unittest.TestCase):
elif packageName == 'com.nextcloud.client.dev':
self.assertEqual('Nextcloud Dev', app['localized']['en-US']['name'])
self.assertEqual(586, len(app['localized']['en-US']['description']))
self.assertEqual(79, len(app['localized']['en-US']['summary']))
self.assertEqual(78, len(app['localized']['en-US']['summary']))
elif packageName == 'eu.siacs.conversations':
self.assertEqual('Conversations', app['localized']['en-US']['name'])