mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-05 18:50:09 +02:00
Merge branch 'random-bag-of-fies' into 'master'
random bag of fixes Closes #719 See merge request fdroid/fdroidserver!710
This commit is contained in:
commit
140bd29c94
@ -1,8 +1,12 @@
|
||||
|
||||
variables:
|
||||
pip: pip3 --timeout 100 --retries 10
|
||||
|
||||
|
||||
test:
|
||||
image: registry.gitlab.com/fdroid/ci-images-server:latest
|
||||
script:
|
||||
- pip3 install -e .
|
||||
- $pip install -e .
|
||||
- cd tests
|
||||
- ./complete-ci-tests
|
||||
|
||||
@ -104,8 +108,8 @@ ubuntu_xenial_pip:
|
||||
- rm -rf env
|
||||
- pyvenv env
|
||||
- . env/bin/activate
|
||||
- pip3 install --upgrade babel pip setuptools
|
||||
- pip3 install -e .
|
||||
- $pip install --upgrade babel pip setuptools
|
||||
- $pip install -e .
|
||||
- ./setup.py compile_catalog
|
||||
- ./tests/run-tests
|
||||
|
||||
@ -135,13 +139,12 @@ lint_format_safety_bandit_checks:
|
||||
script:
|
||||
- apk add --no-cache bash dash ca-certificates python3
|
||||
- python3 -m ensurepip
|
||||
- pip3 install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety
|
||||
- $pip install Babel 'bandit<1.6.0' pycodestyle pyflakes 'pylint<2.0' safety
|
||||
- export EXITVALUE=0
|
||||
- ./hooks/pre-commit || export EXITVALUE=1
|
||||
- bandit
|
||||
-ii
|
||||
-s B110,B310,B322,B404,B408,B410,B603,B607
|
||||
-x fdroidserver/dscanner.py,docker/install_agent.py,docker/drozer.py
|
||||
-s B110,B322,B404,B408,B410,B603,B607
|
||||
-r $CI_PROJECT_DIR fdroid
|
||||
|| export EXITVALUE=1
|
||||
- safety check --full-report || export EXITVALUE=1
|
||||
@ -185,7 +188,7 @@ fedora_latest:
|
||||
which
|
||||
- ./setup.py compile_catalog sdist
|
||||
- useradd -m -c "test account" --password "fakepassword" testuser
|
||||
- su testuser --login --command "cd `pwd`; pip3 install --user dist/fdroidserver-*.tar.gz"
|
||||
- su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz"
|
||||
- test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo
|
||||
- wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.5-linux.zip
|
||||
- unzip -q tools.zip
|
||||
@ -223,5 +226,5 @@ gradle:
|
||||
test -z "$CHANGED" && exit;
|
||||
fi
|
||||
- python3 -m ensurepip
|
||||
- pip3 install beautifulsoup4 requests
|
||||
- $pip install beautifulsoup4 requests
|
||||
- ./tests/gradle-release-checksums.py
|
||||
|
@ -73,7 +73,7 @@ __vercode() {
|
||||
line="${line#*,}"
|
||||
printf "${line%%,*} "
|
||||
fi
|
||||
done < "metadata/${p}.txt" )" -- $cur ) )
|
||||
done < "metadata/${p}.yml" )" -- $cur ) )
|
||||
}
|
||||
|
||||
__complete_options() {
|
||||
@ -253,7 +253,7 @@ __complete_btlog() {
|
||||
|
||||
__complete_mirror() {
|
||||
opts="-v"
|
||||
lopts="--archive --output-dir"
|
||||
lopts="--all --archive --build-logs --pgp-signatures --src-tarballs --output-dir"
|
||||
__complete_options
|
||||
}
|
||||
|
||||
|
@ -2,7 +2,7 @@ AuthorName: .
|
||||
WebSite: ''
|
||||
Bitcoin: null
|
||||
Litecoin: null
|
||||
Donation: null
|
||||
Donate: null
|
||||
|
||||
License: Unknown
|
||||
Categories:
|
||||
@ -17,5 +17,5 @@ Summary: .
|
||||
Description: |
|
||||
.
|
||||
|
||||
Archive Policy: 2 versions
|
||||
Requires Root: No
|
||||
ArchivePolicy: 2 versions
|
||||
RequiresRoot: No
|
||||
|
@ -64,7 +64,7 @@ def check_http(app):
|
||||
if len(urlcode) > 0:
|
||||
logging.debug("...requesting {0}".format(urlcode))
|
||||
req = urllib.request.Request(urlcode, None)
|
||||
resp = urllib.request.urlopen(req, None, 20)
|
||||
resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above
|
||||
page = resp.read().decode('utf-8')
|
||||
|
||||
m = re.search(codeex, page)
|
||||
@ -77,7 +77,7 @@ def check_http(app):
|
||||
if urlver != '.':
|
||||
logging.debug("...requesting {0}".format(urlver))
|
||||
req = urllib.request.Request(urlver, None)
|
||||
resp = urllib.request.urlopen(req, None, 20)
|
||||
resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above
|
||||
page = resp.read().decode('utf-8')
|
||||
|
||||
m = re.search(verex, page)
|
||||
@ -295,7 +295,7 @@ def check_gplay(app):
|
||||
headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
|
||||
req = urllib.request.Request(url, None, headers)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, None, 20)
|
||||
resp = urllib.request.urlopen(req, None, 20) # nosec B310 URL base is hardcoded above
|
||||
page = resp.read().decode()
|
||||
except urllib.error.HTTPError as e:
|
||||
return (None, str(e.code))
|
||||
@ -358,6 +358,18 @@ def possible_subdirs(app):
|
||||
yield subdir
|
||||
|
||||
|
||||
def _getappname(app):
|
||||
if app.Name:
|
||||
return app.Name
|
||||
if app.AutoName:
|
||||
return app.AutoName
|
||||
return app.id
|
||||
|
||||
|
||||
def _getcvname(app):
|
||||
return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
|
||||
|
||||
|
||||
def fetch_autoname(app, tag):
|
||||
|
||||
if not app.RepoType or app.UpdateCheckMode in ('None', 'Static') \
|
||||
@ -393,7 +405,7 @@ def fetch_autoname(app, tag):
|
||||
if new_name != app.AutoName:
|
||||
app.AutoName = new_name
|
||||
if not commitmsg:
|
||||
commitmsg = "Set autoname of {0}".format(common.getappname(app))
|
||||
commitmsg = "Set autoname of {0}".format(_getappname(app))
|
||||
else:
|
||||
logging.debug("...couldn't get autoname")
|
||||
|
||||
@ -472,8 +484,8 @@ def checkupdates_app(app):
|
||||
commitmsg = fetch_autoname(app, tag)
|
||||
|
||||
if updating:
|
||||
name = common.getappname(app)
|
||||
ver = common.getcvname(app)
|
||||
name = _getappname(app)
|
||||
ver = _getcvname(app)
|
||||
logging.info('...updating to version %s' % ver)
|
||||
commitmsg = 'Update CV of %s to %s' % (name, ver)
|
||||
|
||||
@ -513,8 +525,8 @@ def checkupdates_app(app):
|
||||
commit = commit.replace('%c', newbuild.versionCode)
|
||||
newbuild.commit = commit
|
||||
app.builds.append(newbuild)
|
||||
name = common.getappname(app)
|
||||
ver = common.getcvname(app)
|
||||
name = _getappname(app)
|
||||
ver = _getcvname(app)
|
||||
commitmsg = "Update %s to %s" % (name, ver)
|
||||
else:
|
||||
logging.warning('Invalid auto update mode "' + mode + '" on ' + app.id)
|
||||
@ -610,24 +622,24 @@ def main():
|
||||
version, reason = check_gplay(app)
|
||||
if version is None:
|
||||
if reason == '404':
|
||||
logging.info("{0} is not in the Play Store".format(common.getappname(app)))
|
||||
logging.info("{0} is not in the Play Store".format(_getappname(app)))
|
||||
else:
|
||||
logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
|
||||
logging.info("{0} encountered a problem: {1}".format(_getappname(app), reason))
|
||||
if version is not None:
|
||||
stored = app.CurrentVersion
|
||||
if not stored:
|
||||
logging.info("{0} has no Current Version but has version {1} on the Play Store"
|
||||
.format(common.getappname(app), version))
|
||||
.format(_getappname(app), version))
|
||||
elif LooseVersion(stored) < LooseVersion(version):
|
||||
logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
|
||||
.format(common.getappname(app), version, stored))
|
||||
.format(_getappname(app), version, stored))
|
||||
else:
|
||||
if stored != version:
|
||||
logging.info("{0} has version {1} on the Play Store, which differs from {2}"
|
||||
.format(common.getappname(app), version, stored))
|
||||
.format(_getappname(app), version, stored))
|
||||
else:
|
||||
logging.info("{0} has the same version {1} on the Play Store"
|
||||
.format(common.getappname(app), version))
|
||||
.format(_getappname(app), version))
|
||||
update_wiki(gplaylog, None)
|
||||
return
|
||||
|
||||
|
@ -664,18 +664,6 @@ def getsrcname(app, build):
|
||||
return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
|
||||
|
||||
|
||||
def getappname(app):
|
||||
if app.Name:
|
||||
return app.Name
|
||||
if app.AutoName:
|
||||
return app.AutoName
|
||||
return app.id
|
||||
|
||||
|
||||
def getcvname(app):
|
||||
return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
|
||||
|
||||
|
||||
def get_build_dir(app):
|
||||
'''get the dir that this app will be built in'''
|
||||
|
||||
|
@ -40,8 +40,9 @@ SETTINGS_GRADLE = re.compile(r'''include\s+['"]:([^'"]*)['"]''')
|
||||
# when one of these is found it's assumed that's the information we want.
|
||||
# Returns repotype, address, or None, reason
|
||||
def getrepofrompage(url):
|
||||
|
||||
req = urllib.request.urlopen(url)
|
||||
if not url.startswith('http'):
|
||||
return (None, _('{url} does not start with "http"!'.format(url=url)))
|
||||
req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
|
||||
if req.getcode() != 200:
|
||||
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
|
||||
page = req.read().decode(req.headers.get_content_charset())
|
||||
|
@ -46,16 +46,26 @@ def main():
|
||||
parser.add_argument("url", nargs='?',
|
||||
help=_('Base URL to mirror, can include the index signing key '
|
||||
+ 'using the query string: ?fingerprint='))
|
||||
parser.add_argument("--all", action='store_true', default=False,
|
||||
help=_("Mirror the full repo and archive, all file types."))
|
||||
parser.add_argument("--archive", action='store_true', default=False,
|
||||
help=_("Also mirror the full archive section"))
|
||||
parser.add_argument("--build-logs", action='store_true', default=False,
|
||||
help=_("Include the build logs in the mirror"))
|
||||
parser.add_argument("--pgp-signatures", action='store_true', default=False,
|
||||
help=_("Include the PGP signature .asc files in the mirror"))
|
||||
parser.add_argument("--src-tarballs", action='store_true', default=False,
|
||||
help=_("Include the source tarballs in the mirror"))
|
||||
parser.add_argument("--output-dir", default=None,
|
||||
help=_("The directory to write the mirror to"))
|
||||
options = parser.parse_args()
|
||||
|
||||
if options.all:
|
||||
options.archive = True
|
||||
options.build_logs = True
|
||||
options.pgp_signatures = True
|
||||
options.src_tarballs = True
|
||||
|
||||
if options.url is None:
|
||||
logging.error(_('A URL is required as an argument!') + '\n')
|
||||
parser.print_help()
|
||||
@ -152,7 +162,8 @@ def main():
|
||||
if not os.path.exists(f) \
|
||||
or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
|
||||
urls.append(_append_to_url_path(section, f))
|
||||
urls.append(_append_to_url_path(section, f + '.asc'))
|
||||
if options.pgp_signatures:
|
||||
urls.append(_append_to_url_path(section, f + '.asc'))
|
||||
if options.build_logs and f.endswith('.apk'):
|
||||
urls.append(_append_to_url_path(section, f[:-4] + '.log.gz'))
|
||||
|
||||
|
@ -714,7 +714,10 @@ def _set_localized_text_entry(app, locale, key, f):
|
||||
with open(f, errors='replace') as fp:
|
||||
text = fp.read()[:limit]
|
||||
if len(text) > 0:
|
||||
localized[key] = text
|
||||
if key in ('name', 'summary', 'video'): # hardcoded as a single line
|
||||
localized[key] = text.strip('\n')
|
||||
else:
|
||||
localized[key] = text
|
||||
|
||||
|
||||
def _set_author_entry(app, key, f):
|
||||
|
@ -362,8 +362,8 @@ CACHE_FILES = [
|
||||
'1f3067073041bc44554d0efe5d402a33bc3d3c93cc39ab684f308586d732a80d'),
|
||||
('https://services.gradle.org/distributions/gradle-6.0.1-bin.zip',
|
||||
'd364b7098b9f2e58579a3603dc0a12a1991353ac58ed339316e6762b21efba44'),
|
||||
('https://services.gradle.org/distributions/gradle-6.1-bin.zip',
|
||||
'd0c43d14e1c70a48b82442f435d06186351a2d290d72afd5b8866f15e6d7038a'),
|
||||
('https://services.gradle.org/distributions/gradle-6.1.1-bin.zip',
|
||||
'9d94e6e4a28ad328072ef6e56bce79a810494ae756751fdcedffdeaf27c093b1'),
|
||||
('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
|
||||
'102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
|
||||
('https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip',
|
||||
@ -531,6 +531,10 @@ def main():
|
||||
open(logfilename, 'a').close() # create blank file
|
||||
log_cm = vagrant.make_file_cm(logfilename)
|
||||
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
|
||||
# https://phoenhex.re/2018-03-25/not-a-vagrant-bug
|
||||
os_env = os.environ.copy()
|
||||
os_env['VAGRANT_DISABLE_VBOXSYMLINKCREATE'] = '1'
|
||||
v.env = os_env
|
||||
|
||||
if options.verbosity >= 2:
|
||||
tail = fdroidserver.tail.Tail(logfilename)
|
||||
|
@ -19,10 +19,11 @@ if localmodule not in sys.path:
|
||||
|
||||
import fdroidserver.checkupdates
|
||||
import fdroidserver.metadata
|
||||
from fdroidserver.exception import FDroidException
|
||||
|
||||
|
||||
class CommonTest(unittest.TestCase):
|
||||
'''fdroidserver/common.py'''
|
||||
class CheckupdatesTest(unittest.TestCase):
|
||||
'''fdroidserver/checkupdates.py'''
|
||||
|
||||
def setUp(self):
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
@ -123,6 +124,17 @@ class CommonTest(unittest.TestCase):
|
||||
self.assertEqual(vername, '1.1.9')
|
||||
self.assertEqual(vercode, '10109')
|
||||
|
||||
def test_check_http_blocks_unknown_schemes(self):
|
||||
app = fdroidserver.metadata.App()
|
||||
for scheme in ('file', 'ssh', 'http', ';pwn'):
|
||||
app.id = scheme
|
||||
faked = scheme + '://fake.url/for/testing/scheme'
|
||||
app.UpdateCheckData = faked + '|ignored|' + faked + '|ignored'
|
||||
app.metadatapath = 'metadata/' + app.id + '.yml'
|
||||
vername, vercode = fdroidserver.checkupdates.check_http(app)
|
||||
self.assertIsNone(vername)
|
||||
self.assertTrue(FDroidException.__name__ in vercode)
|
||||
|
||||
def test_check_http_ignore(self):
|
||||
fdroidserver.checkupdates.options = mock.Mock()
|
||||
|
||||
@ -151,5 +163,5 @@ if __name__ == "__main__":
|
||||
(fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
|
||||
|
||||
newSuite = unittest.TestSuite()
|
||||
newSuite.addTest(unittest.makeSuite(CommonTest))
|
||||
newSuite.addTest(unittest.makeSuite(CheckupdatesTest))
|
||||
unittest.main(failfast=False)
|
||||
|
@ -26,8 +26,8 @@ keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="
|
||||
keydname = "CN=sova, OU=F-Droid"
|
||||
|
||||
mirrors = (
|
||||
'https://foo.bar/fdroid',
|
||||
'http://foobarfoobarfoobar.onion/fdroid',
|
||||
'https://foo.bar/fdroid',
|
||||
)
|
||||
|
||||
update_stats = True
|
||||
|
@ -198,9 +198,9 @@
|
||||
"description": "full description\n",
|
||||
"featureGraphic": "featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png",
|
||||
"icon": "icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png",
|
||||
"name": "title\n",
|
||||
"summary": "short description\n",
|
||||
"video": "video\n"
|
||||
"name": "title",
|
||||
"summary": "short description",
|
||||
"video": "video"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -114,9 +114,9 @@ class UpdateTest(unittest.TestCase):
|
||||
if packageName == 'info.guardianproject.urzip':
|
||||
self.assertEqual(7, len(app['localized']['en-US']))
|
||||
self.assertEqual('full description\n', app['localized']['en-US']['description'])
|
||||
self.assertEqual('title\n', app['localized']['en-US']['name'])
|
||||
self.assertEqual('short description\n', app['localized']['en-US']['summary'])
|
||||
self.assertEqual('video\n', app['localized']['en-US']['video'])
|
||||
self.assertEqual('title', app['localized']['en-US']['name'])
|
||||
self.assertEqual('short description', app['localized']['en-US']['summary'])
|
||||
self.assertEqual('video', app['localized']['en-US']['video'])
|
||||
self.assertEqual('icon_NJXNzMcyf-v9i5a1ElJi0j9X1LvllibCa48xXYPlOqQ=.png',
|
||||
app['localized']['en-US']['icon'])
|
||||
self.assertEqual('featureGraphic_GFRT5BovZsENGpJq1HqPODGWBRPWQsx25B95Ol5w_wU=.png',
|
||||
@ -140,7 +140,7 @@ class UpdateTest(unittest.TestCase):
|
||||
elif packageName == 'com.nextcloud.client.dev':
|
||||
self.assertEqual('Nextcloud Dev', app['localized']['en-US']['name'])
|
||||
self.assertEqual(586, len(app['localized']['en-US']['description']))
|
||||
self.assertEqual(79, len(app['localized']['en-US']['summary']))
|
||||
self.assertEqual(78, len(app['localized']['en-US']['summary']))
|
||||
elif packageName == 'eu.siacs.conversations':
|
||||
self.assertEqual('Conversations', app['localized']['en-US']['name'])
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user