kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-20 13:50:12 +01:00
Code Activity

Merge branch 'pathlib' into 'master'

Browse Source 
checkupdates.py: use pathlib and support Windows

See merge request fdroid/fdroidserver!935
This commit is contained in:
Jochen Sprickerhof 2021-06-07 13:38:21 +00:00
commit 3acaa7ede5
4 changed files with 80 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
fdroidserver/__main__.py
View File

@ -21,7 +21,6 @@
import re
import sys
import os
import locale
import pkgutil
import logging
@ -198,7 +197,7 @@ def main():
else:
mod = __import__(available_plugins[command]['name'], None, None, [command])
system_langcode, system_encoding = locale.getdefaultlocale()
system_encoding = sys.getdefaultencoding()
if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'):
logging.warning(_("Encoding is set to '{enc}' fdroid might run "
"into encoding issues. Please set it to 'UTF-8' "

61
fdroidserver/checkupdates.py
View File

@ -31,6 +31,7 @@ from distutils.version import LooseVersion
import logging
import copy
import urllib.parse
from pathlib import Path
from . import _
from . import common
@ -62,7 +63,7 @@ def check_http(app):
raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
vercode = None
if len(urlcode) > 0:
if urlcode:
logging.debug("...requesting {0}".format(urlcode))
req = urllib.request.Request(urlcode, None, headers=net.HEADERS)
resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above
@ -74,7 +75,7 @@ def check_http(app):
vercode = m.group(1).strip()
version = "??"
if len(urlver) > 0:
if urlver:
if urlver != '.':
logging.debug("...requesting {0}".format(urlver))
req = urllib.request.Request(urlver, None)
@ -109,10 +110,10 @@ def check_tags(app, pattern):
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
build_dir = Path('build/srclib') / app.Repo
repotype = common.getsrclibvcs(app.Repo)
else:
build_dir = os.path.join('build', app.id)
build_dir = Path('build') / app.id
repotype = app.RepoType
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
@ -161,8 +162,8 @@ def check_tags(app, pattern):
if app.UpdateCheckData:
filecode, codeex, filever, verex = app.UpdateCheckData.split('|')
vercode = None
if len(filecode) > 0:
filecontent = open(os.path.join(build_dir, filecode)).read()
if filecode:
filecontent = (build_dir / filecode).read_text()
m = re.search(codeex, filecontent)
if not m:
@ -170,9 +171,9 @@ def check_tags(app, pattern):
vercode = m.group(1).strip()
version = "??"
if len(filever) > 0:
if filever:
if filever != '.':
filecontent = open(os.path.join(build_dir, filever)).read()
filecontent = (build_dir / filever).read_text()
m = re.search(verex, filecontent)
if not m:
@ -189,12 +190,9 @@ def check_tags(app, pattern):
hver = version
else:
for subdir in possible_subdirs(app):
if subdir == '.':
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
root_dir = build_dir / subdir
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
version, vercode, _package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
.format(subdir, version, vercode))
@ -227,10 +225,10 @@ def check_repomanifest(app, branch=None):
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
build_dir = Path('build/srclib') / app.Repo
repotype = common.getsrclibvcs(app.Repo)
else:
build_dir = os.path.join('build', app.id)
build_dir = Path('build') / app.id
repotype = app.RepoType
# Set up vcs interface and make sure we have the latest code...
@ -248,7 +246,7 @@ def check_repomanifest(app, branch=None):
vcs.gotorevision(None)
last_build = metadata.Build()
if len(app.get('Builds', [])) > 0:
if app.get('Builds', []):
last_build = app.get('Builds', [])[-1]
try_init_submodules(app, last_build, vcs)
@ -257,10 +255,7 @@ def check_repomanifest(app, branch=None):
hver = None
hcode = "0"
for subdir in possible_subdirs(app):
if subdir == '.':
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
root_dir = build_dir / subdir
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
@ -290,10 +285,10 @@ def check_repotrunk(app):
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
build_dir = Path('build/srclib') / app.Repo
repotype = common.getsrclibvcs(app.Repo)
else:
build_dir = os.path.join('build', app.id)
build_dir = Path('build') / app.id
repotype = app.RepoType
if repotype not in ('git-svn', ):
@ -360,10 +355,11 @@ def try_init_submodules(app, last_build, vcs):
# Return all directories under startdir that contain any of the manifest
# files, and thus are probably an Android project.
def dirs_with_manifest(startdir):
for root, dirs, files in os.walk(startdir):
# TODO: Python3.6: Accepts a path-like object.
for root, _dirs, files in os.walk(str(startdir)):
if any(m in files for m in [
'AndroidManifest.xml', 'pom.xml', 'build.gradle', 'build.gradle.kts']):
yield root
yield Path(root)
# Tries to find a new subdir starting from the root build_dir. Returns said
@ -371,9 +367,9 @@ def dirs_with_manifest(startdir):
def possible_subdirs(app):
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
build_dir = Path('build/srclib') / app.Repo
else:
build_dir = os.path.join('build', app.id)
build_dir = Path('build') / app.id
last_build = app.get_last_build()
@ -381,7 +377,7 @@ def possible_subdirs(app):
m_paths = common.manifest_paths(d, last_build.gradle)
package = common.parse_androidmanifests(m_paths, app)[2]
if package is not None:
subdir = os.path.relpath(d, build_dir)
subdir = d.relative_to(build_dir)
logging.debug("Adding possible subdir %s" % subdir)
yield subdir
@ -401,9 +397,9 @@ def fetch_autoname(app, tag):
return None
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
build_dir = Path('build/srclib') / app.Repo
else:
build_dir = os.path.join('build', app.id)
build_dir = Path('build') / app.id
try:
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
@ -413,13 +409,10 @@ def fetch_autoname(app, tag):
last_build = app.get_last_build()
logging.debug("...fetch auto name from " + build_dir)
logging.debug("...fetch auto name from " + str(build_dir))
new_name = None
for subdir in possible_subdirs(app):
if subdir == '.':
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
root_dir = build_dir / subdir
new_name = common.fetch_real_name(root_dir, last_build.gradle)
if new_name is not None:
break

23
fdroidserver/common.py
View File

@ -51,6 +51,7 @@ import yaml
import zipfile
import tempfile
import json
from pathlib import Path
# TODO change to only import defusedxml once its installed everywhere
try:
@ -119,13 +120,13 @@ orig_path = None
default_config = {
'sdk_path': "$ANDROID_HOME",
'ndk_paths': {},
'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
'cachedir': str(Path.home() / '.cache/fdroidserver'),
'java_paths': None,
'scan_binary': False,
'ant': "ant",
'mvn3': "mvn",
'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'),
'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'),
'gradle_version_dir': str(Path.home() / '.cache/fdroidserver/gradle'),
'sync_from_local_copy_dir': False,
'allow_disabled_algorithms': False,
'per_app_repos': False,
@ -884,6 +885,8 @@ def setup_vcs(app):
def getvcs(vcstype, remote, local):
# TODO: Remove this in Python3.6
local = str(local)
if vcstype == 'git':
return vcs_git(remote, local)
if vcstype == 'git-svn':
@ -1082,13 +1085,13 @@ class vcs_git(vcs):
p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
result = p.output.rstrip()
if not result.endswith(self.local):
if Path(result) != Path(self.local).resolve():
raise VCSException('Repository mismatch')
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
# Brand new checkout
p = self.git(['clone', '--', self.remote, self.local])
p = self.git(['clone', '--', self.remote, str(self.local)])
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Git clone failed", p.output)
@ -1197,7 +1200,7 @@ class vcs_gitsvn(vcs):
"""
p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
result = p.output.rstrip()
if not result.endswith(self.local):
if Path(result) != Path(self.local).resolve():
raise VCSException('Repository mismatch')
def git(self, args, envs=dict(), cwd=None, output=True):
@ -1258,7 +1261,7 @@ class vcs_gitsvn(vcs):
raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ')
.format(before=remote, after=location))
gitsvn_args.extend(['--', remote, self.local])
gitsvn_args.extend(['--', remote, str(self.local)])
p = self.git(gitsvn_args)
if p.returncode != 0:
self.clone_failed = True
@ -1358,7 +1361,7 @@ class vcs_hg(vcs):
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local],
p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, str(self.local)],
output=False)
if p.returncode != 0:
self.clone_failed = True
@ -1416,7 +1419,7 @@ class vcs_bzr(vcs):
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
p = self.bzr(['branch', self.remote, self.local], output=False)
p = self.bzr(['branch', self.remote, str(self.local)], output=False)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Bzr branch failed", p.output)
@ -1494,6 +1497,8 @@ def retrieve_string_singleline(app_dir, string, xmlfiles=None):
def manifest_paths(app_dir, flavours):
'''Return list of existing files that will be used to find the highest vercode'''
# TODO: Remove this in Python3.6
app_dir = str(app_dir)
possible_manifests = \
[os.path.join(app_dir, 'AndroidManifest.xml'),
os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
@ -1513,6 +1518,8 @@ def manifest_paths(app_dir, flavours):
def fetch_real_name(app_dir, flavours):
'''Retrieve the package name. Returns the name, or None if not found.'''
# TODO: Remove this in Python3.6
app_dir = str(app_dir)
for path in manifest_paths(app_dir, flavours):
if not path.endswith('.xml') or not os.path.isfile(path):
continue

62
tests/checkupdates.TestCase
View File

@ -2,19 +2,17 @@
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
import inspect
import logging
import optparse
import os
import sys
import unittest
from unittest import mock
from pathlib import Path
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
)
print('localmodule: ' + localmodule)
localmodule = Path(__file__).resolve().parent.parent
print('localmodule: ' + str(localmodule))
if localmodule not in sys.path:
sys.path.insert(0, localmodule)
@ -28,11 +26,11 @@ class CheckupdatesTest(unittest.TestCase):
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
self.basedir = localmodule / 'tests'
self.tmpdir = localmodule / '.testfiles'
self.tmpdir.mkdir(exist_ok=True)
# TODO: Python3.6: Accepts a path-like object.
os.chdir(str(self.basedir))
def test_autoupdatemode_no_suffix(self):
fdroidserver.checkupdates.options = mock.Mock()
@ -52,7 +50,9 @@ class CheckupdatesTest(unittest.TestCase):
build.versionName = app.CurrentVersion
app['Builds'].append(build)
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)):
with mock.patch(
'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
):
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
with mock.patch('subprocess.call', lambda cmd: 0):
fdroidserver.checkupdates.checkupdates_app(app)
@ -71,14 +71,16 @@ class CheckupdatesTest(unittest.TestCase):
app.CurrentVersion = '1.1.8-fdroid'
app.CurrentVersionCode = 10108
app.UpdateCheckMode = 'HTTP'
app.AutoUpdateMode = 'Version +.%c-fdroid v%v_%c'
app.AutoUpdateMode = r'Version +.%c-fdroid v%v_%c'
build = fdroidserver.metadata.Build()
build.versionCode = app.CurrentVersionCode
build.versionName = app.CurrentVersion
app['Builds'].append(build)
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)):
with mock.patch(
'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
):
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
with mock.patch('subprocess.call', lambda cmd: 0):
fdroidserver.checkupdates.checkupdates_app(app)
@ -98,11 +100,17 @@ class CheckupdatesTest(unittest.TestCase):
app.UpdateCheckMode = 'HTTP'
app.UpdateCheckData = 'mock'
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: (None, 'bla')):
with mock.patch(
'fdroidserver.checkupdates.check_http', lambda app: (None, 'bla')
):
fdroidserver.checkupdates.checkupdates_app(app)
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)):
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()) as wrmock:
with mock.patch(
'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
):
with mock.patch(
'fdroidserver.metadata.write_metadata', mock.Mock()
) as wrmock:
with mock.patch('subprocess.call', lambda cmd: 0):
fdroidserver.checkupdates.checkupdates_app(app)
wrmock.assert_called_with(app.metadatapath, app)
@ -125,14 +133,17 @@ class CheckupdatesTest(unittest.TestCase):
build.versionName = app.CurrentVersion
app['Builds'].append(build)
with mock.patch('fdroidserver.checkupdates.check_tags', lambda app,
pattern: (None, 'bla', None)):
with mock.patch(
'fdroidserver.checkupdates.check_tags',
lambda app, pattern: (None, 'bla', None),
):
fdroidserver.checkupdates.checkupdates_app(app)
with mock.patch('fdroidserver.checkupdates.check_tags', lambda app,
pattern: ('1.1.9', 10109, 'v1.1.9')):
with mock.patch('fdroidserver.metadata.write_metadata',
mock.Mock()):
with mock.patch(
'fdroidserver.checkupdates.check_tags',
lambda app, pattern: ('1.1.9', 10109, 'v1.1.9'),
):
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
with mock.patch('subprocess.call', lambda cmd: 0):
fdroidserver.checkupdates.checkupdates_app(app)
build = app['Builds'][-1]
@ -200,16 +211,17 @@ class CheckupdatesTest(unittest.TestCase):
vcs = mock.Mock()
vcs.latesttags.return_value = ['1.1.8', '1.1.9']
with mock.patch(
'builtins.open', mock.mock_open(read_data='v1.1.9\nc10109')
'pathlib.Path.read_text', lambda a: 'v1.1.9\nc10109'
) as _ignored, mock.patch('fdroidserver.common.getvcs', return_value=vcs):
_ignored # silence the linters
vername, vercode, tag = fdroidserver.checkupdates.check_tags(app, None)
vername, vercode, _tag = fdroidserver.checkupdates.check_tags(app, None)
self.assertEqual(vername, '1.1.9')
self.assertEqual(vercode, '10109')
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))
# TODO: Python3.6: Accept path-like object.
os.chdir(str(Path(__file__).parent))
parser = optparse.OptionParser()
parser.add_option(