1
0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-11-20 13:50:12 +01:00

Merge branch 'pathlib' into 'master'

checkupdates.py: use pathlib and support Windows

See merge request fdroid/fdroidserver!935
This commit is contained in:
Jochen Sprickerhof 2021-06-07 13:38:21 +00:00
commit 3acaa7ede5
4 changed files with 80 additions and 69 deletions

View File

@ -21,7 +21,6 @@
import re import re
import sys import sys
import os import os
import locale
import pkgutil import pkgutil
import logging import logging
@ -198,7 +197,7 @@ def main():
else: else:
mod = __import__(available_plugins[command]['name'], None, None, [command]) mod = __import__(available_plugins[command]['name'], None, None, [command])
system_langcode, system_encoding = locale.getdefaultlocale() system_encoding = sys.getdefaultencoding()
if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'): if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'):
logging.warning(_("Encoding is set to '{enc}' fdroid might run " logging.warning(_("Encoding is set to '{enc}' fdroid might run "
"into encoding issues. Please set it to 'UTF-8' " "into encoding issues. Please set it to 'UTF-8' "

View File

@ -31,6 +31,7 @@ from distutils.version import LooseVersion
import logging import logging
import copy import copy
import urllib.parse import urllib.parse
from pathlib import Path
from . import _ from . import _
from . import common from . import common
@ -62,7 +63,7 @@ def check_http(app):
raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode)) raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
vercode = None vercode = None
if len(urlcode) > 0: if urlcode:
logging.debug("...requesting {0}".format(urlcode)) logging.debug("...requesting {0}".format(urlcode))
req = urllib.request.Request(urlcode, None, headers=net.HEADERS) req = urllib.request.Request(urlcode, None, headers=net.HEADERS)
resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above
@ -74,7 +75,7 @@ def check_http(app):
vercode = m.group(1).strip() vercode = m.group(1).strip()
version = "??" version = "??"
if len(urlver) > 0: if urlver:
if urlver != '.': if urlver != '.':
logging.debug("...requesting {0}".format(urlver)) logging.debug("...requesting {0}".format(urlver))
req = urllib.request.Request(urlver, None) req = urllib.request.Request(urlver, None)
@ -109,10 +110,10 @@ def check_tags(app, pattern):
try: try:
if app.RepoType == 'srclib': if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo) build_dir = Path('build/srclib') / app.Repo
repotype = common.getsrclibvcs(app.Repo) repotype = common.getsrclibvcs(app.Repo)
else: else:
build_dir = os.path.join('build', app.id) build_dir = Path('build') / app.id
repotype = app.RepoType repotype = app.RepoType
if repotype not in ('git', 'git-svn', 'hg', 'bzr'): if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
@ -161,8 +162,8 @@ def check_tags(app, pattern):
if app.UpdateCheckData: if app.UpdateCheckData:
filecode, codeex, filever, verex = app.UpdateCheckData.split('|') filecode, codeex, filever, verex = app.UpdateCheckData.split('|')
vercode = None vercode = None
if len(filecode) > 0: if filecode:
filecontent = open(os.path.join(build_dir, filecode)).read() filecontent = (build_dir / filecode).read_text()
m = re.search(codeex, filecontent) m = re.search(codeex, filecontent)
if not m: if not m:
@ -170,9 +171,9 @@ def check_tags(app, pattern):
vercode = m.group(1).strip() vercode = m.group(1).strip()
version = "??" version = "??"
if len(filever) > 0: if filever:
if filever != '.': if filever != '.':
filecontent = open(os.path.join(build_dir, filever)).read() filecontent = (build_dir / filever).read_text()
m = re.search(verex, filecontent) m = re.search(verex, filecontent)
if not m: if not m:
@ -189,12 +190,9 @@ def check_tags(app, pattern):
hver = version hver = version
else: else:
for subdir in possible_subdirs(app): for subdir in possible_subdirs(app):
if subdir == '.': root_dir = build_dir / subdir
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, last_build.gradle) paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app) version, vercode, _package = common.parse_androidmanifests(paths, app)
if vercode: if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
.format(subdir, version, vercode)) .format(subdir, version, vercode))
@ -227,10 +225,10 @@ def check_repomanifest(app, branch=None):
try: try:
if app.RepoType == 'srclib': if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo) build_dir = Path('build/srclib') / app.Repo
repotype = common.getsrclibvcs(app.Repo) repotype = common.getsrclibvcs(app.Repo)
else: else:
build_dir = os.path.join('build', app.id) build_dir = Path('build') / app.id
repotype = app.RepoType repotype = app.RepoType
# Set up vcs interface and make sure we have the latest code... # Set up vcs interface and make sure we have the latest code...
@ -248,7 +246,7 @@ def check_repomanifest(app, branch=None):
vcs.gotorevision(None) vcs.gotorevision(None)
last_build = metadata.Build() last_build = metadata.Build()
if len(app.get('Builds', [])) > 0: if app.get('Builds', []):
last_build = app.get('Builds', [])[-1] last_build = app.get('Builds', [])[-1]
try_init_submodules(app, last_build, vcs) try_init_submodules(app, last_build, vcs)
@ -257,10 +255,7 @@ def check_repomanifest(app, branch=None):
hver = None hver = None
hcode = "0" hcode = "0"
for subdir in possible_subdirs(app): for subdir in possible_subdirs(app):
if subdir == '.': root_dir = build_dir / subdir
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, last_build.gradle) paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app) version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode: if vercode:
@ -290,10 +285,10 @@ def check_repotrunk(app):
try: try:
if app.RepoType == 'srclib': if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo) build_dir = Path('build/srclib') / app.Repo
repotype = common.getsrclibvcs(app.Repo) repotype = common.getsrclibvcs(app.Repo)
else: else:
build_dir = os.path.join('build', app.id) build_dir = Path('build') / app.id
repotype = app.RepoType repotype = app.RepoType
if repotype not in ('git-svn', ): if repotype not in ('git-svn', ):
@ -360,10 +355,11 @@ def try_init_submodules(app, last_build, vcs):
# Return all directories under startdir that contain any of the manifest # Return all directories under startdir that contain any of the manifest
# files, and thus are probably an Android project. # files, and thus are probably an Android project.
def dirs_with_manifest(startdir): def dirs_with_manifest(startdir):
for root, dirs, files in os.walk(startdir): # TODO: Python3.6: Accepts a path-like object.
for root, _dirs, files in os.walk(str(startdir)):
if any(m in files for m in [ if any(m in files for m in [
'AndroidManifest.xml', 'pom.xml', 'build.gradle', 'build.gradle.kts']): 'AndroidManifest.xml', 'pom.xml', 'build.gradle', 'build.gradle.kts']):
yield root yield Path(root)
# Tries to find a new subdir starting from the root build_dir. Returns said # Tries to find a new subdir starting from the root build_dir. Returns said
@ -371,9 +367,9 @@ def dirs_with_manifest(startdir):
def possible_subdirs(app): def possible_subdirs(app):
if app.RepoType == 'srclib': if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo) build_dir = Path('build/srclib') / app.Repo
else: else:
build_dir = os.path.join('build', app.id) build_dir = Path('build') / app.id
last_build = app.get_last_build() last_build = app.get_last_build()
@ -381,7 +377,7 @@ def possible_subdirs(app):
m_paths = common.manifest_paths(d, last_build.gradle) m_paths = common.manifest_paths(d, last_build.gradle)
package = common.parse_androidmanifests(m_paths, app)[2] package = common.parse_androidmanifests(m_paths, app)[2]
if package is not None: if package is not None:
subdir = os.path.relpath(d, build_dir) subdir = d.relative_to(build_dir)
logging.debug("Adding possible subdir %s" % subdir) logging.debug("Adding possible subdir %s" % subdir)
yield subdir yield subdir
@ -401,9 +397,9 @@ def fetch_autoname(app, tag):
return None return None
if app.RepoType == 'srclib': if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo) build_dir = Path('build/srclib') / app.Repo
else: else:
build_dir = os.path.join('build', app.id) build_dir = Path('build') / app.id
try: try:
vcs = common.getvcs(app.RepoType, app.Repo, build_dir) vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
@ -413,13 +409,10 @@ def fetch_autoname(app, tag):
last_build = app.get_last_build() last_build = app.get_last_build()
logging.debug("...fetch auto name from " + build_dir) logging.debug("...fetch auto name from " + str(build_dir))
new_name = None new_name = None
for subdir in possible_subdirs(app): for subdir in possible_subdirs(app):
if subdir == '.': root_dir = build_dir / subdir
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
new_name = common.fetch_real_name(root_dir, last_build.gradle) new_name = common.fetch_real_name(root_dir, last_build.gradle)
if new_name is not None: if new_name is not None:
break break

View File

@ -51,6 +51,7 @@ import yaml
import zipfile import zipfile
import tempfile import tempfile
import json import json
from pathlib import Path
# TODO change to only import defusedxml once its installed everywhere # TODO change to only import defusedxml once its installed everywhere
try: try:
@ -119,13 +120,13 @@ orig_path = None
default_config = { default_config = {
'sdk_path': "$ANDROID_HOME", 'sdk_path': "$ANDROID_HOME",
'ndk_paths': {}, 'ndk_paths': {},
'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'cachedir': str(Path.home() / '.cache/fdroidserver'),
'java_paths': None, 'java_paths': None,
'scan_binary': False, 'scan_binary': False,
'ant': "ant", 'ant': "ant",
'mvn3': "mvn", 'mvn3': "mvn",
'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'), 'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'),
'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'), 'gradle_version_dir': str(Path.home() / '.cache/fdroidserver/gradle'),
'sync_from_local_copy_dir': False, 'sync_from_local_copy_dir': False,
'allow_disabled_algorithms': False, 'allow_disabled_algorithms': False,
'per_app_repos': False, 'per_app_repos': False,
@ -884,6 +885,8 @@ def setup_vcs(app):
def getvcs(vcstype, remote, local): def getvcs(vcstype, remote, local):
# TODO: Remove this in Python3.6
local = str(local)
if vcstype == 'git': if vcstype == 'git':
return vcs_git(remote, local) return vcs_git(remote, local)
if vcstype == 'git-svn': if vcstype == 'git-svn':
@ -1082,13 +1085,13 @@ class vcs_git(vcs):
p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False) p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
result = p.output.rstrip() result = p.output.rstrip()
if not result.endswith(self.local): if Path(result) != Path(self.local).resolve():
raise VCSException('Repository mismatch') raise VCSException('Repository mismatch')
def gotorevisionx(self, rev): def gotorevisionx(self, rev):
if not os.path.exists(self.local): if not os.path.exists(self.local):
# Brand new checkout # Brand new checkout
p = self.git(['clone', '--', self.remote, self.local]) p = self.git(['clone', '--', self.remote, str(self.local)])
if p.returncode != 0: if p.returncode != 0:
self.clone_failed = True self.clone_failed = True
raise VCSException("Git clone failed", p.output) raise VCSException("Git clone failed", p.output)
@ -1197,7 +1200,7 @@ class vcs_gitsvn(vcs):
""" """
p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False) p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
result = p.output.rstrip() result = p.output.rstrip()
if not result.endswith(self.local): if Path(result) != Path(self.local).resolve():
raise VCSException('Repository mismatch') raise VCSException('Repository mismatch')
def git(self, args, envs=dict(), cwd=None, output=True): def git(self, args, envs=dict(), cwd=None, output=True):
@ -1258,7 +1261,7 @@ class vcs_gitsvn(vcs):
raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ') raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ')
.format(before=remote, after=location)) .format(before=remote, after=location))
gitsvn_args.extend(['--', remote, self.local]) gitsvn_args.extend(['--', remote, str(self.local)])
p = self.git(gitsvn_args) p = self.git(gitsvn_args)
if p.returncode != 0: if p.returncode != 0:
self.clone_failed = True self.clone_failed = True
@ -1358,7 +1361,7 @@ class vcs_hg(vcs):
def gotorevisionx(self, rev): def gotorevisionx(self, rev):
if not os.path.exists(self.local): if not os.path.exists(self.local):
p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local], p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, str(self.local)],
output=False) output=False)
if p.returncode != 0: if p.returncode != 0:
self.clone_failed = True self.clone_failed = True
@ -1416,7 +1419,7 @@ class vcs_bzr(vcs):
def gotorevisionx(self, rev): def gotorevisionx(self, rev):
if not os.path.exists(self.local): if not os.path.exists(self.local):
p = self.bzr(['branch', self.remote, self.local], output=False) p = self.bzr(['branch', self.remote, str(self.local)], output=False)
if p.returncode != 0: if p.returncode != 0:
self.clone_failed = True self.clone_failed = True
raise VCSException("Bzr branch failed", p.output) raise VCSException("Bzr branch failed", p.output)
@ -1494,6 +1497,8 @@ def retrieve_string_singleline(app_dir, string, xmlfiles=None):
def manifest_paths(app_dir, flavours): def manifest_paths(app_dir, flavours):
'''Return list of existing files that will be used to find the highest vercode''' '''Return list of existing files that will be used to find the highest vercode'''
# TODO: Remove this in Python3.6
app_dir = str(app_dir)
possible_manifests = \ possible_manifests = \
[os.path.join(app_dir, 'AndroidManifest.xml'), [os.path.join(app_dir, 'AndroidManifest.xml'),
os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'), os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
@ -1513,6 +1518,8 @@ def manifest_paths(app_dir, flavours):
def fetch_real_name(app_dir, flavours): def fetch_real_name(app_dir, flavours):
'''Retrieve the package name. Returns the name, or None if not found.''' '''Retrieve the package name. Returns the name, or None if not found.'''
# TODO: Remove this in Python3.6
app_dir = str(app_dir)
for path in manifest_paths(app_dir, flavours): for path in manifest_paths(app_dir, flavours):
if not path.endswith('.xml') or not os.path.isfile(path): if not path.endswith('.xml') or not os.path.isfile(path):
continue continue

View File

@ -2,19 +2,17 @@
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
import inspect
import logging import logging
import optparse import optparse
import os import os
import sys import sys
import unittest import unittest
from unittest import mock from unittest import mock
from pathlib import Path
localmodule = os.path.realpath( localmodule = Path(__file__).resolve().parent.parent
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') print('localmodule: ' + str(localmodule))
)
print('localmodule: ' + localmodule)
if localmodule not in sys.path: if localmodule not in sys.path:
sys.path.insert(0, localmodule) sys.path.insert(0, localmodule)
@ -28,11 +26,11 @@ class CheckupdatesTest(unittest.TestCase):
def setUp(self): def setUp(self):
logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests') self.basedir = localmodule / 'tests'
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles')) self.tmpdir = localmodule / '.testfiles'
if not os.path.exists(self.tmpdir): self.tmpdir.mkdir(exist_ok=True)
os.makedirs(self.tmpdir) # TODO: Python3.6: Accepts a path-like object.
os.chdir(self.basedir) os.chdir(str(self.basedir))
def test_autoupdatemode_no_suffix(self): def test_autoupdatemode_no_suffix(self):
fdroidserver.checkupdates.options = mock.Mock() fdroidserver.checkupdates.options = mock.Mock()
@ -52,7 +50,9 @@ class CheckupdatesTest(unittest.TestCase):
build.versionName = app.CurrentVersion build.versionName = app.CurrentVersion
app['Builds'].append(build) app['Builds'].append(build)
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)): with mock.patch(
'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
):
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()): with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
with mock.patch('subprocess.call', lambda cmd: 0): with mock.patch('subprocess.call', lambda cmd: 0):
fdroidserver.checkupdates.checkupdates_app(app) fdroidserver.checkupdates.checkupdates_app(app)
@ -71,14 +71,16 @@ class CheckupdatesTest(unittest.TestCase):
app.CurrentVersion = '1.1.8-fdroid' app.CurrentVersion = '1.1.8-fdroid'
app.CurrentVersionCode = 10108 app.CurrentVersionCode = 10108
app.UpdateCheckMode = 'HTTP' app.UpdateCheckMode = 'HTTP'
app.AutoUpdateMode = 'Version +.%c-fdroid v%v_%c' app.AutoUpdateMode = r'Version +.%c-fdroid v%v_%c'
build = fdroidserver.metadata.Build() build = fdroidserver.metadata.Build()
build.versionCode = app.CurrentVersionCode build.versionCode = app.CurrentVersionCode
build.versionName = app.CurrentVersion build.versionName = app.CurrentVersion
app['Builds'].append(build) app['Builds'].append(build)
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)): with mock.patch(
'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
):
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()): with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
with mock.patch('subprocess.call', lambda cmd: 0): with mock.patch('subprocess.call', lambda cmd: 0):
fdroidserver.checkupdates.checkupdates_app(app) fdroidserver.checkupdates.checkupdates_app(app)
@ -98,11 +100,17 @@ class CheckupdatesTest(unittest.TestCase):
app.UpdateCheckMode = 'HTTP' app.UpdateCheckMode = 'HTTP'
app.UpdateCheckData = 'mock' app.UpdateCheckData = 'mock'
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: (None, 'bla')): with mock.patch(
'fdroidserver.checkupdates.check_http', lambda app: (None, 'bla')
):
fdroidserver.checkupdates.checkupdates_app(app) fdroidserver.checkupdates.checkupdates_app(app)
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)): with mock.patch(
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()) as wrmock: 'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
):
with mock.patch(
'fdroidserver.metadata.write_metadata', mock.Mock()
) as wrmock:
with mock.patch('subprocess.call', lambda cmd: 0): with mock.patch('subprocess.call', lambda cmd: 0):
fdroidserver.checkupdates.checkupdates_app(app) fdroidserver.checkupdates.checkupdates_app(app)
wrmock.assert_called_with(app.metadatapath, app) wrmock.assert_called_with(app.metadatapath, app)
@ -125,14 +133,17 @@ class CheckupdatesTest(unittest.TestCase):
build.versionName = app.CurrentVersion build.versionName = app.CurrentVersion
app['Builds'].append(build) app['Builds'].append(build)
with mock.patch('fdroidserver.checkupdates.check_tags', lambda app, with mock.patch(
pattern: (None, 'bla', None)): 'fdroidserver.checkupdates.check_tags',
lambda app, pattern: (None, 'bla', None),
):
fdroidserver.checkupdates.checkupdates_app(app) fdroidserver.checkupdates.checkupdates_app(app)
with mock.patch('fdroidserver.checkupdates.check_tags', lambda app, with mock.patch(
pattern: ('1.1.9', 10109, 'v1.1.9')): 'fdroidserver.checkupdates.check_tags',
with mock.patch('fdroidserver.metadata.write_metadata', lambda app, pattern: ('1.1.9', 10109, 'v1.1.9'),
mock.Mock()): ):
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
with mock.patch('subprocess.call', lambda cmd: 0): with mock.patch('subprocess.call', lambda cmd: 0):
fdroidserver.checkupdates.checkupdates_app(app) fdroidserver.checkupdates.checkupdates_app(app)
build = app['Builds'][-1] build = app['Builds'][-1]
@ -200,16 +211,17 @@ class CheckupdatesTest(unittest.TestCase):
vcs = mock.Mock() vcs = mock.Mock()
vcs.latesttags.return_value = ['1.1.8', '1.1.9'] vcs.latesttags.return_value = ['1.1.8', '1.1.9']
with mock.patch( with mock.patch(
'builtins.open', mock.mock_open(read_data='v1.1.9\nc10109') 'pathlib.Path.read_text', lambda a: 'v1.1.9\nc10109'
) as _ignored, mock.patch('fdroidserver.common.getvcs', return_value=vcs): ) as _ignored, mock.patch('fdroidserver.common.getvcs', return_value=vcs):
_ignored # silence the linters _ignored # silence the linters
vername, vercode, tag = fdroidserver.checkupdates.check_tags(app, None) vername, vercode, _tag = fdroidserver.checkupdates.check_tags(app, None)
self.assertEqual(vername, '1.1.9') self.assertEqual(vername, '1.1.9')
self.assertEqual(vercode, '10109') self.assertEqual(vercode, '10109')
if __name__ == "__main__": if __name__ == "__main__":
os.chdir(os.path.dirname(__file__)) # TODO: Python3.6: Accept path-like object.
os.chdir(str(Path(__file__).parent))
parser = optparse.OptionParser() parser = optparse.OptionParser()
parser.add_option( parser.add_option(