mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-20 13:50:12 +01:00
checkupdates.py: use pathlib and support Windows
This commit is contained in:
linsui
parent
023319efbb
commit
92ec6265a3
@ -21,7 +21,6 @@
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
import locale
|
||||
import pkgutil
|
||||
import logging
|
||||
|
||||
@ -198,7 +197,7 @@ def main():
|
||||
else:
|
||||
mod = __import__(available_plugins[command]['name'], None, None, [command])
|
||||
|
||||
system_langcode, system_encoding = locale.getdefaultlocale()
|
||||
system_encoding = sys.getdefaultencoding()
|
||||
if system_encoding is None or system_encoding.lower() not in ('utf-8', 'utf8'):
|
||||
logging.warning(_("Encoding is set to '{enc}' fdroid might run "
|
||||
"into encoding issues. Please set it to 'UTF-8' "
|
||||
|
||||
@ -31,6 +31,7 @@ from distutils.version import LooseVersion
|
||||
import logging
|
||||
import copy
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
from . import _
|
||||
from . import common
|
||||
@ -62,7 +63,7 @@ def check_http(app):
|
||||
raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
|
||||
|
||||
vercode = None
|
||||
if len(urlcode) > 0:
|
||||
if urlcode:
|
||||
logging.debug("...requesting {0}".format(urlcode))
|
||||
req = urllib.request.Request(urlcode, None, headers=net.HEADERS)
|
||||
resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above
|
||||
@ -74,7 +75,7 @@ def check_http(app):
|
||||
vercode = m.group(1).strip()
|
||||
|
||||
version = "??"
|
||||
if len(urlver) > 0:
|
||||
if urlver:
|
||||
if urlver != '.':
|
||||
logging.debug("...requesting {0}".format(urlver))
|
||||
req = urllib.request.Request(urlver, None)
|
||||
@ -109,10 +110,10 @@ def check_tags(app, pattern):
|
||||
try:
|
||||
|
||||
if app.RepoType == 'srclib':
|
||||
build_dir = os.path.join('build', 'srclib', app.Repo)
|
||||
build_dir = Path('build/srclib') / app.Repo
|
||||
repotype = common.getsrclibvcs(app.Repo)
|
||||
else:
|
||||
build_dir = os.path.join('build', app.id)
|
||||
build_dir = Path('build') / app.id
|
||||
repotype = app.RepoType
|
||||
|
||||
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
|
||||
@ -161,8 +162,8 @@ def check_tags(app, pattern):
|
||||
if app.UpdateCheckData:
|
||||
filecode, codeex, filever, verex = app.UpdateCheckData.split('|')
|
||||
vercode = None
|
||||
if len(filecode) > 0:
|
||||
filecontent = open(os.path.join(build_dir, filecode)).read()
|
||||
if filecode:
|
||||
filecontent = (build_dir / filecode).read_text()
|
||||
|
||||
m = re.search(codeex, filecontent)
|
||||
if not m:
|
||||
@ -170,9 +171,9 @@ def check_tags(app, pattern):
|
||||
vercode = m.group(1).strip()
|
||||
|
||||
version = "??"
|
||||
if len(filever) > 0:
|
||||
if filever:
|
||||
if filever != '.':
|
||||
filecontent = open(os.path.join(build_dir, filever)).read()
|
||||
filecontent = (build_dir / filever).read_text()
|
||||
|
||||
m = re.search(verex, filecontent)
|
||||
if not m:
|
||||
@ -189,12 +190,9 @@ def check_tags(app, pattern):
|
||||
hver = version
|
||||
else:
|
||||
for subdir in possible_subdirs(app):
|
||||
if subdir == '.':
|
||||
root_dir = build_dir
|
||||
else:
|
||||
root_dir = os.path.join(build_dir, subdir)
|
||||
root_dir = build_dir / subdir
|
||||
paths = common.manifest_paths(root_dir, last_build.gradle)
|
||||
version, vercode, package = common.parse_androidmanifests(paths, app)
|
||||
version, vercode, _package = common.parse_androidmanifests(paths, app)
|
||||
if vercode:
|
||||
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
|
||||
.format(subdir, version, vercode))
|
||||
@ -227,10 +225,10 @@ def check_repomanifest(app, branch=None):
|
||||
try:
|
||||
|
||||
if app.RepoType == 'srclib':
|
||||
build_dir = os.path.join('build', 'srclib', app.Repo)
|
||||
build_dir = Path('build/srclib') / app.Repo
|
||||
repotype = common.getsrclibvcs(app.Repo)
|
||||
else:
|
||||
build_dir = os.path.join('build', app.id)
|
||||
build_dir = Path('build') / app.id
|
||||
repotype = app.RepoType
|
||||
|
||||
# Set up vcs interface and make sure we have the latest code...
|
||||
@ -248,7 +246,7 @@ def check_repomanifest(app, branch=None):
|
||||
vcs.gotorevision(None)
|
||||
|
||||
last_build = metadata.Build()
|
||||
if len(app.get('Builds', [])) > 0:
|
||||
if app.get('Builds', []):
|
||||
last_build = app.get('Builds', [])[-1]
|
||||
|
||||
try_init_submodules(app, last_build, vcs)
|
||||
@ -257,10 +255,7 @@ def check_repomanifest(app, branch=None):
|
||||
hver = None
|
||||
hcode = "0"
|
||||
for subdir in possible_subdirs(app):
|
||||
if subdir == '.':
|
||||
root_dir = build_dir
|
||||
else:
|
||||
root_dir = os.path.join(build_dir, subdir)
|
||||
root_dir = build_dir / subdir
|
||||
paths = common.manifest_paths(root_dir, last_build.gradle)
|
||||
version, vercode, package = common.parse_androidmanifests(paths, app)
|
||||
if vercode:
|
||||
@ -290,10 +285,10 @@ def check_repotrunk(app):
|
||||
|
||||
try:
|
||||
if app.RepoType == 'srclib':
|
||||
build_dir = os.path.join('build', 'srclib', app.Repo)
|
||||
build_dir = Path('build/srclib') / app.Repo
|
||||
repotype = common.getsrclibvcs(app.Repo)
|
||||
else:
|
||||
build_dir = os.path.join('build', app.id)
|
||||
build_dir = Path('build') / app.id
|
||||
repotype = app.RepoType
|
||||
|
||||
if repotype not in ('git-svn', ):
|
||||
@ -360,10 +355,11 @@ def try_init_submodules(app, last_build, vcs):
|
||||
# Return all directories under startdir that contain any of the manifest
|
||||
# files, and thus are probably an Android project.
|
||||
def dirs_with_manifest(startdir):
|
||||
for root, dirs, files in os.walk(startdir):
|
||||
# TODO: Python3.6: Accepts a path-like object.
|
||||
for root, _dirs, files in os.walk(str(startdir)):
|
||||
if any(m in files for m in [
|
||||
'AndroidManifest.xml', 'pom.xml', 'build.gradle', 'build.gradle.kts']):
|
||||
yield root
|
||||
yield Path(root)
|
||||
|
||||
|
||||
# Tries to find a new subdir starting from the root build_dir. Returns said
|
||||
@ -371,9 +367,9 @@ def dirs_with_manifest(startdir):
|
||||
def possible_subdirs(app):
|
||||
|
||||
if app.RepoType == 'srclib':
|
||||
build_dir = os.path.join('build', 'srclib', app.Repo)
|
||||
build_dir = Path('build/srclib') / app.Repo
|
||||
else:
|
||||
build_dir = os.path.join('build', app.id)
|
||||
build_dir = Path('build') / app.id
|
||||
|
||||
last_build = app.get_last_build()
|
||||
|
||||
@ -381,7 +377,7 @@ def possible_subdirs(app):
|
||||
m_paths = common.manifest_paths(d, last_build.gradle)
|
||||
package = common.parse_androidmanifests(m_paths, app)[2]
|
||||
if package is not None:
|
||||
subdir = os.path.relpath(d, build_dir)
|
||||
subdir = d.relative_to(build_dir)
|
||||
logging.debug("Adding possible subdir %s" % subdir)
|
||||
yield subdir
|
||||
|
||||
@ -401,9 +397,9 @@ def fetch_autoname(app, tag):
|
||||
return None
|
||||
|
||||
if app.RepoType == 'srclib':
|
||||
build_dir = os.path.join('build', 'srclib', app.Repo)
|
||||
build_dir = Path('build/srclib') / app.Repo
|
||||
else:
|
||||
build_dir = os.path.join('build', app.id)
|
||||
build_dir = Path('build') / app.id
|
||||
|
||||
try:
|
||||
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
|
||||
@ -413,13 +409,10 @@ def fetch_autoname(app, tag):
|
||||
|
||||
last_build = app.get_last_build()
|
||||
|
||||
logging.debug("...fetch auto name from " + build_dir)
|
||||
logging.debug("...fetch auto name from " + str(build_dir))
|
||||
new_name = None
|
||||
for subdir in possible_subdirs(app):
|
||||
if subdir == '.':
|
||||
root_dir = build_dir
|
||||
else:
|
||||
root_dir = os.path.join(build_dir, subdir)
|
||||
root_dir = build_dir / subdir
|
||||
new_name = common.fetch_real_name(root_dir, last_build.gradle)
|
||||
if new_name is not None:
|
||||
break
|
||||
|
||||
@ -51,6 +51,7 @@ import yaml
|
||||
import zipfile
|
||||
import tempfile
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
# TODO change to only import defusedxml once its installed everywhere
|
||||
try:
|
||||
@ -119,13 +120,13 @@ orig_path = None
|
||||
default_config = {
|
||||
'sdk_path': "$ANDROID_HOME",
|
||||
'ndk_paths': {},
|
||||
'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
|
||||
'cachedir': str(Path.home() / '.cache/fdroidserver'),
|
||||
'java_paths': None,
|
||||
'scan_binary': False,
|
||||
'ant': "ant",
|
||||
'mvn3': "mvn",
|
||||
'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'),
|
||||
'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'),
|
||||
'gradle_version_dir': str(Path.home() / '.cache/fdroidserver/gradle'),
|
||||
'sync_from_local_copy_dir': False,
|
||||
'allow_disabled_algorithms': False,
|
||||
'per_app_repos': False,
|
||||
@ -884,6 +885,8 @@ def setup_vcs(app):
|
||||
|
||||
|
||||
def getvcs(vcstype, remote, local):
|
||||
# TODO: Remove this in Python3.6
|
||||
local = str(local)
|
||||
if vcstype == 'git':
|
||||
return vcs_git(remote, local)
|
||||
if vcstype == 'git-svn':
|
||||
@ -1082,13 +1085,13 @@ class vcs_git(vcs):
|
||||
|
||||
p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
|
||||
result = p.output.rstrip()
|
||||
if not result.endswith(self.local):
|
||||
if Path(result) != Path(self.local).resolve():
|
||||
raise VCSException('Repository mismatch')
|
||||
|
||||
def gotorevisionx(self, rev):
|
||||
if not os.path.exists(self.local):
|
||||
# Brand new checkout
|
||||
p = self.git(['clone', '--', self.remote, self.local])
|
||||
p = self.git(['clone', '--', self.remote, str(self.local)])
|
||||
if p.returncode != 0:
|
||||
self.clone_failed = True
|
||||
raise VCSException("Git clone failed", p.output)
|
||||
@ -1197,7 +1200,7 @@ class vcs_gitsvn(vcs):
|
||||
"""
|
||||
p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
|
||||
result = p.output.rstrip()
|
||||
if not result.endswith(self.local):
|
||||
if Path(result) != Path(self.local).resolve():
|
||||
raise VCSException('Repository mismatch')
|
||||
|
||||
def git(self, args, envs=dict(), cwd=None, output=True):
|
||||
@ -1258,7 +1261,7 @@ class vcs_gitsvn(vcs):
|
||||
raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ')
|
||||
.format(before=remote, after=location))
|
||||
|
||||
gitsvn_args.extend(['--', remote, self.local])
|
||||
gitsvn_args.extend(['--', remote, str(self.local)])
|
||||
p = self.git(gitsvn_args)
|
||||
if p.returncode != 0:
|
||||
self.clone_failed = True
|
||||
@ -1358,7 +1361,7 @@ class vcs_hg(vcs):
|
||||
|
||||
def gotorevisionx(self, rev):
|
||||
if not os.path.exists(self.local):
|
||||
p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local],
|
||||
p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, str(self.local)],
|
||||
output=False)
|
||||
if p.returncode != 0:
|
||||
self.clone_failed = True
|
||||
@ -1416,7 +1419,7 @@ class vcs_bzr(vcs):
|
||||
|
||||
def gotorevisionx(self, rev):
|
||||
if not os.path.exists(self.local):
|
||||
p = self.bzr(['branch', self.remote, self.local], output=False)
|
||||
p = self.bzr(['branch', self.remote, str(self.local)], output=False)
|
||||
if p.returncode != 0:
|
||||
self.clone_failed = True
|
||||
raise VCSException("Bzr branch failed", p.output)
|
||||
@ -1494,6 +1497,8 @@ def retrieve_string_singleline(app_dir, string, xmlfiles=None):
|
||||
def manifest_paths(app_dir, flavours):
|
||||
'''Return list of existing files that will be used to find the highest vercode'''
|
||||
|
||||
# TODO: Remove this in Python3.6
|
||||
app_dir = str(app_dir)
|
||||
possible_manifests = \
|
||||
[os.path.join(app_dir, 'AndroidManifest.xml'),
|
||||
os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
|
||||
@ -1513,6 +1518,8 @@ def manifest_paths(app_dir, flavours):
|
||||
|
||||
def fetch_real_name(app_dir, flavours):
|
||||
'''Retrieve the package name. Returns the name, or None if not found.'''
|
||||
# TODO: Remove this in Python3.6
|
||||
app_dir = str(app_dir)
|
||||
for path in manifest_paths(app_dir, flavours):
|
||||
if not path.endswith('.xml') or not os.path.isfile(path):
|
||||
continue
|
||||
|
||||
@ -2,19 +2,17 @@
|
||||
|
||||
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
import optparse
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest import mock
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
localmodule = os.path.realpath(
|
||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
|
||||
)
|
||||
print('localmodule: ' + localmodule)
|
||||
localmodule = Path(__file__).resolve().parent.parent
|
||||
print('localmodule: ' + str(localmodule))
|
||||
if localmodule not in sys.path:
|
||||
sys.path.insert(0, localmodule)
|
||||
|
||||
@ -28,11 +26,11 @@ class CheckupdatesTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
self.basedir = os.path.join(localmodule, 'tests')
|
||||
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
|
||||
if not os.path.exists(self.tmpdir):
|
||||
os.makedirs(self.tmpdir)
|
||||
os.chdir(self.basedir)
|
||||
self.basedir = localmodule / 'tests'
|
||||
self.tmpdir = localmodule / '.testfiles'
|
||||
self.tmpdir.mkdir(exist_ok=True)
|
||||
# TODO: Python3.6: Accepts a path-like object.
|
||||
os.chdir(str(self.basedir))
|
||||
|
||||
def test_autoupdatemode_no_suffix(self):
|
||||
fdroidserver.checkupdates.options = mock.Mock()
|
||||
@ -52,7 +50,9 @@ class CheckupdatesTest(unittest.TestCase):
|
||||
build.versionName = app.CurrentVersion
|
||||
app['Builds'].append(build)
|
||||
|
||||
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)):
|
||||
with mock.patch(
|
||||
'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
|
||||
):
|
||||
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
|
||||
with mock.patch('subprocess.call', lambda cmd: 0):
|
||||
fdroidserver.checkupdates.checkupdates_app(app)
|
||||
@ -71,14 +71,16 @@ class CheckupdatesTest(unittest.TestCase):
|
||||
app.CurrentVersion = '1.1.8-fdroid'
|
||||
app.CurrentVersionCode = 10108
|
||||
app.UpdateCheckMode = 'HTTP'
|
||||
app.AutoUpdateMode = 'Version +.%c-fdroid v%v_%c'
|
||||
app.AutoUpdateMode = r'Version +.%c-fdroid v%v_%c'
|
||||
|
||||
build = fdroidserver.metadata.Build()
|
||||
build.versionCode = app.CurrentVersionCode
|
||||
build.versionName = app.CurrentVersion
|
||||
app['Builds'].append(build)
|
||||
|
||||
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)):
|
||||
with mock.patch(
|
||||
'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
|
||||
):
|
||||
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
|
||||
with mock.patch('subprocess.call', lambda cmd: 0):
|
||||
fdroidserver.checkupdates.checkupdates_app(app)
|
||||
@ -98,11 +100,17 @@ class CheckupdatesTest(unittest.TestCase):
|
||||
app.UpdateCheckMode = 'HTTP'
|
||||
app.UpdateCheckData = 'mock'
|
||||
|
||||
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: (None, 'bla')):
|
||||
with mock.patch(
|
||||
'fdroidserver.checkupdates.check_http', lambda app: (None, 'bla')
|
||||
):
|
||||
fdroidserver.checkupdates.checkupdates_app(app)
|
||||
|
||||
with mock.patch('fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)):
|
||||
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()) as wrmock:
|
||||
with mock.patch(
|
||||
'fdroidserver.checkupdates.check_http', lambda app: ('1.1.9', 10109)
|
||||
):
|
||||
with mock.patch(
|
||||
'fdroidserver.metadata.write_metadata', mock.Mock()
|
||||
) as wrmock:
|
||||
with mock.patch('subprocess.call', lambda cmd: 0):
|
||||
fdroidserver.checkupdates.checkupdates_app(app)
|
||||
wrmock.assert_called_with(app.metadatapath, app)
|
||||
@ -125,14 +133,17 @@ class CheckupdatesTest(unittest.TestCase):
|
||||
build.versionName = app.CurrentVersion
|
||||
app['Builds'].append(build)
|
||||
|
||||
with mock.patch('fdroidserver.checkupdates.check_tags', lambda app,
|
||||
pattern: (None, 'bla', None)):
|
||||
with mock.patch(
|
||||
'fdroidserver.checkupdates.check_tags',
|
||||
lambda app, pattern: (None, 'bla', None),
|
||||
):
|
||||
fdroidserver.checkupdates.checkupdates_app(app)
|
||||
|
||||
with mock.patch('fdroidserver.checkupdates.check_tags', lambda app,
|
||||
pattern: ('1.1.9', 10109, 'v1.1.9')):
|
||||
with mock.patch('fdroidserver.metadata.write_metadata',
|
||||
mock.Mock()):
|
||||
with mock.patch(
|
||||
'fdroidserver.checkupdates.check_tags',
|
||||
lambda app, pattern: ('1.1.9', 10109, 'v1.1.9'),
|
||||
):
|
||||
with mock.patch('fdroidserver.metadata.write_metadata', mock.Mock()):
|
||||
with mock.patch('subprocess.call', lambda cmd: 0):
|
||||
fdroidserver.checkupdates.checkupdates_app(app)
|
||||
build = app['Builds'][-1]
|
||||
@ -200,16 +211,17 @@ class CheckupdatesTest(unittest.TestCase):
|
||||
vcs = mock.Mock()
|
||||
vcs.latesttags.return_value = ['1.1.8', '1.1.9']
|
||||
with mock.patch(
|
||||
'builtins.open', mock.mock_open(read_data='v1.1.9\nc10109')
|
||||
'pathlib.Path.read_text', lambda a: 'v1.1.9\nc10109'
|
||||
) as _ignored, mock.patch('fdroidserver.common.getvcs', return_value=vcs):
|
||||
_ignored # silence the linters
|
||||
vername, vercode, tag = fdroidserver.checkupdates.check_tags(app, None)
|
||||
vername, vercode, _tag = fdroidserver.checkupdates.check_tags(app, None)
|
||||
self.assertEqual(vername, '1.1.9')
|
||||
self.assertEqual(vercode, '10109')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
# TODO: Python3.6: Accept path-like object.
|
||||
os.chdir(str(Path(__file__).parent))
|
||||
|
||||
parser = optparse.OptionParser()
|
||||
parser.add_option(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user