diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index 806c6339..8f53b097 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -48,287 +48,254 @@ def check_http(app): ignoreversions = app.UpdateCheckIgnore ignoresearch = re.compile(ignoreversions).search if ignoreversions else None - try: + if not app.UpdateCheckData: + raise FDroidException('Missing Update Check Data') - if not app.UpdateCheckData: - raise FDroidException('Missing Update Check Data') - - urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|') - parsed = urllib.parse.urlparse(urlcode) + urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|') + parsed = urllib.parse.urlparse(urlcode) + if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https': + raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode)) + if urlver != '.': + parsed = urllib.parse.urlparse(urlver) if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https': raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode)) - if urlver != '.': - parsed = urllib.parse.urlparse(urlver) - if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https': - raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode)) - vercode = None - if urlcode: - logging.debug("...requesting {0}".format(urlcode)) - req = urllib.request.Request(urlcode, None, headers=net.HEADERS) + vercode = None + if urlcode: + logging.debug("...requesting {0}".format(urlcode)) + req = urllib.request.Request(urlcode, None, headers=net.HEADERS) + resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above + page = resp.read().decode('utf-8') + + m = re.search(codeex, page) + if not m: + raise FDroidException("No RE match for version code") + vercode = m.group(1).strip() + + version = "??" + if urlver: + if urlver != '.': + logging.debug("...requesting {0}".format(urlver)) + req = urllib.request.Request(urlver, None) resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above page = resp.read().decode('utf-8') - m = re.search(codeex, page) - if not m: - raise FDroidException("No RE match for version code") - vercode = m.group(1).strip() + m = re.search(verex, page) + if not m: + raise FDroidException("No RE match for version") + version = m.group(1) - version = "??" - if urlver: - if urlver != '.': - logging.debug("...requesting {0}".format(urlver)) - req = urllib.request.Request(urlver, None) - resp = urllib.request.urlopen(req, None, 20) # nosec B310 scheme is filtered above - page = resp.read().decode('utf-8') - - m = re.search(verex, page) - if not m: - raise FDroidException("No RE match for version") - version = m.group(1) - - if ignoresearch and version: - if not ignoresearch(version): - return (version, vercode) - else: - return (None, ("Version {version} is ignored").format(version=version)) - else: + if ignoresearch and version: + if not ignoresearch(version): return (version, vercode) - except FDroidException: - msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app.id, traceback.format_exc()) - return (None, msg) - - -# Check for a new version by looking at the tags in the source repo. -# Whether this can be used reliably or not depends on -# the development procedures used by the project's developers. Use it with -# caution, because it's inappropriate for many projects. -# Returns (None, "a message") if this didn't work, or (version, vercode, tag) for -# the details of the current version. -def check_tags(app, pattern): - - try: - - if app.RepoType == 'srclib': - build_dir = Path('build/srclib') / app.Repo - repotype = common.getsrclibvcs(app.Repo) else: - build_dir = Path('build') / app.id - repotype = app.RepoType + return (None, ("Version {version} is ignored").format(version=version)) + else: + return (version, vercode) - if repotype not in ('git', 'git-svn', 'hg', 'bzr'): - return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None) - if repotype == 'git-svn' and ';' not in app.Repo: - return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None) +def check_tags(app, pattern): + """Check for a new version by looking at the tags in the source repo. - # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) + Whether this can be used reliably or not depends on + the development procedures used by the project's developers. Use it with + caution, because it's inappropriate for many projects. + """ + if app.RepoType == 'srclib': + build_dir = Path('build/srclib') / app.Repo + repotype = common.getsrclibvcs(app.Repo) + else: + build_dir = Path('build') / app.id + repotype = app.RepoType + if repotype not in ('git', 'git-svn', 'hg', 'bzr'): + return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None) + + if repotype == 'git-svn' and ';' not in app.Repo: + return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None) + + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app.RepoType, app.Repo, build_dir) + + vcs.gotorevision(None) + + last_build = app.get_last_build() + + try_init_submodules(app, last_build, vcs) + + htag = None + hver = None + hcode = "0" + + tags = [] + if repotype == 'git': + tags = vcs.latesttags() + else: + tags = vcs.gettags() + if not tags: + return (None, "No tags found", None) + + logging.debug("All tags: " + ','.join(tags)) + if pattern: + pat = re.compile(pattern) + tags = [tag for tag in tags if pat.match(tag)] + if not tags: + return (None, "No matching tags found", None) + logging.debug("Matching tags: " + ','.join(tags)) + + if len(tags) > 5 and repotype == 'git': + tags = tags[:5] + logging.debug("Latest tags: " + ','.join(tags)) + + for tag in tags: + logging.debug("Check tag: '{0}'".format(tag)) + vcs.gotorevision(tag) + + if app.UpdateCheckData: + filecode, codeex, filever, verex = app.UpdateCheckData.split('|') + + if filecode: + filecode = build_dir / filecode + if not filecode.is_file(): + logging.debug("UpdateCheckData file {0} not found in tag {1}".format(filecode, tag)) + continue + filecontent = filecode.read_text() + else: + filecontent = tag + + vercode = tag + if codeex: + m = re.search(codeex, filecontent) + if not m: + continue + + vercode = m.group(1).strip() + + if filever: + if filever != '.': + filever = build_dir / filever + if filever.is_file(): + filecontent = filever.read_text() + else: + logging.debug("UpdateCheckData file {0} not found in tag {1}".format(filever, tag)) + else: + filecontent = tag + + version = tag + if verex: + m = re.search(verex, filecontent) + if m: + version = m.group(1) + + logging.debug("UpdateCheckData found version {0} ({1})" + .format(version, vercode)) + i_vercode = common.version_code_string_to_int(vercode) + if i_vercode > common.version_code_string_to_int(hcode): + htag = tag + hcode = str(i_vercode) + hver = version + else: + for subdir in possible_subdirs(app): + root_dir = build_dir / subdir + paths = common.manifest_paths(root_dir, last_build.gradle) + version, vercode, _package = common.parse_androidmanifests(paths, app) + if version == 'Unknown' or version == 'Ignore': + version = tag + if vercode: + logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" + .format(subdir, version, vercode)) + i_vercode = common.version_code_string_to_int(vercode) + if i_vercode > common.version_code_string_to_int(hcode): + htag = tag + hcode = str(i_vercode) + hver = version + + if hver: + try: + commit = vcs.getref(htag) + if commit: + return (hver, hcode, commit) + except VCSException: + pass + return (hver, hcode, htag) + return (None, "Couldn't find any version information", None) + + +def check_repomanifest(app, branch=None): + """Check for a new version by looking at the AndroidManifest.xml at the HEAD of the source repo. + + Whether this can be used reliably or not depends on + the development procedures used by the project's developers. Use it with + caution, because it's inappropriate for many projects. + """ + if app.RepoType == 'srclib': + build_dir = Path('build/srclib') / app.Repo + repotype = common.getsrclibvcs(app.Repo) + else: + build_dir = Path('build') / app.id + repotype = app.RepoType + + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app.RepoType, app.Repo, build_dir) + + if repotype == 'git': + if branch: + branch = 'origin/' + branch + vcs.gotorevision(branch) + elif repotype == 'git-svn': + vcs.gotorevision(branch) + elif repotype == 'hg': + vcs.gotorevision(branch) + elif repotype == 'bzr': vcs.gotorevision(None) - last_build = app.get_last_build() + last_build = metadata.Build() + if app.get('Builds', []): + last_build = app.get('Builds', [])[-1] - try_init_submodules(app, last_build, vcs) + try_init_submodules(app, last_build, vcs) - htag = None - hver = None - hcode = "0" + hpak = None + hver = None + hcode = "0" + for subdir in possible_subdirs(app): + root_dir = build_dir / subdir + paths = common.manifest_paths(root_dir, last_build.gradle) + version, vercode, package = common.parse_androidmanifests(paths, app) + if vercode: + logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" + .format(subdir, version, vercode)) + i_vercode = common.version_code_string_to_int(vercode) + if i_vercode > common.version_code_string_to_int(hcode): + hpak = package + hcode = str(i_vercode) + hver = version - tags = [] - if repotype == 'git': - tags = vcs.latesttags() - else: - tags = vcs.gettags() - if not tags: - return (None, "No tags found", None) - - logging.debug("All tags: " + ','.join(tags)) - if pattern: - pat = re.compile(pattern) - tags = [tag for tag in tags if pat.match(tag)] - if not tags: - return (None, "No matching tags found", None) - logging.debug("Matching tags: " + ','.join(tags)) - - if len(tags) > 5 and repotype == 'git': - tags = tags[:5] - logging.debug("Latest tags: " + ','.join(tags)) - - for tag in tags: - logging.debug("Check tag: '{0}'".format(tag)) - vcs.gotorevision(tag) - - if app.UpdateCheckData: - filecode, codeex, filever, verex = app.UpdateCheckData.split('|') - - if filecode: - filecode = build_dir / filecode - if not filecode.is_file(): - logging.debug("UpdateCheckData file {0} not found in tag {1}".format(filecode, tag)) - continue - filecontent = filecode.read_text() - else: - filecontent = tag - - vercode = tag - if codeex: - m = re.search(codeex, filecontent) - if not m: - continue - - vercode = m.group(1).strip() - - if filever: - if filever != '.': - filever = build_dir / filever - if filever.is_file(): - filecontent = filever.read_text() - else: - logging.debug("UpdateCheckData file {0} not found in tag {1}".format(filever, tag)) - else: - filecontent = tag - - version = tag - if verex: - m = re.search(verex, filecontent) - if m: - version = m.group(1) - - logging.debug("UpdateCheckData found version {0} ({1})" - .format(version, vercode)) - i_vercode = common.version_code_string_to_int(vercode) - if i_vercode > common.version_code_string_to_int(hcode): - htag = tag - hcode = str(i_vercode) - hver = version - else: - for subdir in possible_subdirs(app): - root_dir = build_dir / subdir - paths = common.manifest_paths(root_dir, last_build.gradle) - version, vercode, _package = common.parse_androidmanifests(paths, app) - if version == 'Unknown' or version == 'Ignore': - version = tag - if vercode: - logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" - .format(subdir, version, vercode)) - i_vercode = common.version_code_string_to_int(vercode) - if i_vercode > common.version_code_string_to_int(hcode): - htag = tag - hcode = str(i_vercode) - hver = version - - if hver: - try: - commit = vcs.getref(htag) - if commit: - return (hver, hcode, commit) - except VCSException: - pass - return (hver, hcode, htag) - return (None, "Couldn't find any version information", None) - - except VCSException as vcse: - msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse) - return (None, msg, None) - except Exception: - msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc()) - return (None, msg, None) - - -# Check for a new version by looking at the AndroidManifest.xml at the HEAD -# of the source repo. Whether this can be used reliably or not depends on -# the development procedures used by the project's developers. Use it with -# caution, because it's inappropriate for many projects. -# Returns (None, "a message") if this didn't work, or (version, vercode) for -# the details of the current version. -def check_repomanifest(app, branch=None): - - try: - - if app.RepoType == 'srclib': - build_dir = Path('build/srclib') / app.Repo - repotype = common.getsrclibvcs(app.Repo) - else: - build_dir = Path('build') / app.id - repotype = app.RepoType - - # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) - - if repotype == 'git': - if branch: - branch = 'origin/' + branch - vcs.gotorevision(branch) - elif repotype == 'git-svn': - vcs.gotorevision(branch) - elif repotype == 'hg': - vcs.gotorevision(branch) - elif repotype == 'bzr': - vcs.gotorevision(None) - - last_build = metadata.Build() - if app.get('Builds', []): - last_build = app.get('Builds', [])[-1] - - try_init_submodules(app, last_build, vcs) - - hpak = None - hver = None - hcode = "0" - for subdir in possible_subdirs(app): - root_dir = build_dir / subdir - paths = common.manifest_paths(root_dir, last_build.gradle) - version, vercode, package = common.parse_androidmanifests(paths, app) - if vercode: - logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})" - .format(subdir, version, vercode)) - i_vercode = common.version_code_string_to_int(vercode) - if i_vercode > common.version_code_string_to_int(hcode): - hpak = package - hcode = str(i_vercode) - hver = version - - if not hpak: - return (None, "Couldn't find package ID") - if hver: - return (hver, hcode) - return (None, "Couldn't find any version information") - - except VCSException as vcse: - msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse) - return (None, msg) - except Exception: - msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc()) - return (None, msg) + if not hpak: + return (None, "Couldn't find package ID") + if hver: + return (hver, hcode) + return (None, "Couldn't find any version information") def check_repotrunk(app): + if app.RepoType == 'srclib': + build_dir = Path('build/srclib') / app.Repo + repotype = common.getsrclibvcs(app.Repo) + else: + build_dir = Path('build') / app.id + repotype = app.RepoType - try: - if app.RepoType == 'srclib': - build_dir = Path('build/srclib') / app.Repo - repotype = common.getsrclibvcs(app.Repo) - else: - build_dir = Path('build') / app.id - repotype = app.RepoType + if repotype not in ('git-svn', ): + return (None, 'RepoTrunk update mode only makes sense in git-svn repositories') - if repotype not in ('git-svn', ): - return (None, 'RepoTrunk update mode only makes sense in git-svn repositories') + # Set up vcs interface and make sure we have the latest code... + vcs = common.getvcs(app.RepoType, app.Repo, build_dir) - # Set up vcs interface and make sure we have the latest code... - vcs = common.getvcs(app.RepoType, app.Repo, build_dir) + vcs.gotorevision(None) - vcs.gotorevision(None) - - ref = vcs.getref() - return (ref, ref) - except VCSException as vcse: - msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse) - return (None, msg) - except Exception: - msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc()) - return (None, msg) + ref = vcs.getref() + return (ref, ref) # Check for a new version by looking at the Google Play Store. diff --git a/tests/checkupdates.TestCase b/tests/checkupdates.TestCase index 06504a07..a8e9ed78 100755 --- a/tests/checkupdates.TestCase +++ b/tests/checkupdates.TestCase @@ -186,9 +186,8 @@ class CheckupdatesTest(unittest.TestCase): faked = scheme + '://fake.url/for/testing/scheme' app.UpdateCheckData = faked + '|ignored|' + faked + '|ignored' app.metadatapath = 'metadata/' + app.id + '.yml' - vername, vercode = fdroidserver.checkupdates.check_http(app) - self.assertIsNone(vername) - self.assertTrue(FDroidException.__name__ in vercode) + with self.assertRaises(FDroidException): + fdroidserver.checkupdates.check_http(app) def test_check_http_ignore(self): fdroidserver.checkupdates.options = mock.Mock()