mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-10 17:30:11 +01:00
Merge branch 'import' into 'master'
import_subcommand.py: misc fixes and updates See merge request fdroid/fdroidserver!1525
This commit is contained in:
linsui
commit
567e3dbaba
@ -101,9 +101,6 @@ VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[
|
||||
re.IGNORECASE)
|
||||
ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''')
|
||||
|
||||
SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?')
|
||||
GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''')
|
||||
|
||||
MAX_VERSION_CODE = 0x7fffffff # Java's Integer.MAX_VALUE (2147483647)
|
||||
|
||||
XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}'
|
||||
@ -2120,37 +2117,6 @@ def is_strict_application_id(name):
|
||||
and '.' in name
|
||||
|
||||
|
||||
def get_all_gradle_and_manifests(build_dir):
|
||||
paths = []
|
||||
# TODO: Python3.6: Accepts a path-like object.
|
||||
for root, dirs, files in os.walk(str(build_dir)):
|
||||
for f in sorted(files):
|
||||
if f == 'AndroidManifest.xml' \
|
||||
or f.endswith('.gradle') or f.endswith('.gradle.kts'):
|
||||
full = Path(root) / f
|
||||
paths.append(full)
|
||||
return paths
|
||||
|
||||
|
||||
def get_gradle_subdir(build_dir, paths):
|
||||
"""Get the subdir where the gradle build is based."""
|
||||
first_gradle_dir = None
|
||||
for path in paths:
|
||||
if not first_gradle_dir:
|
||||
first_gradle_dir = path.parent.relative_to(build_dir)
|
||||
if path.exists() and SETTINGS_GRADLE_REGEX.match(str(path.name)):
|
||||
for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')):
|
||||
for f in (path.parent / m.group(1)).glob('build.gradle*'):
|
||||
with f.open(encoding='utf-8') as fp:
|
||||
for line in fp.readlines():
|
||||
if ANDROID_PLUGIN_REGEX.match(line):
|
||||
return f.parent.relative_to(build_dir)
|
||||
if first_gradle_dir and first_gradle_dir != Path('.'):
|
||||
return first_gradle_dir
|
||||
|
||||
return
|
||||
|
||||
|
||||
def parse_srclib_spec(spec):
|
||||
|
||||
if type(spec) != str:
|
||||
|
||||
@ -18,34 +18,64 @@
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import urllib
|
||||
|
||||
import git
|
||||
import json
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import yaml
|
||||
import urllib
|
||||
from argparse import ArgumentParser
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import git
|
||||
import yaml
|
||||
|
||||
try:
|
||||
from yaml import CSafeLoader as SafeLoader
|
||||
except ImportError:
|
||||
from yaml import SafeLoader
|
||||
|
||||
from . import _
|
||||
from . import common
|
||||
from . import metadata
|
||||
from . import _, common, metadata
|
||||
from .exception import FDroidException
|
||||
|
||||
|
||||
config = None
|
||||
|
||||
SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?')
|
||||
GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:?([^'"]+)['"]''')
|
||||
APPLICATION_ID_REGEX = re.compile(r'''\s*applicationId\s=?\s?['"].*['"]''')
|
||||
|
||||
|
||||
def get_all_gradle_and_manifests(build_dir):
|
||||
paths = []
|
||||
for root, dirs, files in os.walk(build_dir):
|
||||
for f in sorted(files):
|
||||
if f == 'AndroidManifest.xml' or f.endswith(('.gradle', '.gradle.kts')):
|
||||
full = Path(root) / f
|
||||
paths.append(full)
|
||||
return paths
|
||||
|
||||
|
||||
def get_gradle_subdir(build_dir, paths):
|
||||
"""Get the subdir where the gradle build is based."""
|
||||
first_gradle_dir = None
|
||||
for path in paths:
|
||||
if not first_gradle_dir:
|
||||
first_gradle_dir = path.parent.relative_to(build_dir)
|
||||
if path.exists() and SETTINGS_GRADLE_REGEX.match(path.name):
|
||||
for m in GRADLE_SUBPROJECT_REGEX.finditer(path.read_text(encoding='utf-8')):
|
||||
for f in (path.parent / m.group(1)).glob('build.gradle*'):
|
||||
with f.open(encoding='utf-8') as fp:
|
||||
for line in fp:
|
||||
if common.ANDROID_PLUGIN_REGEX.match(
|
||||
line
|
||||
) or APPLICATION_ID_REGEX.match(line):
|
||||
return f.parent.relative_to(build_dir)
|
||||
if first_gradle_dir and first_gradle_dir != Path('.'):
|
||||
return first_gradle_dir
|
||||
|
||||
|
||||
def handle_retree_error_on_windows(function, path, excinfo):
|
||||
"""Python can't remove a readonly file on Windows so chmod first."""
|
||||
@ -100,6 +130,7 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]:
|
||||
The found repository type or None if an error occured.
|
||||
address_or_reason
|
||||
The address to the found repository or the reason if an error occured.
|
||||
|
||||
"""
|
||||
if not url.startswith('http'):
|
||||
return (None, _('{url} does not start with "http"!'.format(url=url)))
|
||||
@ -122,7 +153,7 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]:
|
||||
index = page.find('hg clone')
|
||||
if index != -1:
|
||||
repotype = 'hg'
|
||||
repo = page[index + 9:]
|
||||
repo = page[index + 9 :]
|
||||
index = repo.find('<')
|
||||
if index == -1:
|
||||
return (None, _("Error while getting repo address"))
|
||||
@ -134,7 +165,7 @@ def getrepofrompage(url: str) -> tuple[Optional[str], str]:
|
||||
index = page.find('git clone')
|
||||
if index != -1:
|
||||
repotype = 'git'
|
||||
repo = page[index + 10:]
|
||||
repo = page[index + 10 :]
|
||||
index = repo.find('<')
|
||||
if index == -1:
|
||||
return (None, _("Error while getting repo address"))
|
||||
@ -168,6 +199,7 @@ def get_app_from_url(url: str) -> metadata.App:
|
||||
If the VCS type could not be determined.
|
||||
:exc:`ValueError`
|
||||
If the URL is invalid.
|
||||
|
||||
"""
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
invalid_url = False
|
||||
@ -243,18 +275,29 @@ def main():
|
||||
# Parse command line...
|
||||
parser = ArgumentParser()
|
||||
common.setup_global_opts(parser)
|
||||
parser.add_argument("-u", "--url", default=None,
|
||||
help=_("Project URL to import from."))
|
||||
parser.add_argument("-s", "--subdir", default=None,
|
||||
help=_("Path to main Android project subdirectory, if not in root."))
|
||||
parser.add_argument("-c", "--categories", default=None,
|
||||
help=_("Comma separated list of categories."))
|
||||
parser.add_argument("-l", "--license", default=None,
|
||||
help=_("Overall license of the project."))
|
||||
parser.add_argument("--omit-disable", action="store_true", default=False,
|
||||
help=_("Do not add 'disable:' to the generated build entries"))
|
||||
parser.add_argument("--rev", default=None,
|
||||
help=_("Allows a different revision (or git branch) to be specified for the initial import"))
|
||||
parser.add_argument("-u", "--url", help=_("Project URL to import from."))
|
||||
parser.add_argument(
|
||||
"-s",
|
||||
"--subdir",
|
||||
help=_("Path to main Android project subdirectory, if not in root."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c",
|
||||
"--categories",
|
||||
help=_("Comma separated list of categories."),
|
||||
)
|
||||
parser.add_argument("-l", "--license", help=_("Overall license of the project."))
|
||||
parser.add_argument(
|
||||
"--omit-disable",
|
||||
action="store_true",
|
||||
help=_("Do not add 'disable:' to the generated build entries"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rev",
|
||||
help=_(
|
||||
"Allows a different revision (or git branch) to be specified for the initial import"
|
||||
),
|
||||
)
|
||||
metadata.add_metadata_arguments(parser)
|
||||
options = common.parse_args(parser)
|
||||
metadata.warnings_action = options.W
|
||||
@ -268,24 +311,20 @@ def main():
|
||||
|
||||
local_metadata_files = common.get_local_metadata_files()
|
||||
if local_metadata_files:
|
||||
raise FDroidException(_("This repo already has local metadata: %s") % local_metadata_files[0])
|
||||
raise FDroidException(
|
||||
_("This repo already has local metadata: %s") % local_metadata_files[0]
|
||||
)
|
||||
|
||||
build = metadata.Build()
|
||||
app = metadata.App()
|
||||
if options.url is None and Path('.git').is_dir():
|
||||
app = metadata.App()
|
||||
app.AutoName = Path.cwd().name
|
||||
app.RepoType = 'git'
|
||||
|
||||
if Path('build.gradle').exists() or Path('build.gradle.kts').exists():
|
||||
build.gradle = ['yes']
|
||||
|
||||
git_repo = git.Repo(Path.cwd())
|
||||
tmp_importer_dir = Path.cwd()
|
||||
git_repo = git.Repo(tmp_importer_dir)
|
||||
for remote in git.Remote.iter_items(git_repo):
|
||||
if remote.name == 'origin':
|
||||
url = git_repo.remotes.origin.url
|
||||
if url.startswith('https://git'): # github, gitlab
|
||||
app.SourceCode = url.rstrip('.git')
|
||||
app.Repo = url
|
||||
app = get_app_from_url(url)
|
||||
break
|
||||
write_local_file = True
|
||||
elif options.url:
|
||||
@ -294,17 +333,20 @@ def main():
|
||||
git_repo = git.Repo(tmp_importer_dir)
|
||||
|
||||
if not options.omit_disable:
|
||||
build.disable = 'Generated by `fdroid import` - check version fields and commitid'
|
||||
build.disable = (
|
||||
'Generated by `fdroid import` - check version fields and commitid'
|
||||
)
|
||||
write_local_file = False
|
||||
else:
|
||||
raise FDroidException("Specify project url.")
|
||||
|
||||
app.AutoUpdateMode = 'Version'
|
||||
app.UpdateCheckMode = 'Tags'
|
||||
build.commit = common.get_head_commit_id(git_repo)
|
||||
|
||||
# Extract some information...
|
||||
paths = common.get_all_gradle_and_manifests(tmp_importer_dir)
|
||||
subdir = common.get_gradle_subdir(tmp_importer_dir, paths)
|
||||
paths = get_all_gradle_and_manifests(tmp_importer_dir)
|
||||
gradle_subdir = get_gradle_subdir(tmp_importer_dir, paths)
|
||||
if paths:
|
||||
versionName, versionCode, appid = common.parse_androidmanifests(paths, app)
|
||||
if not appid:
|
||||
@ -322,16 +364,15 @@ def main():
|
||||
|
||||
# Create a build line...
|
||||
build.versionName = versionName or 'Unknown'
|
||||
app.CurrentVersion = build.versionName
|
||||
build.versionCode = versionCode or 0
|
||||
app.CurrentVersionCode = build.versionCode
|
||||
if options.subdir:
|
||||
build.subdir = options.subdir
|
||||
build.gradle = ['yes']
|
||||
elif subdir:
|
||||
build.subdir = subdir.as_posix()
|
||||
build.gradle = ['yes']
|
||||
else:
|
||||
# subdir might be None
|
||||
subdir = Path()
|
||||
elif gradle_subdir:
|
||||
build.subdir = gradle_subdir.as_posix()
|
||||
# subdir might be None
|
||||
subdir = Path(tmp_importer_dir / build.subdir) if build.subdir else tmp_importer_dir
|
||||
|
||||
if options.license:
|
||||
app.License = options.license
|
||||
@ -339,23 +380,23 @@ def main():
|
||||
app.Categories = options.categories.split(',')
|
||||
if (subdir / 'jni').exists():
|
||||
build.buildjni = ['yes']
|
||||
if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle').exists():
|
||||
if (subdir / 'build.gradle').exists() or (subdir / 'build.gradle.kts').exists():
|
||||
build.gradle = ['yes']
|
||||
|
||||
app.AutoName = common.fetch_real_name(subdir, build.gradle)
|
||||
|
||||
package_json = tmp_importer_dir / 'package.json' # react-native
|
||||
pubspec_yaml = tmp_importer_dir / 'pubspec.yaml' # flutter
|
||||
if package_json.exists():
|
||||
build.sudo = [
|
||||
'sysctl fs.inotify.max_user_watches=524288 || true',
|
||||
'curl -Lo node.tar.gz https://nodejs.org/download/release/v19.3.0/node-v19.3.0-linux-x64.tar.gz',
|
||||
'echo "b525028ae5bb71b5b32cb7fce903ccce261dbfef4c7dd0f3e0ffc27cd6fc0b3f node.tar.gz" | sha256sum -c -',
|
||||
'tar xzf node.tar.gz --strip-components=1 -C /usr/local/',
|
||||
'npm -g install yarn',
|
||||
'apt-get update',
|
||||
'apt-get install -y npm',
|
||||
]
|
||||
build.init = ['npm install --build-from-source']
|
||||
with package_json.open() as fp:
|
||||
data = json.load(fp)
|
||||
app.AutoName = data.get('name', app.AutoName)
|
||||
app.AutoName = app.AutoName or data.get('name')
|
||||
app.License = data.get('license', app.License)
|
||||
app.Description = data.get('description', app.Description)
|
||||
app.WebSite = data.get('homepage', app.WebSite)
|
||||
@ -365,11 +406,11 @@ def main():
|
||||
if app_json.exists():
|
||||
with app_json.open() as fp:
|
||||
data = json.load(fp)
|
||||
app.AutoName = data.get('name', app.AutoName)
|
||||
app.AutoName = app.AutoName or data.get('name')
|
||||
if pubspec_yaml.exists():
|
||||
with pubspec_yaml.open() as fp:
|
||||
data = yaml.load(fp, Loader=SafeLoader)
|
||||
app.AutoName = data.get('name', app.AutoName)
|
||||
app.AutoName = app.AutoName or data.get('name')
|
||||
app.License = data.get('license', app.License)
|
||||
app.Description = data.get('description', app.Description)
|
||||
app.UpdateCheckData = 'pubspec.yaml|version:\\s.+\\+(\\d+)|.|version:\\s(.+)\\+'
|
||||
@ -405,8 +446,11 @@ def main():
|
||||
Path('build').mkdir(exist_ok=True)
|
||||
build_dir = Path('build') / appid
|
||||
if build_dir.exists():
|
||||
logging.warning(_('{path} already exists, ignoring import results!')
|
||||
.format(path=build_dir))
|
||||
logging.warning(
|
||||
_('{path} already exists, ignoring import results!').format(
|
||||
path=build_dir
|
||||
)
|
||||
)
|
||||
sys.exit(1)
|
||||
elif tmp_importer_dir:
|
||||
# For Windows: Close the repo or a git.exe instance holds handles to repo
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
|
||||
# We ignore the following PEP8 warnings
|
||||
# * E123: closing bracket does not match indentation of opening bracket's line
|
||||
# - Broken if multiple indentation levels start on a single line
|
||||
@ -38,7 +37,6 @@ force-exclude = '''(
|
||||
| fdroidserver/build\.py
|
||||
| fdroidserver/checkupdates\.py
|
||||
| fdroidserver/common\.py
|
||||
| fdroidserver/import_subcommand\.py
|
||||
| fdroidserver/index\.py
|
||||
| fdroidserver/metadata\.py
|
||||
| fdroidserver/nightly\.py
|
||||
|
||||
@ -1522,45 +1522,6 @@ class CommonTest(unittest.TestCase):
|
||||
self.assertEqual(('2021-06-30', 34, 'de.varengold.activeTAN'),
|
||||
fdroidserver.common.parse_androidmanifests(paths, app))
|
||||
|
||||
def test_get_all_gradle_and_manifests(self):
|
||||
"""Test whether the function works with relative and absolute paths"""
|
||||
a = fdroidserver.common.get_all_gradle_and_manifests(Path('source-files/cn.wildfirechat.chat'))
|
||||
paths = [
|
||||
'avenginekit/build.gradle',
|
||||
'build.gradle',
|
||||
'chat/build.gradle',
|
||||
'client/build.gradle',
|
||||
'client/src/main/AndroidManifest.xml',
|
||||
'emojilibrary/build.gradle',
|
||||
'gradle/build_libraries.gradle',
|
||||
'imagepicker/build.gradle',
|
||||
'mars-core-release/build.gradle',
|
||||
'push/build.gradle',
|
||||
'settings.gradle',
|
||||
]
|
||||
paths = [Path('source-files/cn.wildfirechat.chat') / path for path in paths]
|
||||
self.assertEqual(sorted(paths), sorted(a))
|
||||
|
||||
abspath = Path(self.basedir) / 'source-files/realm'
|
||||
p = fdroidserver.common.get_all_gradle_and_manifests(abspath)
|
||||
self.assertEqual(1, len(p))
|
||||
self.assertTrue(p[0].is_relative_to(abspath))
|
||||
|
||||
def test_get_gradle_subdir(self):
|
||||
subdirs = {
|
||||
'cn.wildfirechat.chat': 'chat',
|
||||
'com.anpmech.launcher': 'app',
|
||||
'org.tasks': 'app',
|
||||
'ut.ewh.audiometrytest': 'app',
|
||||
'org.noise_planet.noisecapture': 'app',
|
||||
}
|
||||
for k, v in subdirs.items():
|
||||
build_dir = Path('source-files') / k
|
||||
paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
|
||||
logging.info(paths)
|
||||
subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths)
|
||||
self.assertEqual(v, str(subdir))
|
||||
|
||||
def test_parse_srclib_spec_good(self):
|
||||
self.assertEqual(fdroidserver.common.parse_srclib_spec('osmand-external-skia@android/oreo'),
|
||||
('osmand-external-skia', 'android/oreo', None, None))
|
||||
|
||||
@ -2,29 +2,30 @@
|
||||
|
||||
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
|
||||
|
||||
import git
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import yaml
|
||||
from unittest import mock
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import git
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
localmodule = Path(__file__).resolve().parent.parent
|
||||
print('localmodule: ' + str(localmodule))
|
||||
if localmodule not in sys.path:
|
||||
sys.path.insert(0, str(localmodule))
|
||||
|
||||
from testcommon import TmpCwd, mkdtemp, parse_args_for_test
|
||||
|
||||
import fdroidserver.common
|
||||
import fdroidserver.import_subcommand
|
||||
import fdroidserver.metadata
|
||||
from fdroidserver.exception import FDroidException
|
||||
from testcommon import TmpCwd, mkdtemp, parse_args_for_test
|
||||
|
||||
|
||||
class ImportTest(unittest.TestCase):
|
||||
@ -41,6 +42,49 @@ class ImportTest(unittest.TestCase):
|
||||
os.chdir(self.basedir)
|
||||
self._td.cleanup()
|
||||
|
||||
def test_get_all_gradle_and_manifests(self):
|
||||
"""Test whether the function works with relative and absolute paths"""
|
||||
a = fdroidserver.import_subcommand.get_all_gradle_and_manifests(
|
||||
Path('source-files/cn.wildfirechat.chat')
|
||||
)
|
||||
paths = [
|
||||
'avenginekit/build.gradle',
|
||||
'build.gradle',
|
||||
'chat/build.gradle',
|
||||
'client/build.gradle',
|
||||
'client/src/main/AndroidManifest.xml',
|
||||
'emojilibrary/build.gradle',
|
||||
'gradle/build_libraries.gradle',
|
||||
'imagepicker/build.gradle',
|
||||
'mars-core-release/build.gradle',
|
||||
'push/build.gradle',
|
||||
'settings.gradle',
|
||||
]
|
||||
paths = [Path('source-files/cn.wildfirechat.chat') / path for path in paths]
|
||||
self.assertEqual(sorted(paths), sorted(a))
|
||||
|
||||
abspath = Path(self.basedir) / 'source-files/realm'
|
||||
p = fdroidserver.import_subcommand.get_all_gradle_and_manifests(abspath)
|
||||
self.assertEqual(1, len(p))
|
||||
self.assertTrue(p[0].is_relative_to(abspath))
|
||||
|
||||
def test_get_gradle_subdir(self):
|
||||
subdirs = {
|
||||
'cn.wildfirechat.chat': 'chat',
|
||||
'com.anpmech.launcher': 'app',
|
||||
'org.tasks': 'app',
|
||||
'ut.ewh.audiometrytest': 'app',
|
||||
'org.noise_planet.noisecapture': 'app',
|
||||
}
|
||||
for k, v in subdirs.items():
|
||||
build_dir = Path('source-files') / k
|
||||
paths = fdroidserver.import_subcommand.get_all_gradle_and_manifests(
|
||||
build_dir
|
||||
)
|
||||
logging.info(paths)
|
||||
subdir = fdroidserver.import_subcommand.get_gradle_subdir(build_dir, paths)
|
||||
self.assertEqual(v, str(subdir))
|
||||
|
||||
def test_import_gitlab(self):
|
||||
with tempfile.TemporaryDirectory() as testdir, TmpCwd(testdir):
|
||||
# FDroidPopen needs some config to work
|
||||
@ -106,7 +150,9 @@ class ImportTest(unittest.TestCase):
|
||||
self.assertEqual(url, app.Repo)
|
||||
self.assertEqual(url, app.SourceCode)
|
||||
logging.info(build_dir)
|
||||
paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
|
||||
paths = fdroidserver.import_subcommand.get_all_gradle_and_manifests(
|
||||
build_dir
|
||||
)
|
||||
self.assertNotEqual(paths, [])
|
||||
(
|
||||
versionName,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user