kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-06-03 06:10:10 +02:00
Code Activity

Merge branch 'build_no_vcs' into 'master'

Browse Source 
Don't check VCS for completed builds

Closes #920

See merge request fdroid/fdroidserver!1036
This commit is contained in:
Jochen Sprickerhof 2024-02-15 01:11:57 +00:00
commit e337e29ccf
4 changed files with 99 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
fdroidserver/build.py
View File

@ -53,7 +53,7 @@ ssh_channel = None
# Note that 'force' here also implies test mode. # Note that 'force' here also implies test mode.
def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): def build_server(app, build, output_dir, log_dir, force, refresh):
"""Do a build on the builder vm. """Do a build on the builder vm.
Parameters Parameters
@ -61,10 +61,6 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
app app
app metadata dict app metadata dict
build build
vcs
version control system controller object
build_dir
local source-code checkout of app
output_dir output_dir
target folder for the build result target folder for the build result
force force
@ -194,10 +190,6 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
srclibpaths.append( srclibpaths.append(
common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False)) common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
for name, number, lib in srclibpaths: for name, number, lib in srclibpaths:
logging.info("Sending srclib '%s'" % lib) logging.info("Sending srclib '%s'" % lib)
ftp.chdir(posixpath.join(homedir, 'build', 'srclib')) ftp.chdir(posixpath.join(homedir, 'build', 'srclib'))
@ -214,9 +206,19 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
else: else:
raise BuildException(_('cannot find required srclibs: "{path}"') raise BuildException(_('cannot find required srclibs: "{path}"')
.format(path=srclibsfile)) .format(path=srclibsfile))
vcs, build_dir = common.setup_vcs(app)
# When using server mode, still keep a local cache of the repo, by
# grabbing the source now.
vcs.gotorevision(build.commit, refresh)
# Initialise submodules if required
if build.submodules:
vcs.initsubmodules()
# Copy the main app source code # Copy the main app source code
# (no need if it's a srclib) # (no need if it's a srclib)
if (not basesrclib) and os.path.exists(build_dir): if os.path.exists(build_dir):
ftp.chdir(posixpath.join(homedir, 'build')) ftp.chdir(posixpath.join(homedir, 'build'))
fv = '.fdroidvcs-' + app.id fv = '.fdroidvcs-' + app.id
ftp.put(os.path.join('build', fv), fv) ftp.put(os.path.join('build', fv), fv)
@ -359,7 +361,7 @@ def get_metadata_from_apk(app, build, apkfile):
return versionCode, versionName return versionCode, versionName
def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): def build_local(app, build, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
"""Do a build locally.""" """Do a build locally."""
ndk_path = build.ndk_path() ndk_path = build.ndk_path()
if build.ndk or (build.buildjni and build.buildjni != ['no']): if build.ndk or (build.buildjni and build.buildjni != ['no']):
@ -412,6 +414,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
% (app.id, build.versionName, build.sudo)) % (app.id, build.versionName, build.sudo))
# Prepare the source code... # Prepare the source code...
vcs, build_dir = common.setup_vcs(app)
root_dir, srclibpaths = common.prepare_source(vcs, app, build, root_dir, srclibpaths = common.prepare_source(vcs, app, build,
build_dir, srclib_dir, build_dir, srclib_dir,
extlib_dir, onserver, refresh) extlib_dir, onserver, refresh)
@ -755,8 +758,8 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
os.path.join(output_dir, tarname)) os.path.join(output_dir, tarname))
def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, def trybuild(app, build, output_dir, log_dir, also_check_dir,
srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, test, srclib_dir, extlib_dir, tmp_dir, repo_dir, test,
server, force, onserver, refresh): server, force, onserver, refresh):
"""Build a particular version of an application, if it needs building. """Build a particular version of an application, if it needs building.
@ -802,17 +805,9 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
build.versionName, build.versionCode, app.id)) build.versionName, build.versionCode, app.id))
if server: if server:
# When using server mode, still keep a local cache of the repo, by build_server(app, build, output_dir, log_dir, force, refresh)
# grabbing the source now.
vcs.gotorevision(build.commit, refresh)
# Initialise submodules if required
if build.submodules:
vcs.initsubmodules()
build_server(app, build, vcs, build_dir, output_dir, log_dir, force)
else: else:
build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh) build_local(app, build, output_dir, log_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
return True return True
@ -1020,9 +1015,6 @@ def main():
endtime = time.time() + 72 * 60 * 60 endtime = time.time() + 72 * 60 * 60
max_build_time_reached = False max_build_time_reached = False
for appid, app in apps.items(): for appid, app in apps.items():
first = True
for build in app.get('Builds', []): for build in app.get('Builds', []):
if time.time() > endtime: if time.time() > endtime:
max_build_time_reached = True max_build_time_reached = True
@ -1043,18 +1035,10 @@ def main():
tools_version_log = '' tools_version_log = ''
try: try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
vcs, build_dir = common.setup_vcs(app)
first = False
logging.debug("Checking %s:%s" % (appid, build.versionCode)) logging.debug("Checking %s:%s" % (appid, build.versionCode))
if trybuild(app, build, build_dir, output_dir, log_dir, if trybuild(app, build, output_dir, log_dir,
also_check_dir, srclib_dir, extlib_dir, also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, options.test, tmp_dir, repo_dir, options.test,
options.server, options.force, options.server, options.force,
options.onserver, options.refresh): options.onserver, options.refresh):
toolslog = os.path.join(log_dir, toolslog = os.path.join(log_dir,

10
fdroidserver/common.py
View File

@ -1111,7 +1111,6 @@ class vcs:
self.local = local self.local = local
self.clone_failed = False self.clone_failed = False
self.refreshed = False self.refreshed = False
self.srclib = None
def _gettags(self): def _gettags(self):
raise NotImplementedError raise NotImplementedError
@ -1220,10 +1219,6 @@ class vcs:
"""Get current commit reference (hash, revision, etc).""" """Get current commit reference (hash, revision, etc)."""
raise VCSException('getref not supported for this vcs type') raise VCSException('getref not supported for this vcs type')
def getsrclib(self):
"""Return the srclib (name, path) used in setting up the current revision, or None."""
return self.srclib
class vcs_git(vcs): class vcs_git(vcs):
@ -2258,11 +2253,6 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
for name, number, libpath in srclibpaths: for name, number, libpath in srclibpaths:
place_srclib(root_dir, int(number) if number else None, libpath) place_srclib(root_dir, int(number) if number else None, libpath)
basesrclib = vcs.getsrclib()
# If one was used for the main source, add that too.
if basesrclib:
srclibpaths.append(basesrclib)
# Update the local.properties file # Update the local.properties file
localprops = [os.path.join(build_dir, 'local.properties')] localprops = [os.path.join(build_dir, 'local.properties')]
if build.subdir: if build.subdir:

117
tests/build.TestCase
View File

@ -223,6 +223,7 @@ class BuildTest(unittest.TestCase):
@mock.patch('fdroidserver.build.FDroidPopen') @mock.patch('fdroidserver.build.FDroidPopen')
@mock.patch('fdroidserver.common.is_apk_and_debuggable', lambda f: False) @mock.patch('fdroidserver.common.is_apk_and_debuggable', lambda f: False)
@mock.patch('fdroidserver.common.get_native_code', lambda f: 'x86') @mock.patch('fdroidserver.common.get_native_code', lambda f: 'x86')
@mock.patch('fdroidserver.common.setup_vcs', lambda f: (mock.Mock(), "."))
def test_build_local_maven(self, fake_FDroidPopen, fake_get_apk_id): def test_build_local_maven(self, fake_FDroidPopen, fake_get_apk_id):
"""Test build_local() with a maven project""" """Test build_local() with a maven project"""
@ -258,14 +259,11 @@ class BuildTest(unittest.TestCase):
build.versionCode, build.versionCode,
build.versionName, build.versionName,
) )
vcs = mock.Mock()
build.maven = 'yes@..' build.maven = 'yes@..'
fdroidserver.build.build_local( fdroidserver.build.build_local(
app, app,
build, build,
vcs,
build_dir=self.testdir,
output_dir=self.testdir, output_dir=self.testdir,
log_dir=os.getcwd(), log_dir=os.getcwd(),
srclib_dir=None, srclib_dir=None,
@ -280,8 +278,6 @@ class BuildTest(unittest.TestCase):
fdroidserver.build.build_local( fdroidserver.build.build_local(
app, app,
build, build,
vcs,
build_dir=self.testdir,
output_dir=self.testdir, output_dir=self.testdir,
log_dir=os.getcwd(), log_dir=os.getcwd(),
srclib_dir=None, srclib_dir=None,
@ -316,7 +312,6 @@ class BuildTest(unittest.TestCase):
build.ndk = 'r21e' # aka 21.4.7075529 build.ndk = 'r21e' # aka 21.4.7075529
ndk_version = '21.4.7075529' ndk_version = '21.4.7075529'
ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version
vcs = mock.Mock()
def make_fake_apk(output, build): def make_fake_apk(output, build):
with open(build.output, 'w') as fp: with open(build.output, 'w') as fp:
@ -347,6 +342,8 @@ class BuildTest(unittest.TestCase):
'fdroidserver.build.FDroidPopen', FakeProcess 'fdroidserver.build.FDroidPopen', FakeProcess
) as _ignored, mock.patch( ) as _ignored, mock.patch(
'sdkmanager.install', wraps=fake_sdkmanager_install 'sdkmanager.install', wraps=fake_sdkmanager_install
) as _ignored, mock.patch(
'fdroidserver.common.setup_vcs', return_value=(mock.Mock(), testdir)
) as _ignored: ) as _ignored:
_ignored # silence the linters _ignored # silence the linters
with self.assertRaises( with self.assertRaises(
@ -356,8 +353,6 @@ class BuildTest(unittest.TestCase):
fdroidserver.build.build_local( fdroidserver.build.build_local(
app, app,
build, build,
vcs,
build_dir=testdir,
output_dir=testdir, output_dir=testdir,
log_dir=None, log_dir=None,
srclib_dir=None, srclib_dir=None,
@ -375,8 +370,6 @@ class BuildTest(unittest.TestCase):
fdroidserver.build.build_local( fdroidserver.build.build_local(
app, app,
build, build,
vcs,
build_dir=testdir,
output_dir=testdir, output_dir=testdir,
log_dir=os.getcwd(), log_dir=os.getcwd(),
srclib_dir=None, srclib_dir=None,
@ -399,6 +392,7 @@ class BuildTest(unittest.TestCase):
'fdroidserver.common.sha256sum', 'fdroidserver.common.sha256sum',
lambda f: 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e', lambda f: 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
) )
@mock.patch('fdroidserver.common.setup_vcs', lambda f: (mock.Mock(), "."))
def test_build_local_ndk_some_installed(self): def test_build_local_ndk_some_installed(self):
"""Test if `fdroid build` detects installed NDKs and auto-installs when missing""" """Test if `fdroid build` detects installed NDKs and auto-installs when missing"""
with tempfile.TemporaryDirectory() as testdir, TmpCwd( with tempfile.TemporaryDirectory() as testdir, TmpCwd(
@ -426,7 +420,6 @@ class BuildTest(unittest.TestCase):
build.ndk = 'r21e' # aka 21.4.7075529 build.ndk = 'r21e' # aka 21.4.7075529
ndk_version = '21.4.7075529' ndk_version = '21.4.7075529'
ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version
vcs = mock.Mock()
def make_fake_apk(output, build): def make_fake_apk(output, build):
with open(build.output, 'w') as fp: with open(build.output, 'w') as fp:
@ -456,8 +449,6 @@ class BuildTest(unittest.TestCase):
fdroidserver.build.build_local( fdroidserver.build.build_local(
app, app,
build, build,
vcs,
build_dir=testdir,
output_dir=testdir, output_dir=testdir,
log_dir=os.getcwd(), log_dir=os.getcwd(),
srclib_dir=None, srclib_dir=None,
@ -491,7 +482,6 @@ class BuildTest(unittest.TestCase):
build.scanignore = ['foo.aar'] build.scanignore = ['foo.aar']
build.versionCode = 1 build.versionCode = 1
build.versionName = '1.0' build.versionName = '1.0'
vcs = mock.Mock()
os.mkdir('reports') os.mkdir('reports')
os.mkdir('target') os.mkdir('target')
@ -524,29 +514,31 @@ class BuildTest(unittest.TestCase):
fp.write('APK PLACEHOLDER') fp.write('APK PLACEHOLDER')
return output return output
with mock.patch('fdroidserver.common.replace_build_vars', wraps=make_fake_apk): with mock.patch(
with mock.patch('fdroidserver.common.get_native_code', return_value='x86'): 'fdroidserver.common.replace_build_vars', wraps=make_fake_apk
with mock.patch( ) as _ignored, mock.patch(
'fdroidserver.common.get_apk_id', 'fdroidserver.common.get_native_code', return_value='x86'
return_value=(app.id, build.versionCode, build.versionName), ) as _ignored, mock.patch(
): 'fdroidserver.common.get_apk_id',
with mock.patch( return_value=(app.id, build.versionCode, build.versionName),
'fdroidserver.common.is_apk_and_debuggable', return_value=False ) as _ignored, mock.patch(
): 'fdroidserver.common.is_apk_and_debuggable', return_value=False
fdroidserver.build.build_local( ) as _ignored, mock.patch(
app, 'fdroidserver.common.setup_vcs', return_value=(mock.Mock(), self.testdir)
build, ) as _ignored:
vcs, _ignored # silence the linters
build_dir=self.testdir, fdroidserver.build.build_local(
output_dir=self.testdir, app,
log_dir=None, build,
srclib_dir=None, output_dir=self.testdir,
extlib_dir=None, log_dir=None,
tmp_dir=None, srclib_dir=None,
force=False, extlib_dir=None,
onserver=False, tmp_dir=None,
refresh=False, force=False,
) onserver=False,
refresh=False,
)
self.assertTrue(os.path.exists('foo.aar')) self.assertTrue(os.path.exists('foo.aar'))
self.assertTrue(os.path.isdir('build')) self.assertTrue(os.path.isdir('build'))
@ -881,8 +873,6 @@ class BuildTest(unittest.TestCase):
} }
fdroidserver.common.config = {'sdk_path': '/fake/android/sdk/path'} fdroidserver.common.config = {'sdk_path': '/fake/android/sdk/path'}
fdroidserver.build.options = mock.MagicMock() fdroidserver.build.options = mock.MagicMock()
vcs = mock.Mock()
vcs.getsrclib = mock.Mock(return_value=None)
app = fdroidserver.metadata.App() app = fdroidserver.metadata.App()
app['metadatapath'] = 'metadata/fake.id.yml' app['metadatapath'] = 'metadata/fake.id.yml'
app['id'] = 'fake.id' app['id'] = 'fake.id'
@ -897,29 +887,33 @@ class BuildTest(unittest.TestCase):
) )
app['Builds'] = [build] app['Builds'] = [build]
test_flag = ('--on-server', True) with mock.patch(
fdroidserver.build.build_server(app, build, vcs, '', '', '', False) 'fdroidserver.common.setup_vcs', return_value=(mock.Mock(), "")
self.assertTrue(fdroidserver_vmtools_get_build_vm.called) ) as _ignored:
_ignored # silence the linters
test_flag = ('--on-server', True)
fdroidserver.build.build_server(app, build, '', '', False, False)
self.assertTrue(fdroidserver_vmtools_get_build_vm.called)
for force in (True, False): for force in (True, False):
test_flag = ('--force', force) test_flag = ('--force', force)
fdroidserver.build.build_server(app, build, vcs, '', '', '', force) fdroidserver.build.build_server(app, build, '', '', force, False)
fdroidserver.build.options.notarball = True fdroidserver.build.options.notarball = True
test_flag = ('--no-tarball', True) test_flag = ('--no-tarball', True)
fdroidserver.build.build_server(app, build, vcs, '', '', '', False) fdroidserver.build.build_server(app, build, '', '', False, False)
fdroidserver.build.options.notarball = False fdroidserver.build.options.notarball = False
test_flag = ('--no-tarball', False) test_flag = ('--no-tarball', False)
fdroidserver.build.build_server(app, build, vcs, '', '', '', False) fdroidserver.build.build_server(app, build, '', '', False, False)
fdroidserver.build.options.skipscan = False fdroidserver.build.options.skipscan = False
test_flag = ('--scan-binary', True) test_flag = ('--scan-binary', True)
fdroidserver.build.build_server(app, build, vcs, '', '', '', False) fdroidserver.build.build_server(app, build, '', '', False, False)
fdroidserver.build.options.skipscan = True fdroidserver.build.options.skipscan = True
test_flag = ('--scan-binary', False) test_flag = ('--scan-binary', False)
fdroidserver.build.build_server(app, build, vcs, '', '', '', False) fdroidserver.build.build_server(app, build, '', '', False, False)
test_flag = ('--skip-scan', True) test_flag = ('--skip-scan', True)
fdroidserver.build.build_server(app, build, vcs, '', '', '', False) fdroidserver.build.build_server(app, build, '', '', False, False)
@mock.patch('fdroidserver.vmtools.get_build_vm') @mock.patch('fdroidserver.vmtools.get_build_vm')
@mock.patch('fdroidserver.vmtools.get_clean_builder') @mock.patch('fdroidserver.vmtools.get_clean_builder')
@ -930,6 +924,7 @@ class BuildTest(unittest.TestCase):
@mock.patch('fdroidserver.build.build_local') @mock.patch('fdroidserver.build.build_local')
@mock.patch('fdroidserver.common.get_android_tools_version_log', lambda: 'versions') @mock.patch('fdroidserver.common.get_android_tools_version_log', lambda: 'versions')
@mock.patch('fdroidserver.common.deploy_build_log_with_rsync', lambda a, b, c: None) @mock.patch('fdroidserver.common.deploy_build_log_with_rsync', lambda a, b, c: None)
@mock.patch('fdroidserver.common.setup_vcs', lambda f: (mock.Mock(), "."))
def test_build_server_no_local_prepare( def test_build_server_no_local_prepare(
self, self,
build_build_local, build_build_local,
@ -1005,8 +1000,6 @@ class BuildTest(unittest.TestCase):
fdroidserver.build.options = options fdroidserver.build.options = options
fdroidserver.build.config = {'sdk_path': '/fake/android/sdk/path'} fdroidserver.build.config = {'sdk_path': '/fake/android/sdk/path'}
vcs = mock.Mock()
vcs.getsrclib = mock.Mock(return_value=None)
app = fdroidserver.metadata.App() app = fdroidserver.metadata.App()
app['metadatapath'] = 'metadata/fake.id.yml' app['metadatapath'] = 'metadata/fake.id.yml'
app['id'] = 'fake.id' app['id'] = 'fake.id'
@ -1033,7 +1026,6 @@ class BuildTest(unittest.TestCase):
fdroidserver.build.trybuild( fdroidserver.build.trybuild(
app, app,
build, build,
build_dir,
'unsigned', 'unsigned',
'logs', 'logs',
None, None,
@ -1041,7 +1033,6 @@ class BuildTest(unittest.TestCase):
extlib_dir, extlib_dir,
'tmp', 'tmp',
'repo', 'repo',
vcs,
options.test, options.test,
options.server, options.server,
options.force, options.force,

49
tests/scanner.TestCase
View File

@ -246,7 +246,6 @@ class ScannerTest(unittest.TestCase):
build.scanignore = ['baz.so', 'foo.aar'] build.scanignore = ['baz.so', 'foo.aar']
build.versionCode = 1 build.versionCode = 1
build.versionName = '1.0' build.versionName = '1.0'
vcs = mock.Mock()
for f in ('baz.so', 'foo.aar', 'gradle-wrapper.jar'): for f in ('baz.so', 'foo.aar', 'gradle-wrapper.jar'):
with open(f, 'w') as fp: with open(f, 'w') as fp:
@ -269,29 +268,31 @@ class ScannerTest(unittest.TestCase):
fp.write('APK PLACEHOLDER') fp.write('APK PLACEHOLDER')
return output return output
with mock.patch('fdroidserver.common.replace_build_vars', wraps=make_fake_apk): with mock.patch(
with mock.patch('fdroidserver.common.get_native_code', return_value='x86'): 'fdroidserver.common.replace_build_vars', wraps=make_fake_apk
with mock.patch( ) as _ignored, mock.patch(
'fdroidserver.common.get_apk_id', 'fdroidserver.common.get_native_code', return_value='x86'
return_value=(app.id, build.versionCode, build.versionName), ) as _ignored, mock.patch(
): 'fdroidserver.common.get_apk_id',
with mock.patch( return_value=(app.id, build.versionCode, build.versionName),
'fdroidserver.common.is_apk_and_debuggable', return_value=False ) as _ignored, mock.patch(
): 'fdroidserver.common.is_apk_and_debuggable', return_value=False
fdroidserver.build.build_local( ) as _ignored, mock.patch(
app, 'fdroidserver.common.setup_vcs', return_value=(mock.Mock(), self.testdir)
build, ) as _ignored:
vcs, _ignored # silence the linters
build_dir=self.testdir, fdroidserver.build.build_local(
output_dir=self.testdir, app,
log_dir=None, build,
srclib_dir=None, output_dir=self.testdir,
extlib_dir=None, log_dir=None,
tmp_dir=None, srclib_dir=None,
force=False, extlib_dir=None,
onserver=False, tmp_dir=None,
refresh=False, force=False,
) onserver=False,
refresh=False,
)
self.assertTrue(os.path.exists('baz.so')) self.assertTrue(os.path.exists('baz.so'))
self.assertTrue(os.path.exists('foo.aar')) self.assertTrue(os.path.exists('foo.aar'))
self.assertFalse(os.path.exists('gradle-wrapper.jar')) self.assertFalse(os.path.exists('gradle-wrapper.jar'))