mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-05 06:50:10 +01:00
Merge branch 'master' into 'master'
modernize PEP8 checks to prepare for Python 3.7 See merge request fdroid/fdroidserver!512
This commit is contained in:
Hans-Christoph Steiner
commit
e8bad73ae4
@ -105,14 +105,14 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
||||
logging.debug("rsyncing " + path + " to " + ftp.getcwd())
|
||||
# TODO this should move to `vagrant rsync` from >= v1.5
|
||||
try:
|
||||
subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh=' +
|
||||
'ssh -o StrictHostKeyChecking=no' +
|
||||
' -o UserKnownHostsFile=/dev/null' +
|
||||
' -o LogLevel=FATAL' +
|
||||
' -o IdentitiesOnly=yes' +
|
||||
' -o PasswordAuthentication=no' +
|
||||
' -p ' + str(sshinfo['port']) +
|
||||
' -i ' + sshinfo['idfile'],
|
||||
subprocess.check_output(['rsync', '--recursive', '--perms', '--links', '--quiet', '--rsh='
|
||||
+ 'ssh -o StrictHostKeyChecking=no'
|
||||
+ ' -o UserKnownHostsFile=/dev/null'
|
||||
+ ' -o LogLevel=FATAL'
|
||||
+ ' -o IdentitiesOnly=yes'
|
||||
+ ' -o PasswordAuthentication=no'
|
||||
+ ' -p ' + str(sshinfo['port'])
|
||||
+ ' -i ' + sshinfo['idfile'],
|
||||
path,
|
||||
sshinfo['user'] + "@" + sshinfo['hostname'] + ":" + ftp.getcwd()],
|
||||
stderr=subprocess.STDOUT)
|
||||
|
||||
@ -352,8 +352,8 @@ def assert_config_keystore(config):
|
||||
nosigningkey = True
|
||||
logging.critical(_("'keypass' not found in config.py!"))
|
||||
if nosigningkey:
|
||||
raise FDroidException("This command requires a signing key, " +
|
||||
"you can create one using: fdroid update --create-key")
|
||||
raise FDroidException("This command requires a signing key, "
|
||||
+ "you can create one using: fdroid update --create-key")
|
||||
|
||||
|
||||
def find_sdk_tools_cmd(cmd):
|
||||
@ -560,7 +560,7 @@ def clean_description(description):
|
||||
paragraph = re.sub('\r', '', paragraph)
|
||||
paragraph = re.sub('\n', ' ', paragraph)
|
||||
paragraph = re.sub(' {2,}', ' ', paragraph)
|
||||
paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
|
||||
paragraph = re.sub(r'^\s*(\w)', r'\1', paragraph)
|
||||
returnstring += paragraph + '\n\n'
|
||||
return returnstring.rstrip('\n')
|
||||
|
||||
@ -575,8 +575,8 @@ def publishednameinfo(filename):
|
||||
return result
|
||||
|
||||
|
||||
apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
|
||||
apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
|
||||
apk_release_filename = re.compile(r'(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
|
||||
apk_release_filename_with_sigfp = re.compile(r'(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
|
||||
|
||||
|
||||
def apk_parse_release_filename(apkname):
|
||||
@ -2148,8 +2148,8 @@ def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdou
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
||||
stderr=stderr_param)
|
||||
except OSError as e:
|
||||
raise BuildException("OSError while trying to execute " +
|
||||
' '.join(commands) + ': ' + str(e))
|
||||
raise BuildException("OSError while trying to execute "
|
||||
+ ' '.join(commands) + ': ' + str(e))
|
||||
|
||||
# TODO are these AsynchronousFileReader threads always exiting?
|
||||
if not stderr_to_stdout and options.verbose:
|
||||
@ -2479,7 +2479,7 @@ def metadata_find_signing_files(appid, vercode):
|
||||
sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
|
||||
glob.glob(os.path.join(sigdir, '*.EC')) + \
|
||||
glob.glob(os.path.join(sigdir, '*.RSA'))
|
||||
extre = re.compile('(\.DSA|\.EC|\.RSA)$')
|
||||
extre = re.compile(r'(\.DSA|\.EC|\.RSA)$')
|
||||
for sig in sigs:
|
||||
sf = extre.sub('.SF', sig)
|
||||
if os.path.isfile(sf):
|
||||
@ -3011,9 +3011,9 @@ def write_to_config(thisconfig, key, value=None, config_file=None):
|
||||
|
||||
# regex for finding and replacing python string variable
|
||||
# definitions/initializations
|
||||
pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
|
||||
pattern = re.compile(r'^[\s#]*' + key + r'\s*=\s*"[^"]*"')
|
||||
repl = key + ' = "' + value + '"'
|
||||
pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
|
||||
pattern2 = re.compile(r'^[\s#]*' + key + r"\s*=\s*'[^']*'")
|
||||
repl2 = key + " = '" + value + "'"
|
||||
|
||||
# If we replaced this line once, we make sure won't be a
|
||||
|
||||
@ -179,8 +179,8 @@ class DockerDriver(object):
|
||||
)['Id']
|
||||
output = self.cli.exec_start(exec_id).decode('utf-8')
|
||||
|
||||
if ("Could not access the Package Manager" in output or
|
||||
"device offline" in output):
|
||||
if ("Could not access the Package Manager" in output
|
||||
or "device offline" in output):
|
||||
logging.info("Device or package manager isn't up")
|
||||
|
||||
if app_id.split('_')[0] in output: # TODO: this is a temporary fix
|
||||
|
||||
@ -32,7 +32,7 @@ from . import metadata
|
||||
from .exception import FDroidException
|
||||
|
||||
|
||||
SETTINGS_GRADLE = re.compile('''include\s+['"]:([^'"]*)['"]''')
|
||||
SETTINGS_GRADLE = re.compile(r'''include\s+['"]:([^'"]*)['"]''')
|
||||
|
||||
|
||||
# Get the repo type and address from the given web page. The page is scanned
|
||||
@ -146,18 +146,18 @@ def get_metadata_from_url(app, url):
|
||||
app.SourceCode = ""
|
||||
app.WebSite = ""
|
||||
if not projecttype:
|
||||
raise FDroidException("Unable to determine the project type. " +
|
||||
"The URL you supplied was not in one of the supported formats. " +
|
||||
"Please consult the manual for a list of supported formats, " +
|
||||
"and supply one of those.")
|
||||
raise FDroidException("Unable to determine the project type. "
|
||||
+ "The URL you supplied was not in one of the supported formats. "
|
||||
+ "Please consult the manual for a list of supported formats, "
|
||||
+ "and supply one of those.")
|
||||
|
||||
# Ensure we have a sensible-looking repo address at this point. If not, we
|
||||
# might have got a page format we weren't expecting. (Note that we
|
||||
# specifically don't want git@...)
|
||||
if ((repotype != 'bzr' and (not repo.startswith('http://') and
|
||||
not repo.startswith('https://') and
|
||||
not repo.startswith('git://'))) or
|
||||
' ' in repo):
|
||||
if ((repotype != 'bzr' and (not repo.startswith('http://')
|
||||
and not repo.startswith('https://')
|
||||
and not repo.startswith('git://')))
|
||||
or ' ' in repo):
|
||||
raise FDroidException("Repo address '{0}' does not seem to be valid".format(repo))
|
||||
|
||||
# Get a copy of the source so we can extract some info...
|
||||
|
||||
@ -39,7 +39,7 @@ def disable_in_config(key, value):
|
||||
'''write a key/value to the local config.py, then comment it out'''
|
||||
with open('config.py', 'r', encoding='utf8') as f:
|
||||
data = f.read()
|
||||
pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
|
||||
pattern = r'\n[\s#]*' + key + r'\s*=\s*"[^"]*"'
|
||||
repl = '\n#' + key + ' = "' + value + '"'
|
||||
data = re.sub(pattern, repl, data)
|
||||
with open('config.py', 'w', encoding='utf8') as f:
|
||||
@ -106,7 +106,7 @@ def main():
|
||||
except KeyboardInterrupt:
|
||||
print('')
|
||||
sys.exit(1)
|
||||
if re.match('^\s*$', s) is not None:
|
||||
if re.match(r'^\s*$', s) is not None:
|
||||
test_config['sdk_path'] = default_sdk_path
|
||||
else:
|
||||
test_config['sdk_path'] = s
|
||||
@ -209,8 +209,8 @@ def main():
|
||||
opensc_so = files[0]
|
||||
else:
|
||||
opensc_so = '/usr/lib/opensc-pkcs11.so'
|
||||
logging.warn('No OpenSC PKCS#11 module found, ' +
|
||||
'install OpenSC then edit "opensc-fdroid.cfg"!')
|
||||
logging.warn('No OpenSC PKCS#11 module found, '
|
||||
+ 'install OpenSC then edit "opensc-fdroid.cfg"!')
|
||||
with open(os.path.join(examplesdir, 'opensc-fdroid.cfg'), 'r') as f:
|
||||
opensc_fdroid = f.read()
|
||||
opensc_fdroid = re.sub('^library.*', 'library = ' + opensc_so, opensc_fdroid,
|
||||
|
||||
@ -811,7 +811,7 @@ def get_default_app_info(metadatapath=None):
|
||||
if os.path.exists('AndroidManifest.xml'):
|
||||
manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
|
||||
else:
|
||||
pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
|
||||
pattern = re.compile(r""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
|
||||
for root, dirs, files in os.walk(os.getcwd()):
|
||||
if 'build.gradle' in files:
|
||||
p = os.path.join(root, 'build.gradle')
|
||||
@ -1036,7 +1036,7 @@ def write_yaml(mf, app):
|
||||
raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
|
||||
if not ruamel.yaml.__version__:
|
||||
raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
|
||||
m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
|
||||
m = re.match(r'(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
|
||||
ruamel.yaml.__version__)
|
||||
if not m:
|
||||
raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
|
||||
|
||||
@ -85,7 +85,7 @@ def read_fingerprints_from_keystore():
|
||||
raise FDroidException('could not read keysotre {}'.format(config['keystore']))
|
||||
|
||||
realias = re.compile('Alias name: (?P<alias>.+)\n')
|
||||
resha256 = re.compile('\s+SHA256: (?P<sha256>[:0-9A-F]{95})\n')
|
||||
resha256 = re.compile(r'\s+SHA256: (?P<sha256>[:0-9A-F]{95})\n')
|
||||
fps = {}
|
||||
for block in p.output.split(('*' * 43) + '\n' + '*' * 43):
|
||||
s_alias = realias.search(block)
|
||||
|
||||
@ -110,20 +110,20 @@ def update_awsbucket_s3cmd(repo_section):
|
||||
s3url = s3bucketurl + '/fdroid/'
|
||||
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
|
||||
logging.debug(_('Running first pass with MD5 checking disabled'))
|
||||
if subprocess.call(s3cmd_sync +
|
||||
['--no-check-md5', '--skip-existing',
|
||||
'--exclude', indexxml,
|
||||
'--exclude', indexjar,
|
||||
'--exclude', indexv1jar,
|
||||
repo_section, s3url]) != 0:
|
||||
if subprocess.call(s3cmd_sync
|
||||
+ ['--no-check-md5', '--skip-existing',
|
||||
'--exclude', indexxml,
|
||||
'--exclude', indexjar,
|
||||
'--exclude', indexv1jar,
|
||||
repo_section, s3url]) != 0:
|
||||
raise FDroidException()
|
||||
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
|
||||
if subprocess.call(s3cmd_sync +
|
||||
['--no-check-md5',
|
||||
'--exclude', indexxml,
|
||||
'--exclude', indexjar,
|
||||
'--exclude', indexv1jar,
|
||||
repo_section, s3url]) != 0:
|
||||
if subprocess.call(s3cmd_sync
|
||||
+ ['--no-check-md5',
|
||||
'--exclude', indexxml,
|
||||
'--exclude', indexjar,
|
||||
'--exclude', indexv1jar,
|
||||
repo_section, s3url]) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
|
||||
@ -256,10 +256,11 @@ def update_serverwebroot(serverwebroot, repo_section):
|
||||
# the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
|
||||
# (serverwebroot is guaranteed to have a trailing slash in common.py)
|
||||
logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
|
||||
if subprocess.call(rsyncargs +
|
||||
['--exclude', indexxml, '--exclude', indexjar,
|
||||
'--exclude', indexv1jar,
|
||||
repo_section, serverwebroot]) != 0:
|
||||
if subprocess.call(rsyncargs
|
||||
+ ['--exclude', indexxml,
|
||||
'--exclude', indexjar,
|
||||
'--exclude', indexv1jar,
|
||||
repo_section, serverwebroot]) != 0:
|
||||
raise FDroidException()
|
||||
if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
@ -57,7 +57,7 @@ def extract(options):
|
||||
sys.exit(1)
|
||||
|
||||
# iterate over supplied APKs downlaod and extract them…
|
||||
httpre = re.compile('https?:\/\/')
|
||||
httpre = re.compile(r'https?:\/\/')
|
||||
for apk in options.APK:
|
||||
try:
|
||||
if os.path.isfile(apk):
|
||||
|
||||
@ -113,8 +113,8 @@ def main():
|
||||
|
||||
destpath = os.path.join(logsdir, f)
|
||||
destsize = ftp.stat(f).st_size
|
||||
if (not os.path.exists(destpath) or
|
||||
os.path.getsize(destpath) != destsize):
|
||||
if not os.path.exists(destpath) \
|
||||
or os.path.getsize(destpath) != destsize:
|
||||
logging.debug("...retrieving " + f)
|
||||
ftp.get(f, destpath)
|
||||
except Exception:
|
||||
@ -135,9 +135,9 @@ def main():
|
||||
logging.info('Processing logs...')
|
||||
appscount = Counter()
|
||||
appsvercount = Counter()
|
||||
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] ' + \
|
||||
'"GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) ' + \
|
||||
'\d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
|
||||
logexpr = r'(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] ' \
|
||||
+ r'"GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) ' \
|
||||
+ r'\d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
|
||||
logsearch = re.compile(logexpr).search
|
||||
for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
|
||||
logging.debug('...' + logfile)
|
||||
|
||||
@ -53,7 +53,7 @@ UNSET_VERSION_CODE = -0x100000000
|
||||
APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
|
||||
APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
|
||||
APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
|
||||
APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*?)'")
|
||||
APK_LABEL_ICON_PAT = re.compile(r".*\s+label='(.*)'\s+icon='(.*?)'")
|
||||
APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
|
||||
APK_PERMISSION_PAT = \
|
||||
re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
|
||||
@ -1083,7 +1083,7 @@ def _get_apk_icons_src(apkfile, icon_name):
|
||||
|
||||
"""
|
||||
icons_src = dict()
|
||||
density_re = re.compile('^res/(.*)/{}\.(png|xml)$'.format(icon_name))
|
||||
density_re = re.compile(r'^res/(.*)/{}\.(png|xml)$'.format(icon_name))
|
||||
with zipfile.ZipFile(apkfile) as zf:
|
||||
for filename in zf.namelist():
|
||||
m = density_re.match(filename)
|
||||
@ -1853,8 +1853,8 @@ def main():
|
||||
parser.add_argument("-I", "--icons", action="store_true", default=False,
|
||||
help=_("Resize all the icons exceeding the max pixel size and exit"))
|
||||
parser.add_argument("-e", "--editor", default="/etc/alternatives/editor",
|
||||
help=_("Specify editor to use in interactive mode. Default " +
|
||||
"is {path}").format(path='/etc/alternatives/editor'))
|
||||
help=_("Specify editor to use in interactive mode. Default is {path}")
|
||||
.format(path='/etc/alternatives/editor'))
|
||||
parser.add_argument("-w", "--wiki", default=False, action="store_true",
|
||||
help=_("Update the wiki"))
|
||||
parser.add_argument("--pretty", action="store_true", default=False,
|
||||
|
||||
@ -368,8 +368,8 @@ class LibvirtBuildVm(FDroidBuildVm):
|
||||
def package(self, output=None, keep_box_file=False):
|
||||
if not output:
|
||||
output = "buildserver.box"
|
||||
logger.debug('no output name set for packaging \'%s\',' +
|
||||
'defaulting to %s', self.srvname, output)
|
||||
logger.debug('no output name set for packaging \'%s\','
|
||||
+ 'defaulting to %s', self.srvname, output)
|
||||
storagePool = self.conn.storagePoolLookupByName('default')
|
||||
domainInfo = self.conn.lookupByName(self.srvname).info()
|
||||
if storagePool:
|
||||
@ -434,8 +434,8 @@ class LibvirtBuildVm(FDroidBuildVm):
|
||||
os.remove('box.img')
|
||||
|
||||
else:
|
||||
logger.warn('could not connect to storage-pool \'default\',' +
|
||||
'skipping packaging buildserver box')
|
||||
logger.warn('could not connect to storage-pool \'default\','
|
||||
+ 'skipping packaging buildserver box')
|
||||
|
||||
def box_add(self, boxname, boxfile, force=True):
|
||||
boximg = '%s_vagrant_box_image_0.img' % (boxname)
|
||||
|
||||
@ -602,8 +602,7 @@ def main():
|
||||
sys.exit(1)
|
||||
|
||||
if not options.keep_box_file:
|
||||
logger.debug('box added to vagrant, ' +
|
||||
'removing generated box file \'%s\'',
|
||||
logger.debug("""box added to vagrant, removing generated box file '%s'""",
|
||||
boxfile)
|
||||
os.remove(boxfile)
|
||||
|
||||
|
||||
@ -61,13 +61,13 @@ class BuildTest(unittest.TestCase):
|
||||
os.path.join(testdir, 'source-files'))
|
||||
teststring = 'FAKE_VERSION_FOR_TESTING'
|
||||
fdroidserver.build.force_gradle_build_tools(testdir, teststring)
|
||||
pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8'))
|
||||
pattern = re.compile(r"buildToolsVersion[\s=]+'%s'\s+" % teststring)
|
||||
for p in ('source-files/fdroid/fdroidclient/build.gradle',
|
||||
'source-files/Zillode/syncthing-silk/build.gradle',
|
||||
'source-files/open-keychain/open-keychain/build.gradle',
|
||||
'source-files/osmandapp/osmand/build.gradle',
|
||||
'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle'):
|
||||
with open(os.path.join(testdir, p), 'rb') as f:
|
||||
with open(os.path.join(testdir, p), 'r') as f:
|
||||
filedata = f.read()
|
||||
self.assertIsNotNone(pattern.search(filedata))
|
||||
|
||||
|
||||
@ -204,7 +204,7 @@ class CommonTest(unittest.TestCase):
|
||||
|
||||
with open(os.path.join(fdroidclient_testdir, 'build.gradle'), 'r') as f:
|
||||
filedata = f.read()
|
||||
self.assertIsNotNone(re.search("\s+compileSdkVersion %s\s+" % testint, filedata))
|
||||
self.assertIsNotNone(re.search(r"\s+compileSdkVersion %s\s+" % testint, filedata))
|
||||
|
||||
with open(os.path.join(fdroidclient_testdir, 'AndroidManifest.xml')) as f:
|
||||
filedata = f.read()
|
||||
|
||||
@ -53,9 +53,9 @@ class IndexTest(unittest.TestCase):
|
||||
_, fingerprint = fdroidserver.index.get_public_key_from_jar(jar)
|
||||
# comparing fingerprints should be sufficient
|
||||
if f == 'testy.jar':
|
||||
self.assertTrue(fingerprint ==
|
||||
'818E469465F96B704E27BE2FEE4C63AB' +
|
||||
'9F83DDF30E7A34C7371A4728D83B0BC1')
|
||||
self.assertEqual(fingerprint,
|
||||
'818E469465F96B704E27BE2FEE4C63AB'
|
||||
+ '9F83DDF30E7A34C7371A4728D83B0BC1')
|
||||
if f == 'guardianproject.jar':
|
||||
self.assertTrue(fingerprint == GP_FINGERPRINT)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user