mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-05 18:50:09 +02:00
encode filenames as bytes to handle all locale setups
This was failing on environments that did not have any LANG or LC_* locale variables set. This is a valid setup, and is common in headless setups, so it needs to be handled. This also adds a new pass of the test suite without the locale env vars set so that this situation is also tests on gitlab-ci, not only gpjenkins. The error this caused was: UnicodeEncodeError: 'ascii' codec can't encode characters in position 6-18: ordinal not in range(128)
This commit is contained in:
parent
4d50ab9bad
commit
e58ad330f4
@ -1,4 +1,4 @@
|
|||||||
image: fdroid/ci:server-20161223
|
image: registry.gitlab.com/fdroid/ci-images:server-latest
|
||||||
|
|
||||||
test:
|
test:
|
||||||
script:
|
script:
|
||||||
|
@ -1700,7 +1700,8 @@ class KnownApks:
|
|||||||
|
|
||||||
def get_file_extension(filename):
|
def get_file_extension(filename):
|
||||||
"""get the normalized file extension, can be blank string but never None"""
|
"""get the normalized file extension, can be blank string but never None"""
|
||||||
|
if isinstance(filename, bytes):
|
||||||
|
filename = filename.decode('utf-8')
|
||||||
return os.path.splitext(filename)[1].lower()[1:]
|
return os.path.splitext(filename)[1].lower()[1:]
|
||||||
|
|
||||||
|
|
||||||
@ -2333,15 +2334,17 @@ def get_per_app_repos():
|
|||||||
|
|
||||||
def is_repo_file(filename):
|
def is_repo_file(filename):
|
||||||
'''Whether the file in a repo is a build product to be delivered to users'''
|
'''Whether the file in a repo is a build product to be delivered to users'''
|
||||||
|
if isinstance(filename, str):
|
||||||
|
filename = filename.encode('utf-8', errors="surrogateescape")
|
||||||
return os.path.isfile(filename) \
|
return os.path.isfile(filename) \
|
||||||
and not filename.endswith('.asc') \
|
and not filename.endswith(b'.asc') \
|
||||||
and not filename.endswith('.sig') \
|
and not filename.endswith(b'.sig') \
|
||||||
and os.path.basename(filename) not in [
|
and os.path.basename(filename) not in [
|
||||||
'index.jar',
|
b'index.jar',
|
||||||
'index_unsigned.jar',
|
b'index_unsigned.jar',
|
||||||
'index.xml',
|
b'index.xml',
|
||||||
'index.html',
|
b'index.html',
|
||||||
'index-v1.jar',
|
b'index-v1.jar',
|
||||||
'index-v1.json',
|
b'index-v1.json',
|
||||||
'categories.txt',
|
b'categories.txt',
|
||||||
]
|
]
|
||||||
|
@ -436,14 +436,14 @@ def make_v0(apps, apks, repodir, repodict, requestsdict):
|
|||||||
and common.config['make_current_version_link'] \
|
and common.config['make_current_version_link'] \
|
||||||
and repodir == 'repo': # only create these
|
and repodir == 'repo': # only create these
|
||||||
namefield = common.config['current_version_name_source']
|
namefield = common.config['current_version_name_source']
|
||||||
sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield))
|
sanitized_name = re.sub(b'''[ '"&%?+=/]''', b'', app.get(namefield).encode('utf-8'))
|
||||||
apklinkname = sanitized_name + '.apk'
|
apklinkname = sanitized_name + b'.apk'
|
||||||
current_version_path = os.path.join(repodir, current_version_file)
|
current_version_path = os.path.join(repodir, current_version_file).encode('utf-8', 'surrogateescape')
|
||||||
if os.path.islink(apklinkname):
|
if os.path.islink(apklinkname):
|
||||||
os.remove(apklinkname)
|
os.remove(apklinkname)
|
||||||
os.symlink(current_version_path, apklinkname)
|
os.symlink(current_version_path, apklinkname)
|
||||||
# also symlink gpg signature, if it exists
|
# also symlink gpg signature, if it exists
|
||||||
for extension in ('.asc', '.sig'):
|
for extension in (b'.asc', b'.sig'):
|
||||||
sigfile_path = current_version_path + extension
|
sigfile_path = current_version_path + extension
|
||||||
if os.path.exists(sigfile_path):
|
if os.path.exists(sigfile_path):
|
||||||
siglinkname = apklinkname + extension
|
siglinkname = apklinkname + extension
|
||||||
|
@ -656,13 +656,15 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
|||||||
|
|
||||||
cachechanged = False
|
cachechanged = False
|
||||||
repo_files = []
|
repo_files = []
|
||||||
|
repodir = repodir.encode('utf-8')
|
||||||
for name in os.listdir(repodir):
|
for name in os.listdir(repodir):
|
||||||
file_extension = common.get_file_extension(name)
|
file_extension = common.get_file_extension(name)
|
||||||
if file_extension == 'apk' or file_extension == 'obb':
|
if file_extension == 'apk' or file_extension == 'obb':
|
||||||
continue
|
continue
|
||||||
filename = os.path.join(repodir, name)
|
filename = os.path.join(repodir, name)
|
||||||
if filename.endswith('_src.tar.gz'):
|
name_utf8 = name.decode('utf-8')
|
||||||
logging.debug('skipping source tarball: ' + filename)
|
if filename.endswith(b'_src.tar.gz'):
|
||||||
|
logging.debug('skipping source tarball: ' + filename.decode('utf-8'))
|
||||||
continue
|
continue
|
||||||
if not common.is_repo_file(filename):
|
if not common.is_repo_file(filename):
|
||||||
continue
|
continue
|
||||||
@ -683,34 +685,34 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
|||||||
else:
|
else:
|
||||||
repo_file['added'] = datetime(*a[:6])
|
repo_file['added'] = datetime(*a[:6])
|
||||||
if repo_file.get('hash') == shasum:
|
if repo_file.get('hash') == shasum:
|
||||||
logging.debug("Reading " + name + " from cache")
|
logging.debug("Reading " + name_utf8 + " from cache")
|
||||||
usecache = True
|
usecache = True
|
||||||
else:
|
else:
|
||||||
logging.debug("Ignoring stale cache data for " + name)
|
logging.debug("Ignoring stale cache data for " + name)
|
||||||
|
|
||||||
if not usecache:
|
if not usecache:
|
||||||
logging.debug("Processing " + name)
|
logging.debug("Processing " + name_utf8)
|
||||||
repo_file = collections.OrderedDict()
|
repo_file = collections.OrderedDict()
|
||||||
# TODO rename apkname globally to something more generic
|
# TODO rename apkname globally to something more generic
|
||||||
repo_file['name'] = name
|
repo_file['name'] = name_utf8
|
||||||
repo_file['apkName'] = name
|
repo_file['apkName'] = name_utf8
|
||||||
repo_file['hash'] = shasum
|
repo_file['hash'] = shasum
|
||||||
repo_file['hashType'] = 'sha256'
|
repo_file['hashType'] = 'sha256'
|
||||||
repo_file['versionCode'] = 0
|
repo_file['versionCode'] = 0
|
||||||
repo_file['versionName'] = shasum
|
repo_file['versionName'] = shasum
|
||||||
# the static ID is the SHA256 unless it is set in the metadata
|
# the static ID is the SHA256 unless it is set in the metadata
|
||||||
repo_file['packageName'] = shasum
|
repo_file['packageName'] = shasum
|
||||||
n = name.split('_')
|
n = name_utf8.split('_')
|
||||||
if len(n) == 2:
|
if len(n) == 2:
|
||||||
packageName = n[0]
|
packageName = n[0]
|
||||||
versionCode = n[1].split('.')[0]
|
versionCode = n[1].split('.')[0]
|
||||||
if re.match(r'^-?[0-9]+$', versionCode) \
|
if re.match('^-?[0-9]+$', versionCode) \
|
||||||
and common.is_valid_package_name(name.split('_')[0]):
|
and common.is_valid_package_name(name_utf8.split('_')[0]):
|
||||||
repo_file['packageName'] = packageName
|
repo_file['packageName'] = packageName
|
||||||
repo_file['versionCode'] = int(versionCode)
|
repo_file['versionCode'] = int(versionCode)
|
||||||
srcfilename = name + "_src.tar.gz"
|
srcfilename = name + b'_src.tar.gz'
|
||||||
if os.path.exists(os.path.join(repodir, srcfilename)):
|
if os.path.exists(os.path.join(repodir, srcfilename)):
|
||||||
repo_file['srcname'] = srcfilename
|
repo_file['srcname'] = srcfilename.decode('utf-8')
|
||||||
repo_file['size'] = stat.st_size
|
repo_file['size'] = stat.st_size
|
||||||
|
|
||||||
apkcache[name] = repo_file
|
apkcache[name] = repo_file
|
||||||
|
Loading…
Reference in New Issue
Block a user