kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-10-03 17:50:11 +02:00
Code Activity

Merge branch 'fix-file-read-write-encoding' into 'master'

Browse Source 
Fix file read/write encoding

This hopefully provides a reasonably complete set of fixes for handling file encoding when reading and writing files.  It is not perfect since it does not handle _.java_, _.gradle_, or _pom.xml_ in any encoding, as is allowed.  But I think it should only be an improvement of the current state, and at worst, should work only as bad as the current setup for most language setups 

See merge request !129
This commit is contained in:
Daniel Martí 2016-06-09 15:13:27 +00:00
commit 83a0be960d
8 changed files with 31 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitlab-ci.yml
View File

@ -2,6 +2,8 @@ image: mvdan/fdroid-ci:server-20160429
test:
script:
- ls -l /usr/lib/python3*
- mkdir -p /usr/lib/python3.4/site-packages/
- pip3 install -e .
- cd tests
- ./complete-ci-tests

32
fdroidserver/common.py
View File

@ -184,10 +184,10 @@ def fill_config_defaults(thisconfig):
def regsub_file(pattern, repl, path):
with open(path, 'r') as f:
with open(path, 'rb') as f:
text = f.read()
text = re.sub(pattern, repl, text)
with open(path, 'w') as f:
text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
with open(path, 'wb') as f:
f.write(text)
@ -1017,7 +1017,7 @@ def get_library_references(root_dir):
proppath = os.path.join(root_dir, 'project.properties')
if not os.path.isfile(proppath):
return libraries
with open(proppath, 'r') as f:
with open(proppath, 'r', encoding='iso-8859-1') as f:
for line in f:
if not line.startswith('android.library.reference.'):
continue
@ -1356,7 +1356,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
props = ""
if os.path.isfile(path):
logging.info("Updating local.properties file at %s" % path)
with open(path, 'r') as f:
with open(path, 'r', encoding='iso-8859-1') as f:
props += f.read()
props += '\n'
else:
@ -1378,7 +1378,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
# Add java.encoding if necessary
if build.encoding:
props += "java.encoding=%s\n" % build.encoding
with open(path, 'w') as f:
with open(path, 'w', encoding='iso-8859-1') as f:
f.write(props)
flavours = []
@ -1539,7 +1539,7 @@ class KnownApks:
self.path = os.path.join('stats', 'known_apks.txt')
self.apks = {}
if os.path.isfile(self.path):
with open(self.path, 'r') as f:
with open(self.path, 'r', encoding='utf8') as f:
for line in f:
t = line.rstrip().split(' ')
if len(t) == 2:
@ -1563,7 +1563,7 @@ class KnownApks:
line += ' ' + time.strftime('%Y-%m-%d', added)
lst.append(line)
with open(self.path, 'w') as f:
with open(self.path, 'w', encoding='utf8') as f:
for line in sorted(lst, key=natural_key):
f.write(line + '\n')
@ -1724,14 +1724,14 @@ def remove_signing_keys(build_dir):
if 'build.gradle' in files:
path = os.path.join(root, 'build.gradle')
with open(path, "r") as o:
with open(path, "r", encoding='utf8') as o:
lines = o.readlines()
changed = False
opened = 0
i = 0
with open(path, "w") as o:
with open(path, "w", encoding='utf8') as o:
while i < len(lines):
line = lines[i]
i += 1
@ -1771,12 +1771,12 @@ def remove_signing_keys(build_dir):
if propfile in files:
path = os.path.join(root, propfile)
with open(path, "r") as o:
with open(path, "r", encoding='iso-8859-1') as o:
lines = o.readlines()
changed = False
with open(path, "w") as o:
with open(path, "w", encoding='iso-8859-1') as o:
for line in lines:
if any(line.startswith(s) for s in ('key.store', 'key.alias')):
changed = True
@ -1838,10 +1838,10 @@ def place_srclib(root_dir, number, libpath):
lines = []
if os.path.isfile(proppath):
with open(proppath, "r") as o:
with open(proppath, "r", encoding='iso-8859-1') as o:
lines = o.readlines()
with open(proppath, "w") as o:
with open(proppath, "w", encoding='iso-8859-1') as o:
placed = False
for line in lines:
if line.startswith('android.library.reference.%d=' % number):
@ -2009,7 +2009,7 @@ def write_to_config(thisconfig, key, value=None):
if value is None:
origkey = key + '_orig'
value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
with open('config.py', 'r') as f:
with open('config.py', 'r', encoding='utf8') as f:
data = f.read()
pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
repl = '\n' + key + ' = "' + value + '"'
@ -2020,7 +2020,7 @@ def write_to_config(thisconfig, key, value=None):
# make sure the file ends with a carraige return
if not re.match('\n$', data):
data += '\n'
with open('config.py', 'w') as f:
with open('config.py', 'w', encoding='utf8') as f:
f.writelines(data)

4
fdroidserver/init.py
View File

@ -35,12 +35,12 @@ options = None
def disable_in_config(key, value):
'''write a key/value to the local config.py, then comment it out'''
with open('config.py', 'r') as f:
with open('config.py', 'r', encoding='utf8') as f:
data = f.read()
pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
repl = '\n#' + key + ' = "' + value + '"'
data = re.sub(pattern, repl, data)
with open('config.py', 'w') as f:
with open('config.py', 'w', encoding='utf8') as f:
f.writelines(data)

4
fdroidserver/metadata.py
View File

@ -836,7 +836,7 @@ def get_default_app_info(metadatapath=None):
for root, dirs, files in os.walk(os.getcwd()):
if 'build.gradle' in files:
p = os.path.join(root, 'build.gradle')
with open(p) as f:
with open(p, 'rb') as f:
data = f.read()
m = pattern.search(data)
if m:
@ -1384,7 +1384,7 @@ def write_metadata(metadatapath, app):
raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % (
metadatapath, ', '.join(accepted)))
with open(metadatapath, 'w') as mf:
with open(metadatapath, 'w', encoding='utf8') as mf:
if ext == 'txt':
return write_txt(mf, app)
elif ext == 'yml':

2
fdroidserver/rewritemeta.py
View File

@ -33,7 +33,7 @@ def proper_format(app):
s = io.StringIO()
# TODO: currently reading entire file again, should reuse first
# read in metadata.py
with open(app.metadatapath, 'r') as f:
with open(app.metadatapath, 'r', encoding='utf8') as f:
cur_content = f.read()
metadata.write_txt(s, app)
content = s.getvalue()

4
fdroidserver/scanner.py
View File

@ -199,7 +199,7 @@ def scan_source(build_dir, root_dir, build):
elif ext == 'java':
if not os.path.isfile(fp):
continue
with open(fp, 'r') as f:
with open(fp, 'r', encoding='utf8') as f:
for line in f:
if 'DexClassLoader' in line:
count += handleproblem('DexClassLoader', fd, fp)
@ -208,7 +208,7 @@ def scan_source(build_dir, root_dir, build):
elif ext == 'gradle':
if not os.path.isfile(fp):
continue
with open(fp, 'r') as f:
with open(fp, 'r', encoding='utf8') as f:
lines = f.readlines()
for i, line in enumerate(lines):
if is_used_by_gradle(line):

8
fdroidserver/update.py
View File

@ -1017,7 +1017,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
catdata = ''
for cat in categories:
catdata += cat + '\n'
with open(os.path.join(repodir, 'categories.txt'), 'w') as f:
with open(os.path.join(repodir, 'categories.txt'), 'w', encoding='utf8') as f:
f.write(catdata)
@ -1229,7 +1229,7 @@ def main():
if 'name' not in apk:
logging.error(apk['id'] + ' does not have a name! Skipping...')
continue
f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w', encoding='utf8')
f.write("License:Unknown\n")
f.write("Web Site:\n")
f.write("Source Code:\n")
@ -1339,7 +1339,7 @@ def main():
# Generate latest apps data for widget
if os.path.exists(os.path.join('stats', 'latestapps.txt')):
data = ''
with open(os.path.join('stats', 'latestapps.txt'), 'r') as f:
with open(os.path.join('stats', 'latestapps.txt'), 'r', encoding='utf8') as f:
for line in f:
appid = line.rstrip()
data += appid + "\t"
@ -1348,7 +1348,7 @@ def main():
if app.icon is not None:
data += app.icon + "\t"
data += app.License + "\n"
with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w') as f:
with open(os.path.join(repodirs[0], 'latestapps.dat'), 'w', encoding='utf8') as f:
f.write(data)
if cachechanged:

4
tests/build.TestCase
View File

@ -57,13 +57,13 @@ class BuildTest(unittest.TestCase):
fdroidserver.build.config = {}
fdroidserver.build.config['build_tools'] = teststring
fdroidserver.build.adapt_gradle(testsdir)
pattern = re.compile("buildToolsVersion[\s=]+'%s'\s+" % teststring)
pattern = re.compile(bytes("buildToolsVersion[\s=]+'%s'\s+" % teststring, 'utf8'))
for p in ('source-files/fdroid/fdroidclient/build.gradle',
'source-files/Zillode/syncthing-silk/build.gradle',
'source-files/open-keychain/open-keychain/build.gradle',
'source-files/osmandapp/osmand/build.gradle',
'source-files/open-keychain/open-keychain/OpenKeychain/build.gradle'):
with open(os.path.join(testsdir, p), 'r') as f:
with open(os.path.join(testsdir, p), 'rb') as f:
filedata = f.read()
self.assertIsNotNone(pattern.search(filedata))