kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-21 04:10:37 +02:00
Code Activity

feat(deploy): support index-only mode in libcloud

Browse Source
This commit is contained in:
proletarius101 2023-12-14 15:34:37 +00:00
parent bd62007b54
commit 3ec997d58e
No known key found for this signature in database
1 changed files with 42 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
fdroidserver/deploy.py
View File

@ -401,49 +401,52 @@ def update_awsbucket_libcloud(repo_section):
if obj.name.startswith(upload_dir + '/'): if obj.name.startswith(upload_dir + '/'):
objs[obj.name] = obj objs[obj.name] = obj
for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)): if options.index_only:
for name in files: files_to_upload = _get_index_includes(repo_section)
upload = False else:
file_to_upload = os.path.join(root, name) files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
if object_name not in objs: for file_to_upload in files_to_upload:
upload = False
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
if object_name not in objs:
upload = True
else:
obj = objs.pop(object_name)
if obj.size != os.path.getsize(file_to_upload):
upload = True upload = True
else: else:
obj = objs.pop(object_name) # if the sizes match, then compare by MD5
if obj.size != os.path.getsize(file_to_upload): md5 = hashlib.md5() # nosec AWS uses MD5
with open(file_to_upload, 'rb') as f:
while True:
data = f.read(8192)
if not data:
break
md5.update(data)
if obj.hash != md5.hexdigest():
s3url = 's3://' + awsbucket + '/' + obj.name
logging.info(' deleting ' + s3url)
if not driver.delete_object(obj):
logging.warning('Could not delete ' + s3url)
upload = True upload = True
else:
# if the sizes match, then compare by MD5
md5 = hashlib.md5() # nosec AWS uses MD5
with open(file_to_upload, 'rb') as f:
while True:
data = f.read(8192)
if not data:
break
md5.update(data)
if obj.hash != md5.hexdigest():
s3url = 's3://' + awsbucket + '/' + obj.name
logging.info(' deleting ' + s3url)
if not driver.delete_object(obj):
logging.warning('Could not delete ' + s3url)
upload = True
if upload: if upload:
logging.debug(' uploading "' + file_to_upload + '"...') logging.debug(' uploading "' + file_to_upload + '"...')
extra = {'acl': 'public-read'} extra = {'acl': 'public-read'}
if file_to_upload.endswith('.sig'): if file_to_upload.endswith('.sig'):
extra['content_type'] = 'application/pgp-signature' extra['content_type'] = 'application/pgp-signature'
elif file_to_upload.endswith('.asc'): elif file_to_upload.endswith('.asc'):
extra['content_type'] = 'application/pgp-signature' extra['content_type'] = 'application/pgp-signature'
path = os.path.relpath(file_to_upload) path = os.path.relpath(file_to_upload)
logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}') logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}')
with open(file_to_upload, 'rb') as iterator: with open(file_to_upload, 'rb') as iterator:
obj = driver.upload_object_via_stream( obj = driver.upload_object_via_stream(
iterator=iterator, iterator=iterator,
container=container, container=container,
object_name=object_name, object_name=object_name,
extra=extra, extra=extra,
) )
# delete the remnants in the bucket, they do not exist locally # delete the remnants in the bucket, they do not exist locally
while objs: while objs:
object_name, obj = objs.popitem() object_name, obj = objs.popitem()