mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
feat(deploy): support index-only mode in libcloud
This commit is contained in:
parent
bd62007b54
commit
3ec997d58e
@ -401,49 +401,52 @@ def update_awsbucket_libcloud(repo_section):
|
||||
if obj.name.startswith(upload_dir + '/'):
|
||||
objs[obj.name] = obj
|
||||
|
||||
for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)):
|
||||
for name in files:
|
||||
upload = False
|
||||
file_to_upload = os.path.join(root, name)
|
||||
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
|
||||
if object_name not in objs:
|
||||
if options.index_only:
|
||||
files_to_upload = _get_index_includes(repo_section)
|
||||
else:
|
||||
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
|
||||
|
||||
for file_to_upload in files_to_upload:
|
||||
upload = False
|
||||
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
|
||||
if object_name not in objs:
|
||||
upload = True
|
||||
else:
|
||||
obj = objs.pop(object_name)
|
||||
if obj.size != os.path.getsize(file_to_upload):
|
||||
upload = True
|
||||
else:
|
||||
obj = objs.pop(object_name)
|
||||
if obj.size != os.path.getsize(file_to_upload):
|
||||
# if the sizes match, then compare by MD5
|
||||
md5 = hashlib.md5() # nosec AWS uses MD5
|
||||
with open(file_to_upload, 'rb') as f:
|
||||
while True:
|
||||
data = f.read(8192)
|
||||
if not data:
|
||||
break
|
||||
md5.update(data)
|
||||
if obj.hash != md5.hexdigest():
|
||||
s3url = 's3://' + awsbucket + '/' + obj.name
|
||||
logging.info(' deleting ' + s3url)
|
||||
if not driver.delete_object(obj):
|
||||
logging.warning('Could not delete ' + s3url)
|
||||
upload = True
|
||||
else:
|
||||
# if the sizes match, then compare by MD5
|
||||
md5 = hashlib.md5() # nosec AWS uses MD5
|
||||
with open(file_to_upload, 'rb') as f:
|
||||
while True:
|
||||
data = f.read(8192)
|
||||
if not data:
|
||||
break
|
||||
md5.update(data)
|
||||
if obj.hash != md5.hexdigest():
|
||||
s3url = 's3://' + awsbucket + '/' + obj.name
|
||||
logging.info(' deleting ' + s3url)
|
||||
if not driver.delete_object(obj):
|
||||
logging.warning('Could not delete ' + s3url)
|
||||
upload = True
|
||||
|
||||
if upload:
|
||||
logging.debug(' uploading "' + file_to_upload + '"...')
|
||||
extra = {'acl': 'public-read'}
|
||||
if file_to_upload.endswith('.sig'):
|
||||
extra['content_type'] = 'application/pgp-signature'
|
||||
elif file_to_upload.endswith('.asc'):
|
||||
extra['content_type'] = 'application/pgp-signature'
|
||||
path = os.path.relpath(file_to_upload)
|
||||
logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}')
|
||||
with open(file_to_upload, 'rb') as iterator:
|
||||
obj = driver.upload_object_via_stream(
|
||||
iterator=iterator,
|
||||
container=container,
|
||||
object_name=object_name,
|
||||
extra=extra,
|
||||
)
|
||||
if upload:
|
||||
logging.debug(' uploading "' + file_to_upload + '"...')
|
||||
extra = {'acl': 'public-read'}
|
||||
if file_to_upload.endswith('.sig'):
|
||||
extra['content_type'] = 'application/pgp-signature'
|
||||
elif file_to_upload.endswith('.asc'):
|
||||
extra['content_type'] = 'application/pgp-signature'
|
||||
path = os.path.relpath(file_to_upload)
|
||||
logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}')
|
||||
with open(file_to_upload, 'rb') as iterator:
|
||||
obj = driver.upload_object_via_stream(
|
||||
iterator=iterator,
|
||||
container=container,
|
||||
object_name=object_name,
|
||||
extra=extra,
|
||||
)
|
||||
# delete the remnants in the bucket, they do not exist locally
|
||||
while objs:
|
||||
object_name, obj = objs.popitem()
|
||||
|
Loading…
Reference in New Issue
Block a user