diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 12b81bb6..5a6f7c7c 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -401,49 +401,52 @@ def update_awsbucket_libcloud(repo_section): if obj.name.startswith(upload_dir + '/'): objs[obj.name] = obj - for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)): - for name in files: - upload = False - file_to_upload = os.path.join(root, name) - object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd()) - if object_name not in objs: + if options.index_only: + files_to_upload = _get_index_includes(repo_section) + else: + files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files] + + for file_to_upload in files_to_upload: + upload = False + object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd()) + if object_name not in objs: + upload = True + else: + obj = objs.pop(object_name) + if obj.size != os.path.getsize(file_to_upload): upload = True else: - obj = objs.pop(object_name) - if obj.size != os.path.getsize(file_to_upload): + # if the sizes match, then compare by MD5 + md5 = hashlib.md5() # nosec AWS uses MD5 + with open(file_to_upload, 'rb') as f: + while True: + data = f.read(8192) + if not data: + break + md5.update(data) + if obj.hash != md5.hexdigest(): + s3url = 's3://' + awsbucket + '/' + obj.name + logging.info(' deleting ' + s3url) + if not driver.delete_object(obj): + logging.warning('Could not delete ' + s3url) upload = True - else: - # if the sizes match, then compare by MD5 - md5 = hashlib.md5() # nosec AWS uses MD5 - with open(file_to_upload, 'rb') as f: - while True: - data = f.read(8192) - if not data: - break - md5.update(data) - if obj.hash != md5.hexdigest(): - s3url = 's3://' + awsbucket + '/' + obj.name - logging.info(' deleting ' + s3url) - if not driver.delete_object(obj): - logging.warning('Could not delete ' + s3url) - upload = True - if upload: - logging.debug(' uploading "' + file_to_upload + '"...') - extra = {'acl': 'public-read'} - if file_to_upload.endswith('.sig'): - extra['content_type'] = 'application/pgp-signature' - elif file_to_upload.endswith('.asc'): - extra['content_type'] = 'application/pgp-signature' - path = os.path.relpath(file_to_upload) - logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}') - with open(file_to_upload, 'rb') as iterator: - obj = driver.upload_object_via_stream( - iterator=iterator, - container=container, - object_name=object_name, - extra=extra, - ) + if upload: + logging.debug(' uploading "' + file_to_upload + '"...') + extra = {'acl': 'public-read'} + if file_to_upload.endswith('.sig'): + extra['content_type'] = 'application/pgp-signature' + elif file_to_upload.endswith('.asc'): + extra['content_type'] = 'application/pgp-signature' + path = os.path.relpath(file_to_upload) + logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}') + with open(file_to_upload, 'rb') as iterator: + obj = driver.upload_object_via_stream( + iterator=iterator, + container=container, + object_name=object_name, + extra=extra, + ) # delete the remnants in the bucket, they do not exist locally while objs: object_name, obj = objs.popitem()