diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 866a2ec8..229d4e4f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -663,3 +663,30 @@ docker: fi - docker push $RELEASE_IMAGE - docker push $RELEASE_IMAGE-bullseye + +ipfs: + image: debian:testing + only: + changes: + - .gitlab-ci.yml + - fdroidserver/deploy.py + <<: *apt-template + script: + - apt-get install + ca-certificates + fdroidserver + wget + - export FDROIDSERVER=$PWD + - cd /tmp + - wget https://dist.ipfs.tech/kubo/v0.20.0/kubo_v0.20.0_linux-amd64.tar.gz + - tar -xvzf kubo_v0.20.0_linux-amd64.tar.gz + - /tmp/kubo/ipfs init + - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo + - cp $FDROIDSERVER/tests/config.py $FDROIDSERVER/tests/keystore.jks /tmp/fdroid/ + - cp $FDROIDSERVER/tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/ + - cd /tmp/fdroid + - 'echo ipfs = \"/tmp/kubo/ipfs\" >> config.py' + - $FDROIDSERVER/fdroid update --verbose --create-metadata + - $FDROIDSERVER/fdroid deploy --verbose + - /tmp/kubo/ipfs files ls /repo + - /tmp/kubo/ipfs files ls /repo/com.politedroid_6.apk diff --git a/examples/config.yml b/examples/config.yml index b094a032..c5a49cfb 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -277,6 +277,13 @@ # awssecretkey: {env: awssecretkey} +# To deploy to IPFS you need the ipfs command line utility and specify +# the path as an argument. To serve the repo permanently, you either need to +# keep the daemon running or use a pinning service. +# +# ipfs: /path/to/ipfs + + # If you want to force 'fdroid server' to use a non-standard serverwebroot. # This will allow you to have 'serverwebroot' entries which do not end in # '/fdroid'. (Please note that some client features expect repository URLs @@ -300,6 +307,26 @@ # virustotal_apikey: {env: virustotal_apikey} +# If you want to upload the repo to https://estuary.tech/ +# You have to enter your profile apikey to enable the upload. +# Note that his is experimental and could be removed without warning. +# +# estuary_apikey: ESTXXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXXARY +# +# Or get it from an environment variable: +# +# estuary_apikey: {env: estuary_apikey} + + +# To publish to a permanent IPNS you need the ipfs command line utility in +# $PATH and have a `ipfs daemon` running. Make sure to generate the keys before +# the first run, using: `ipfs key gen `. +# +# ipfns: +# repo: fdroid +# archive: fdroid_archive + + # Keep a log of all generated index files in a git repo to provide a # "binary transparency" log for anyone to check the history of the # binaries that are published. This is in the form of a "git remote", diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 3da0a193..db58a610 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -507,7 +507,7 @@ def read_config(opts=None): for k, v in dictvalue.items(): new[str(k)] = v config[configname] = new - elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases'): + elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases', 'ipns'): continue elif isinstance(dictvalue, dict): for k, v in dictvalue.items(): diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 92287f1b..338e9a19 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -363,6 +363,44 @@ def sync_from_localcopy(repo_section, local_copy_dir): push_binary_transparency(offline_copy, online_copy) +def update_ipfs(repo_section): + """Upload using the CLI tool ipfs.""" + logging.debug(_('adding {section} to ipfs').format(section=repo_section)) + + return subprocess.check_output( + [ + config.get("ipfs"), + 'add', + '-r', + '-Q', + '--to-files', + f"/{repo_section}", + repo_section, + ], + text=True, + ).strip() + + +def update_ipns(ipfs_hash, ipns_key): + """Update ipns using the CLI tool ipfs.""" + logging.debug( + _('publish {ipfs_hash} with ipns key {key}').format( + ipfs_hash=ipfs_hash, key=ipns_key + ) + ) + + subprocess.check_call( + [ + config.get("ipfs"), + 'name', + 'publish', + '--key', + ipns_key, + '/ipfs/{}'.format(ipfs_hash), + ] + ) + + def update_localcopy(repo_section, local_copy_dir): """Copy data from offline to the "local copy dir" filesystem. @@ -705,6 +743,32 @@ def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash, return outputfilename +def upload_to_estuary(repo_section, apikey): + """Publish repo to https://estuary.tech/.""" + import requests + from requests_toolbelt.multipart.encoder import MultipartEncoder + + session = requests.Session() + index_hash = "" + for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)): + for name in files: + file_to_upload = os.path.join(root, name) + logging.debug(' uploading "' + file_to_upload + '"...') + data = MultipartEncoder({'data': (name, open(file_to_upload, 'rb'))}) + headers = {"Authorization": "Bearer {}".format(apikey), 'Content-Type': data.content_type} + response = session.post( + "https://shuttle-4.estuary.tech/content/add", + headers=headers, + data=data + ) + if not response.ok: + logging.error(_("Failed to upload {} to Estuary".format(file_to_upload))) + return + if name == "index-v1.jar": + index_hash = response.json()["cid"] + return index_hash + + def push_binary_transparency(git_repo_path, git_remote): """Push the binary transparency git repo to the specifed remote. @@ -812,10 +876,13 @@ def main(): and not config.get('androidobservatory') \ and not config.get('binary_transparency_remote') \ and not config.get('virustotal_apikey') \ + and not config.get('ipfs') \ + and not config.get('estuary_apikey') \ and local_copy_dir is None: logging.warning(_('No option set! Edit your config.yml to set at least one of these:') + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, ' - + 'virustotal_apikey, androidobservatory, or binary_transparency_remote') + + 'virustotal_apikey, androidobservatory, binary_transparency_remote, ' + + 'ipfs, or estuary_apikey') sys.exit(1) repo_sections = ['repo'] @@ -831,6 +898,7 @@ def main(): repo_sections.append('unsigned') for repo_section in repo_sections: + ipfs_hash = "" if local_copy_dir is not None: if config['sync_from_local_copy_dir']: sync_from_localcopy(repo_section, local_copy_dir) @@ -850,6 +918,12 @@ def main(): upload_to_android_observatory(repo_section) if config.get('virustotal_apikey'): upload_to_virustotal(repo_section, config.get('virustotal_apikey')) + if config.get("ipfs"): + ipfs_hash = update_ipfs(repo_section) + if config.get("estuary_apikey"): + ipfs_hash = upload_to_estuary(repo_section, config.get("estuary_apikey")) + if ipfs_hash and config.get("ipns", {}).get(repo_section): + update_ipns(ipfs_hash, config.get("ipns")[repo_section]) binary_transparency_remote = config.get('binary_transparency_remote') if binary_transparency_remote: