From 467e211a23ec58d5c68b20398cd8fdd7f4c09ca1 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Sun, 31 Oct 2021 18:04:42 +0100 Subject: [PATCH 1/5] Add publishing to IPFS option --- examples/config.yml | 12 ++++++++++++ fdroidserver/common.py | 2 +- fdroidserver/deploy.py | 21 ++++++++++++++++++++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/examples/config.yml b/examples/config.yml index ce3d3b2e..ec7178c6 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -268,6 +268,18 @@ # awssecretkey: {env: awssecretkey} +# To deploy to IPFS you need the ipfs command line utility in $PATH and have a +# `ipfs daemon` running. To serve the repo permanently, you either need to keep +# the daemon running or use a pinning service. +# +# You can specify a ipnfs key per repo section. Make sure to generate the keys +# before the first run, using: `ipfs key gen `. +# +# ipfs: +# repo: fdroid +# archive: fdroid_archive + + # If you want to force 'fdroid server' to use a non-standard serverwebroot. # This will allow you to have 'serverwebroot' entries which do not end in # '/fdroid'. (Please note that some client features expect repository URLs diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 1af0fc82..7b706de0 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -467,7 +467,7 @@ def read_config(opts=None): for k, v in dictvalue.items(): new[str(k)] = v config[configname] = new - elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases'): + elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases', 'ipfs'): continue elif isinstance(dictvalue, dict): for k, v in dictvalue.items(): diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 54496f78..cfb5c705 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -328,6 +328,22 @@ def sync_from_localcopy(repo_section, local_copy_dir): push_binary_transparency(offline_copy, online_copy) +def update_ipfs(repo_section, ipns_key): + """Upload using the CLI tool ipfs.""" + logging.debug( + _('adding {section} to ipfs and publish with ipns key {key}').format( + section=repo_section, key=ipns_key + ) + ) + + final_hash = subprocess.check_output( + ['ipfs', 'add', '-r', '-Q', repo_section], text=True + ).strip() + subprocess.check_call( + ['ipfs', 'name', 'publish', '--key', ipns_key, '/ipfs/{}'.format(final_hash)] + ) + + def update_localcopy(repo_section, local_copy_dir): """Copy data from offline to the "local copy dir" filesystem. @@ -795,10 +811,11 @@ def main(): and not config.get('androidobservatory') \ and not config.get('binary_transparency_remote') \ and not config.get('virustotal_apikey') \ + and not config.get('ipfs') \ and local_copy_dir is None: logging.warning(_('No option set! Edit your config.yml to set at least one of these:') + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, ' - + 'virustotal_apikey, androidobservatory, or binary_transparency_remote') + + 'virustotal_apikey, androidobservatory, binary_transparency_remote, or ipfs') sys.exit(1) repo_sections = ['repo'] @@ -831,6 +848,8 @@ def main(): upload_to_android_observatory(repo_section) if config.get('virustotal_apikey'): upload_to_virustotal(repo_section, config.get('virustotal_apikey')) + if config.get('ipfs', {}).get(repo_section): + update_ipfs(repo_section, config.get('ipfs')[repo_section]) binary_transparency_remote = config.get('binary_transparency_remote') if binary_transparency_remote: From 258bc61b8a540b500a6b9718b174c8f3362659e6 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Thu, 25 Nov 2021 17:16:38 +0100 Subject: [PATCH 2/5] Add estuary.tech (IPFS) support --- examples/config.yml | 27 ++++++++++++++----- fdroidserver/common.py | 2 +- fdroidserver/deploy.py | 59 +++++++++++++++++++++++++++++++++++------- 3 files changed, 71 insertions(+), 17 deletions(-) diff --git a/examples/config.yml b/examples/config.yml index ec7178c6..1c6b1be2 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -272,12 +272,7 @@ # `ipfs daemon` running. To serve the repo permanently, you either need to keep # the daemon running or use a pinning service. # -# You can specify a ipnfs key per repo section. Make sure to generate the keys -# before the first run, using: `ipfs key gen `. -# -# ipfs: -# repo: fdroid -# archive: fdroid_archive +# ipfs: true # If you want to force 'fdroid server' to use a non-standard serverwebroot. @@ -303,6 +298,26 @@ # virustotal_apikey: {env: virustotal_apikey} +# If you want to upload the repo to https://estuary.tech/ +# You have to enter your profile apikey to enable the upload. +# Note that his is experimental and could be removed without warning. +# +# estuary_apikey: ESTXXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXXARY +# +# Or get it from an environment variable: +# +# estuary_apikey: {env: estuary_apikey} + + +# To publish to a permanent IPNS you need the ipfs command line utility in +# $PATH and have a `ipfs daemon` running. Make sure to generate the keys before +# the first run, using: `ipfs key gen `. +# +# ipfns: +# repo: fdroid +# archive: fdroid_archive + + # Keep a log of all generated index files in a git repo to provide a # "binary transparency" log for anyone to check the history of the # binaries that are published. This is in the form of a "git remote", diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7b706de0..1107ef27 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -467,7 +467,7 @@ def read_config(opts=None): for k, v in dictvalue.items(): new[str(k)] = v config[configname] = new - elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases', 'ipfs'): + elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases', 'ipns'): continue elif isinstance(dictvalue, dict): for k, v in dictvalue.items(): diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index cfb5c705..f12bac01 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -328,19 +328,25 @@ def sync_from_localcopy(repo_section, local_copy_dir): push_binary_transparency(offline_copy, online_copy) -def update_ipfs(repo_section, ipns_key): +def update_ipfs(repo_section): """Upload using the CLI tool ipfs.""" + logging.debug(_('adding {section} to ipfs').format(section=repo_section)) + + return subprocess.check_output( + ['ipfs', 'add', '-r', '-Q', repo_section], text=True + ).strip() + + +def update_ipns(ipfs_hash, ipns_key): + """Update ipns using the CLI tool ipfs.""" logging.debug( - _('adding {section} to ipfs and publish with ipns key {key}').format( - section=repo_section, key=ipns_key + _('publish {ipfs_hash} with ipns key {key}').format( + ipfs_hash=ipfs_hash, key=ipns_key ) ) - final_hash = subprocess.check_output( - ['ipfs', 'add', '-r', '-Q', repo_section], text=True - ).strip() subprocess.check_call( - ['ipfs', 'name', 'publish', '--key', ipns_key, '/ipfs/{}'.format(final_hash)] + ['ipfs', 'name', 'publish', '--key', ipns_key, '/ipfs/{}'.format(ipfs_hash)] ) @@ -686,6 +692,32 @@ def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash, return outputfilename +def upload_to_estuary(repo_section, apikey): + """Publish repo to https://estuary.tech/.""" + import requests + from requests_toolbelt.multipart.encoder import MultipartEncoder + + session = requests.Session() + index_hash = "" + for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)): + for name in files: + file_to_upload = os.path.join(root, name) + logging.debug(' uploading "' + file_to_upload + '"...') + data = MultipartEncoder({'data': (name, open(file_to_upload, 'rb'))}) + headers = {"Authorization": "Bearer {}".format(apikey), 'Content-Type': data.content_type} + response = session.post( + "https://shuttle-4.estuary.tech/content/add", + headers=headers, + data=data + ) + if not response.ok: + logging.error(_("Failed to upload {} to Estuary".format(file_to_upload))) + return + if name == "index-v1.jar": + index_hash = response.json()["cid"] + return index_hash + + def push_binary_transparency(git_repo_path, git_remote): """Push the binary transparency git repo to the specifed remote. @@ -812,10 +844,12 @@ def main(): and not config.get('binary_transparency_remote') \ and not config.get('virustotal_apikey') \ and not config.get('ipfs') \ + and not config.get('estuary_apikey') \ and local_copy_dir is None: logging.warning(_('No option set! Edit your config.yml to set at least one of these:') + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, ' - + 'virustotal_apikey, androidobservatory, binary_transparency_remote, or ipfs') + + 'virustotal_apikey, androidobservatory, binary_transparency_remote, ' + + 'ipfs, or estuary_apikey') sys.exit(1) repo_sections = ['repo'] @@ -831,6 +865,7 @@ def main(): repo_sections.append('unsigned') for repo_section in repo_sections: + ipfs_hash = "" if local_copy_dir is not None: if config['sync_from_local_copy_dir']: sync_from_localcopy(repo_section, local_copy_dir) @@ -848,8 +883,12 @@ def main(): upload_to_android_observatory(repo_section) if config.get('virustotal_apikey'): upload_to_virustotal(repo_section, config.get('virustotal_apikey')) - if config.get('ipfs', {}).get(repo_section): - update_ipfs(repo_section, config.get('ipfs')[repo_section]) + if config.get("ipfs"): + ipfs_hash = update_ipfs(repo_section) + if config.get("estuary_apikey"): + ipfs_hash = upload_to_estuary(repo_section, config.get("estuary_apikey")) + if ipfs_hash and config.get("ipns", {}).get(repo_section): + update_ipns(ipfs_hash, config.get("ipns")[repo_section]) binary_transparency_remote = config.get('binary_transparency_remote') if binary_transparency_remote: From 90d9e9e04579de37975e9eb3f51a21ca8470eca5 Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Wed, 31 May 2023 13:25:30 +0200 Subject: [PATCH 3/5] ipfs: Add File API reference --- fdroidserver/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index f12bac01..135ec83e 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -333,7 +333,7 @@ def update_ipfs(repo_section): logging.debug(_('adding {section} to ipfs').format(section=repo_section)) return subprocess.check_output( - ['ipfs', 'add', '-r', '-Q', repo_section], text=True + ['ipfs', 'add', '-r', '-Q', '--to-files', f"/{repo_section}", repo_section], text=True ).strip() From 92612f3565f58a10672ce58bcb2f4614c5a13dbf Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Tue, 30 May 2023 14:28:51 +0200 Subject: [PATCH 4/5] Test ipfs in CI --- .gitlab-ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9878be9..e65bd826 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -602,3 +602,31 @@ docker: fi - docker push $RELEASE_IMAGE - docker push $RELEASE_IMAGE-bullseye + +ipfs: + image: debian:testing + only: + changes: + - .gitlab-ci.yml + - fdroidserver/deploy.py + <<: *apt-template + script: + - apt-get install + ca-certificates + fdroidserver + wget + - export FDROIDSERVER=$PWD + - cd /tmp + - wget https://dist.ipfs.tech/kubo/v0.20.0/kubo_v0.20.0_linux-amd64.tar.gz + - tar -xvzf kubo_v0.20.0_linux-amd64.tar.gz + - export PATH=/tmp/kubo:$PATH + - ipfs init + - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo + - cp $FDROIDSERVER/tests/config.py $FDROIDSERVER/tests/keystore.jks /tmp/fdroid/ + - cp $FDROIDSERVER/tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/ + - cd /tmp/fdroid + - 'printf "ipfs = True\n" >> config.py' + - $FDROIDSERVER/fdroid update --verbose --create-metadata + - $FDROIDSERVER/fdroid deploy --verbose + - ipfs files ls /repo + - ipfs files ls /repo/com.politedroid_6.apk From 6b1bb9ced5bf47330448943bb9eb3acc6dc06a2a Mon Sep 17 00:00:00 2001 From: Jochen Sprickerhof Date: Tue, 6 Jun 2023 14:17:04 +0200 Subject: [PATCH 5/5] Make ipfs path configurable --- .gitlab-ci.yml | 9 ++++----- examples/config.yml | 8 ++++---- fdroidserver/deploy.py | 20 ++++++++++++++++++-- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e65bd826..0e44e9ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -619,14 +619,13 @@ ipfs: - cd /tmp - wget https://dist.ipfs.tech/kubo/v0.20.0/kubo_v0.20.0_linux-amd64.tar.gz - tar -xvzf kubo_v0.20.0_linux-amd64.tar.gz - - export PATH=/tmp/kubo:$PATH - - ipfs init + - /tmp/kubo/ipfs init - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo - cp $FDROIDSERVER/tests/config.py $FDROIDSERVER/tests/keystore.jks /tmp/fdroid/ - cp $FDROIDSERVER/tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/ - cd /tmp/fdroid - - 'printf "ipfs = True\n" >> config.py' + - 'echo ipfs = \"/tmp/kubo/ipfs\" >> config.py' - $FDROIDSERVER/fdroid update --verbose --create-metadata - $FDROIDSERVER/fdroid deploy --verbose - - ipfs files ls /repo - - ipfs files ls /repo/com.politedroid_6.apk + - /tmp/kubo/ipfs files ls /repo + - /tmp/kubo/ipfs files ls /repo/com.politedroid_6.apk diff --git a/examples/config.yml b/examples/config.yml index 1c6b1be2..1bb974cd 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -268,11 +268,11 @@ # awssecretkey: {env: awssecretkey} -# To deploy to IPFS you need the ipfs command line utility in $PATH and have a -# `ipfs daemon` running. To serve the repo permanently, you either need to keep -# the daemon running or use a pinning service. +# To deploy to IPFS you need the ipfs command line utility and specify +# the path as an argument. To serve the repo permanently, you either need to +# keep the daemon running or use a pinning service. # -# ipfs: true +# ipfs: /path/to/ipfs # If you want to force 'fdroid server' to use a non-standard serverwebroot. diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 135ec83e..82c4ddde 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -333,7 +333,16 @@ def update_ipfs(repo_section): logging.debug(_('adding {section} to ipfs').format(section=repo_section)) return subprocess.check_output( - ['ipfs', 'add', '-r', '-Q', '--to-files', f"/{repo_section}", repo_section], text=True + [ + config.get("ipfs"), + 'add', + '-r', + '-Q', + '--to-files', + f"/{repo_section}", + repo_section, + ], + text=True, ).strip() @@ -346,7 +355,14 @@ def update_ipns(ipfs_hash, ipns_key): ) subprocess.check_call( - ['ipfs', 'name', 'publish', '--key', ipns_key, '/ipfs/{}'.format(ipfs_hash)] + [ + config.get("ipfs"), + 'name', + 'publish', + '--key', + ipns_key, + '/ipfs/{}'.format(ipfs_hash), + ] )