Merge branch 'ipfs' into 'master'

Draft: Add publishing to IPFS option

See merge request fdroid/fdroidserver!1041

.gitlab-ci.yml

@@ -663,3 +663,30 @@ docker:
       fi
     - docker push $RELEASE_IMAGE
     - docker push $RELEASE_IMAGE-bullseye
+
+ipfs:
+  image: debian:testing
+  only:
+    changes:
+      - .gitlab-ci.yml
+      - fdroidserver/deploy.py
+  <<: *apt-template
+  script:
+    - apt-get install
+        ca-certificates
+        fdroidserver
+        wget
+    - export FDROIDSERVER=$PWD
+    - cd /tmp
+    - wget https://dist.ipfs.tech/kubo/v0.20.0/kubo_v0.20.0_linux-amd64.tar.gz
+    - tar -xvzf kubo_v0.20.0_linux-amd64.tar.gz
+    - /tmp/kubo/ipfs init
+    - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
+    - cp $FDROIDSERVER/tests/config.py $FDROIDSERVER/tests/keystore.jks /tmp/fdroid/
+    - cp $FDROIDSERVER/tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
+    - cd /tmp/fdroid
+    - 'echo ipfs = \"/tmp/kubo/ipfs\" >> config.py'
+    - $FDROIDSERVER/fdroid update --verbose --create-metadata
+    - $FDROIDSERVER/fdroid deploy --verbose
+    - /tmp/kubo/ipfs files ls /repo
+    - /tmp/kubo/ipfs files ls /repo/com.politedroid_6.apk

examples/config.yml

@@ -277,6 +277,13 @@
 # awssecretkey: {env: awssecretkey}
 
 
+# To deploy to IPFS you need the ipfs command line utility and specify
+# the path as an argument. To serve the repo permanently, you either need to
+# keep the daemon running or use a pinning service.
+#
+# ipfs: /path/to/ipfs
+
+
 # If you want to force 'fdroid server' to use a non-standard serverwebroot.
 # This will allow you to have 'serverwebroot' entries which do not end in
 # '/fdroid'. (Please note that some client features expect repository URLs
@@ -300,6 +307,26 @@
 # virustotal_apikey: {env: virustotal_apikey}
 
 
+# If you want to upload the repo to  https://estuary.tech/
+# You have to enter your profile apikey to enable the upload.
+# Note that his is experimental and could be removed without warning.
+#
+# estuary_apikey: ESTXXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXXARY
+#
+# Or get it from an environment variable:
+#
+# estuary_apikey: {env: estuary_apikey}
+
+
+# To publish to a permanent IPNS you need the ipfs command line utility in
+# $PATH and have a `ipfs daemon` running. Make sure to generate the keys before
+# the first run, using: `ipfs key gen <key>`.
+#
+# ipfns:
+#   repo: fdroid
+#   archive: fdroid_archive
+
+
 # Keep a log of all generated index files in a git repo to provide a
 # "binary transparency" log for anyone to check the history of the
 # binaries that are published.  This is in the form of a "git remote",

fdroidserver/common.py

@@ -507,7 +507,7 @@ def read_config(opts=None):
             for k, v in dictvalue.items():
                 new[str(k)] = v
             config[configname] = new
-        elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases'):
+        elif configname in ('ndk_paths', 'java_paths', 'char_limits', 'keyaliases', 'ipns'):
             continue
         elif isinstance(dictvalue, dict):
             for k, v in dictvalue.items():

fdroidserver/deploy.py

@@ -363,6 +363,44 @@ def sync_from_localcopy(repo_section, local_copy_dir):
         push_binary_transparency(offline_copy, online_copy)
 
 
+def update_ipfs(repo_section):
+    """Upload using the CLI tool ipfs."""
+    logging.debug(_('adding {section} to ipfs').format(section=repo_section))
+
+    return subprocess.check_output(
+        [
+            config.get("ipfs"),
+            'add',
+            '-r',
+            '-Q',
+            '--to-files',
+            f"/{repo_section}",
+            repo_section,
+        ],
+        text=True,
+    ).strip()
+
+
+def update_ipns(ipfs_hash, ipns_key):
+    """Update ipns using the CLI tool ipfs."""
+    logging.debug(
+        _('publish {ipfs_hash} with ipns key {key}').format(
+            ipfs_hash=ipfs_hash, key=ipns_key
+        )
+    )
+
+    subprocess.check_call(
+        [
+            config.get("ipfs"),
+            'name',
+            'publish',
+            '--key',
+            ipns_key,
+            '/ipfs/{}'.format(ipfs_hash),
+        ]
+    )
+
+
 def update_localcopy(repo_section, local_copy_dir):
     """Copy data from offline to the "local copy dir" filesystem.
 
@@ -705,6 +743,32 @@ def upload_apk_to_virustotal(virustotal_apikey, packageName, apkName, hash,
     return outputfilename
 
 
+def upload_to_estuary(repo_section, apikey):
+    """Publish repo to https://estuary.tech/."""
+    import requests
+    from requests_toolbelt.multipart.encoder import MultipartEncoder
+
+    session = requests.Session()
+    index_hash = ""
+    for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)):
+        for name in files:
+            file_to_upload = os.path.join(root, name)
+            logging.debug(' uploading "' + file_to_upload + '"...')
+            data = MultipartEncoder({'data': (name, open(file_to_upload, 'rb'))})
+            headers = {"Authorization": "Bearer {}".format(apikey), 'Content-Type': data.content_type}
+            response = session.post(
+                "https://shuttle-4.estuary.tech/content/add",
+                headers=headers,
+                data=data
+            )
+            if not response.ok:
+                logging.error(_("Failed to upload {} to Estuary".format(file_to_upload)))
+                return
+            if name == "index-v1.jar":
+                index_hash = response.json()["cid"]
+    return index_hash
+
+
 def push_binary_transparency(git_repo_path, git_remote):
     """Push the binary transparency git repo to the specifed remote.
 
@@ -812,10 +876,13 @@ def main():
             and not config.get('androidobservatory') \
             and not config.get('binary_transparency_remote') \
             and not config.get('virustotal_apikey') \
+            and not config.get('ipfs') \
+            and not config.get('estuary_apikey') \
             and local_copy_dir is None:
         logging.warning(_('No option set! Edit your config.yml to set at least one of these:')
                         + '\nserverwebroot, servergitmirrors, local_copy_dir, awsbucket, '
-                        + 'virustotal_apikey, androidobservatory, or binary_transparency_remote')
+                        + 'virustotal_apikey, androidobservatory, binary_transparency_remote, '
+                        + 'ipfs, or estuary_apikey')
         sys.exit(1)
 
     repo_sections = ['repo']
@@ -831,6 +898,7 @@ def main():
         repo_sections.append('unsigned')
 
     for repo_section in repo_sections:
+        ipfs_hash = ""
         if local_copy_dir is not None:
             if config['sync_from_local_copy_dir']:
                 sync_from_localcopy(repo_section, local_copy_dir)
@@ -850,6 +918,12 @@ def main():
             upload_to_android_observatory(repo_section)
         if config.get('virustotal_apikey'):
             upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
+        if config.get("ipfs"):
+            ipfs_hash = update_ipfs(repo_section)
+        if config.get("estuary_apikey"):
+            ipfs_hash = upload_to_estuary(repo_section, config.get("estuary_apikey"))
+        if ipfs_hash and config.get("ipns", {}).get(repo_section):
+            update_ipns(ipfs_hash, config.get("ipns")[repo_section])
 
     binary_transparency_remote = config.get('binary_transparency_remote')
     if binary_transparency_remote:

fdroidserver/scanner.py

@@ -825,6 +825,8 @@ def main():
     if not appids:
         if options.exit_code and probcount > 0:
             sys.exit(ExitCode.NONFREE_CODE)
+        if options.refresh_scanner:
+            _get_tool()
         return
 
     # Read all app and srclib metadata