index: fix GitLab Raw URLs with gitlab.com and recent versions

GitLab seems to be moving to always having "-" as the first path segment in
all the project URLs.  So the URL without a "-" is now a redirect.

.gitlab-ci.yml

@@ -291,3 +291,40 @@ fdroid build:
     # each `fdroid build --on-server` run expects sudo, then uninstalls it
     - apt-get install sudo
     - fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid
+
+
+# test a full update and deploy cycle to gitlab.com
+servergitmirrors:
+  image: debian
+  <<: *apt-template
+  only:
+    - master@fdroid/fdroidserver
+  script:
+    - apt-get install
+        default-jdk-headless
+        git
+        openssh-client
+        openssl
+        python3-pip
+        python3-venv
+        rsync
+        wget
+    - python3 -m venv env
+    - . env/bin/activate
+    - export PYTHONPATH=`pwd`
+    - $pip install -e .
+    - mkdir /root/.ssh/
+    - ./tests/key-tricks.py
+    - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
+    - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
+    - cp tests/config.py tests/keystore.jks /tmp/fdroid/
+    - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
+    - cd /tmp/fdroid
+    - touch fdroid-icon.png
+    - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
+    - $PYTHONPATH/fdroid update --verbose --create-metadata
+    - $PYTHONPATH/fdroid deploy --verbose
+    - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
+    - echo $DLURL
+    - wget $DLURL/index-v1.jar
+    - diff repo/index-v1.jar index-v1.jar

fdroidserver/index.py

@@ -648,7 +648,7 @@ def get_mirror_service_urls(url):
     '''
 
     if url.startswith('git@'):
-        url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
+        url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url)
 
     segments = url.split("/")
 
@@ -676,10 +676,9 @@ def get_mirror_service_urls(url):
         # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
         gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
         urls.append('/'.join(gitlab_pages))
-        # Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
-        gitlab_raw = segments + ['raw', branch, folder]
+        # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder"
+        gitlab_raw = segments + ['-', 'raw', branch, folder]
         urls.append('/'.join(gitlab_raw))
-        return urls
 
     return urls
 

tests/index.TestCase

@@ -247,6 +247,26 @@ class IndexTest(unittest.TestCase):
                                                     fdroidserver.common.default_config['repo_icon'])))
         self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
 
+    def test_get_mirror_service_urls(self):
+        for url in [
+                'git@github.com:foo/bar',
+                'git@github.com:foo/bar.git',
+                'https://github.com/foo/bar',
+                'https://github.com/foo/bar.git',
+        ]:
+            self.assertEqual(['https://raw.githubusercontent.com/foo/bar/master/fdroid'],
+                             fdroidserver.index.get_mirror_service_urls(url))
+
+        for url in [
+                'git@gitlab.com:group/project',
+                'git@gitlab.com:group/project.git',
+                'https://gitlab.com/group/project',
+                'https://gitlab.com/group/project.git',
+        ]:
+            self.assertEqual(['https://group.gitlab.io/project/fdroid',
+                              'https://gitlab.com/group/project/-/raw/master/fdroid'],
+                             fdroidserver.index.get_mirror_service_urls(url))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/key-tricks.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+import os
+import fdroidserver
+import shutil
+import sys
+from fdroidserver import common, nightly
+
+if os.getenv('CI') is None:
+    print('ERROR: This can overwrite SSH keys, so it should only be run in CI')
+    sys.exit(1)
+
+os.chdir(os.path.dirname(__file__))
+config = fdroidserver.common.read_config(common.options)
+nightly.PASSWORD = config['keystorepass']
+nightly.KEY_ALIAS = config['repo_keyalias']
+
+privkey = nightly._ssh_key_from_debug_keystore('keystore.jks')
+print('privkey', privkey)
+ssh_private_key_file = os.path.join(os.getenv('HOME'), '.ssh', 'id_rsa')
+if os.path.exists(ssh_private_key_file):
+    print('ERROR:', ssh_private_key_file, 'exists!')
+    sys.exit(1)
+shutil.move(privkey, ssh_private_key_file)
+shutil.move(privkey + '.pub', ssh_private_key_file + '.pub')