fdroidserver/deploy.py

@@ -16,6 +16,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from math import log
 import sys
 import glob
 import hashlib
@@ -94,7 +95,7 @@ def _get_index_includes(base_dir):
     return index_includes
 
 
-def update_awsbucket(repo_section, is_index_only=False):
+def update_awsbucket(repo_section, index_only=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
     The contents of that subdir of the
@@ -106,12 +107,12 @@ def update_awsbucket(repo_section, is_index_only=False):
                   + config['awsbucket'] + '"')
 
     if common.set_command_in_config('s3cmd'):
-        update_awsbucket_s3cmd(repo_section, is_index_only)
+        update_awsbucket_s3cmd(repo_section, index_only)
     else:
-        update_awsbucket_libcloud(repo_section, is_index_only)
+        update_awsbucket_libcloud(repo_section, index_only)
 
 
-def update_awsbucket_s3cmd(repo_section, is_index_only=False):
+def update_awsbucket_s3cmd(repo_section, index_only=False):
     """Upload using the CLI tool s3cmd, which provides rsync-like sync.
 
     The upload is done in multiple passes to reduce the chance of
@@ -155,7 +156,7 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
 
     s3url = s3bucketurl + '/fdroid/'
 
-    if is_index_only:
+    if index_only:
         logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
                       .format(path=repo_section, url=s3url))
         sync_indexes_flags = []
@@ -199,7 +200,7 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
             raise FDroidException()
 
 
-def update_awsbucket_libcloud(repo_section, is_index_only=False):
+def update_awsbucket_libcloud(repo_section, index_only=False):
     """No summary.
 
     Upload the contents of the directory `repo_section` (including
@@ -242,7 +243,7 @@ def update_awsbucket_libcloud(repo_section, is_index_only=False):
         if obj.name.startswith(upload_dir + '/'):
             objs[obj.name] = obj
 
-    if is_index_only:
+    if index_only:
         index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
         files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
         files_to_upload = list(set(files_to_upload) & set(index_files))
@@ -335,9 +336,9 @@ def update_serverwebroot(serverwebroot, repo_section):
     elif config and config.get('identity_file'):
         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
     url = serverwebroot['url']
-    is_index_only = serverwebroot.get('index_only', False)
+    index_only = serverwebroot.get('index_only', False)
     logging.info('rsyncing ' + repo_section + ' to ' + url)
-    if is_index_only:
+    if index_only:
         rsyncargs += _get_index_file_paths(repo_section)
         rsyncargs += [f'{url}/{repo_section}/']
         logging.info(rsyncargs)
@@ -457,11 +458,11 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         logging.debug(_('Offline machine, skipping git mirror generation until `fdroid deploy`'))
         return
 
-    workspace_dir = Path(os.getcwd())
+    working_dir_root = Path(os.getcwd())
 
     # right now we support only 'repo' git-mirroring
     if repo_section == 'repo':
-        git_mirror_path = workspace_dir / 'git-mirror'
+        git_mirror_path = working_dir_root / 'git-mirror'
         dotgit = os.path.join(git_mirror_path, '.git')
         git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
         git_repodir = os.path.join(git_fdroiddir, repo_section)
@@ -519,9 +520,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             # trailing slashes have a meaning in rsync which is not needed here, so
             # make sure both paths have exactly one trailing slash
             if is_index_only:
-                files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES]
+                files_to_sync = [str(working_dir_root / repo_section / index_file) for index_file in INDEX_FILES]
             else:
-                files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
+                files_to_sync = [str(working_dir_root / repo_section).rstrip('/') + '/']
             common.local_rsync(options, files_to_sync, git_repodir.rstrip('/') + '/')
 
             upload_to_servergitmirror(mirror_config=d,
@@ -533,6 +534,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                                       git_mirror_path=str(git_mirror_path),
                                       ssh_cmd=ssh_cmd,
                                       progress=progress)
+
+            # Switch to the initial branch
+            repo.git.switch(GIT_BRANCH)
         if progress:
             progressbar.done()
 
@@ -546,6 +550,12 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
                               git_mirror_path: str,
                               ssh_cmd: str,
                               progress: git.RemoteProgress) -> None:
+    # Test
+    logging.info(f"mirror_config: {mirror_config}")
+    # List all files in the git repo
+    logging.info("files:")
+    logging.info(local_repo.git.ls_files())
+
     remote_branch_name = GIT_BRANCH
     local_branch_name = local_repo.active_branch.name
 
@@ -563,6 +573,8 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
 
     logging.info("git status:", local_repo.git.status())
     if is_index_only:
+        # test
+        logging.debug('Adding index files to git mirror')
         local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
     else:
         # sadly index.add don't allow the --all parameter
@@ -572,6 +584,10 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     logging.debug('Committing files into git mirror')
     local_repo.index.commit("fdroidserver git-mirror")
 
+    # Test
+    logging.info(f"In index-only: {is_index_only} mode")
+    logging.info(local_repo.git.status())
+    logging.info(local_repo.head.log())
 
     # only deploy to GitLab Artifacts if too big for GitLab Pages
     if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
@@ -609,7 +625,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
         local_repo.index.add(['.gitlab-ci.yml'])
         local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
 
-    logging.info(_('Pushing to {url}').format(url=remote.url))
+    logging.debug(_('Pushing to {url}').format(url=remote.url))
     with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
         pushinfos = remote.push(
             f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress
@@ -859,6 +875,12 @@ def main():
                         help=_("Don't use rsync checksums"))
     parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
                         help=_("If a git mirror gets to big, allow the archive to be deleted"))
+    parser.add_argument(
+        "--index-only",
+        action="store_true",
+        default=False,
+        help="Only deploy the index files entry.* and index-v*.json",
+    )
     options = parser.parse_args()
     config = common.read_config(options)
 

tests/deploy.TestCase

@@ -6,6 +6,7 @@ import optparse
 import os
 import sys
 import tempfile
+from unicodedata import mirrored
 import unittest
 from pathlib import Path
 from unittest import mock