Merge branch 'add-local_copy_dir-and-v0.2-fixes' of https://gitlab.com/eighthave/fdroidserver

MANIFEST.in

@@ -23,6 +23,10 @@ include docs/update.sh
 include examples/config.py
 include examples/fdroid-icon.png
 include examples/makebs.config.py
+include examples/opensc-fdroid.cfg
+include fdroidserver/getsig/run.sh
+include fdroidserver/getsig/make.sh
+include fdroidserver/getsig/getsig.java
 include tests/run-tests
 include wp-fdroid/AndroidManifest.xml
 include wp-fdroid/android-permissions.php

examples/config.py

@@ -98,28 +98,56 @@ keyaliases['com.example.app'] = 'example'
 # the @ prefix.
 keyaliases['com.example.another.plugin'] = '@com.example.another'
 
+
 # The full path to the root of the repository.  It must be specified in
 # rsync/ssh format for a remote host/path. This is used for syncing a locally
 # generated repo to the server that is it hosted on.  It must end in the
 # standard public repo name of "/fdroid", but can be in up to three levels of
 # sub-directories (i.e. /var/www/packagerepos/fdroid).
+#
 # serverwebroot = 'user@example:/var/www/fdroid'
 
+
 # optionally specific which identity file to use when using rsync over SSH
+#
 # identity_file = '~/.ssh/fdroid_id_rsa'
 
+
+# If you are running the repo signing process on a completely offline machine,
+# which provides the best security, then you can specify a folder to sync the
+# repo to when running `fdroid server update`.  This is most likely going to
+# be a USB thumb drive, SD Card, or some other kind of removable media.  Make
+# sure it is mounted before running `fdroid server update`.  Using the
+# standard folder called 'fdroid' as the specified folder is recommended, like
+# with serverwebroot.
+#
+# local_copy_dir = '/media/MyUSBThumbDrive/fdroid'
+
+
+# If you are using local_copy_dir on an offline build/signing server, once the
+# thumb drive has been plugged into the online machine, it will need to be
+# synced to the copy on the online machine.  To make that happen
+# automatically, set sync_from_local_copy_dir to True:
+#
+# sync_from_local_copy_dir = True
+
+
 # To upload the repo to an Amazon S3 bucket using `fdroid server update`.
 # Warning, this deletes and recreates the whole fdroid/ directory each
 # time. This is based on apache-libcloud, which supports basically all cloud
 # storage services, so it should be easy to port the fdroid server tools to
 # any of them.
+#
 # awsbucket = 'myawsfdroid'
 # awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA'
 # awssecretkey = 'yourverysecretkeywordpassphraserighthere'
 
+
 # If you want to force 'fdroid server' to use a non-standard serverwebroot
+#
 # nonstandardwebroot = False
 
+
 # Wiki details
 wiki_protocol = "http"
 wiki_server = "server"

fdroidserver/common.py

@@ -47,6 +47,7 @@ def get_default_config():
         'mvn3': "mvn",
         'gradle': 'gradle',
         'archive_older': 0,
+        'sync_from_local_copy_dir': False,
         'update_stats': False,
         'stats_to_carbon': False,
         'repo_maxage': 0,

fdroidserver/server.py

@@ -115,7 +115,7 @@ def update_awsbucket(repo_section):
 
 
 def update_serverwebroot(repo_section):
-    rsyncargs = ['rsync', '--update', '--recursive', '--delete']
+    rsyncargs = ['rsync', '--archive', '--delete']
     if options.verbose:
         rsyncargs += ['--verbose']
     if options.quiet:
@@ -140,6 +140,33 @@ def update_serverwebroot(repo_section):
         sys.exit(1)
 
 
+def _local_sync(fromdir, todir):
+    rsyncargs = ['rsync', '--archive', '--one-file-system', '--delete']
+    # use stricter rsync checking on all files since people using offline mode
+    # are already prioritizing security above ease and speed
+    rsyncargs += ['--checksum']
+    if options.verbose:
+        rsyncargs += ['--verbose']
+    if options.quiet:
+        rsyncargs += ['--quiet']
+    logging.debug(' '.join(rsyncargs + [fromdir, todir]))
+    if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
+        sys.exit(1)
+
+
+def sync_from_localcopy(repo_section, local_copy_dir):
+    logging.info('Syncing from local_copy_dir to this repo.')
+    # trailing slashes have a meaning in rsync which is not needed here, so
+    # remove them all
+    _local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/'),
+                repo_section.rstrip('/'))
+
+
+def update_localcopy(repo_section, local_copy_dir):
+    # local_copy_dir is guaranteed to have a trailing slash in main() below
+    _local_sync(repo_section, local_copy_dir)
+
+
 def main():
     global config, options
 
@@ -147,6 +174,10 @@ def main():
     parser = OptionParser()
     parser.add_option("-i", "--identity-file", default=None,
                       help="Specify an identity file to provide to SSH for rsyncing")
+    parser.add_option("--local-copy-dir", default=None,
+                      help="Specify a local folder to sync the repo to")
+    parser.add_option("--sync-from-local-copy-dir", action="store_true", default=False,
+                      help="Before uploading to servers, sync from local copy dir")
     parser.add_option("-v", "--verbose", action="store_true", default=False,
                       help="Spew out even more information than normal")
     parser.add_option("-q", "--quiet", action="store_true", default=False,
@@ -171,20 +202,56 @@ def main():
     if config.get('serverwebroot'):
         serverwebroot = config['serverwebroot']
         host, fdroiddir = serverwebroot.rstrip('/').split(':')
-        serverrepobase = os.path.basename(fdroiddir)
-        if serverrepobase != 'fdroid' and standardwebroot:
+        repobase = os.path.basename(fdroiddir)
+        if standardwebroot and repobase != 'fdroid':
             logging.error('serverwebroot does not end with "fdroid", '
                           + 'perhaps you meant one of these:\n\t'
                           + serverwebroot.rstrip('/') + '/fdroid\n\t'
-                          + serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid')
+                          + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
             sys.exit(1)
-    elif not config.get('awsbucket'):
-        logging.warn('No serverwebroot or awsbucket set! Edit your config.py to set one or both.')
+
+    if options.local_copy_dir is not None:
+        local_copy_dir = options.local_copy_dir
+    elif config.get('local_copy_dir'):
+        local_copy_dir = config['local_copy_dir']
+    else:
+        local_copy_dir = None
+    if local_copy_dir is not None:
+        fdroiddir = local_copy_dir.rstrip('/')
+        if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir):
+            logging.error('local_copy_dir must be directory, not a file!')
+            sys.exit(1)
+        if not os.path.exists(os.path.dirname(fdroiddir)):
+            logging.error('The root dir for local_copy_dir "'
+                          + os.path.dirname(fdroiddir)
+                          + '" does not exist!')
+            sys.exit(1)
+        if not os.path.isabs(fdroiddir):
+            logging.error('local_copy_dir must be an absolute path!')
+            sys.exit(1)
+        repobase = os.path.basename(fdroiddir)
+        if standardwebroot and repobase != 'fdroid':
+            logging.error('local_copy_dir does not end with "fdroid", '
+                          + 'perhaps you meant: ' + fdroiddir + '/fdroid')
+            sys.exit(1)
+        if local_copy_dir[-1] != '/':
+            local_copy_dir += '/'
+        local_copy_dir = local_copy_dir.replace('//', '/')
+        if not os.path.exists(fdroiddir):
+            os.mkdir(fdroiddir)
+
+    if not config.get('awsbucket') \
+            and not config.get('serverwebroot') \
+            and local_copy_dir is None:
+        logging.warn('No serverwebroot, local_copy_dir, or awsbucket set!'
+                     + 'Edit your config.py to set at least one.')
         sys.exit(1)
 
     repo_sections = ['repo']
     if config['archive_older'] != 0:
         repo_sections.append('archive')
+        if not os.path.exists('archive'):
+            os.mkdir('archive')
 
     if args[0] == 'init':
         if config.get('serverwebroot'):
@@ -201,6 +268,11 @@ def main():
                     sys.exit(1)
     elif args[0] == 'update':
         for repo_section in repo_sections:
+            if local_copy_dir is not None:
+                if config['sync_from_local_copy_dir'] and os.path.exists(repo_section):
+                    sync_from_localcopy(repo_section, local_copy_dir)
+                else:
+                    update_localcopy(repo_section, local_copy_dir)
             if config.get('serverwebroot'):
                 update_serverwebroot(repo_section)
             if config.get('awsbucket'):

setup.py

@@ -10,7 +10,7 @@ if not os.path.exists('fdroidserver/getsig/getsig.class'):
                             shell=True)
 
 setup(name='fdroidserver',
-      version='0.1',
+      version='0.2',
       description='F-Droid Server Tools',
       long_description=open('README').read(),
       author='The F-Droid Project',

tests/run-tests

@@ -71,7 +71,42 @@ if [ -z $aapt ]; then
 fi
 
 #------------------------------------------------------------------------------#
-echo_header "setup a new repo from scratch using ANDROID_HOME"
+echo_header "test config checks of local_copy_dir"
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+$fdroid init
+$fdroid update --create-metadata
+$fdroid server update --local-copy-dir=/tmp/fdroid
+
+# now test the errors work
+set +e
+$fdroid server update --local-copy-dir=thisisnotanabsolutepath
+if [ $? -eq 0 ]; then
+    echo "This should have failed because thisisnotanabsolutepath is not an absolute path!"
+    exit 1
+else
+    echo "testing absolute path checker passed"
+fi
+$fdroid server update --local-copy-dir=/tmp/IReallyDoubtThisPathExistsasdfasdf
+if [ $? -eq 0 ]; then
+    echo "This should have failed because the path does not end with 'fdroid'!"
+    exit 1
+else
+    echo "testing dirname exists checker passed"
+fi
+$fdroid server update --local-copy-dir=/tmp/IReallyDoubtThisPathExistsasdfasdf/fdroid
+if [ $? -eq 0 ]; then
+    echo "This should have failed because the dirname path does not exist!"
+    exit 1
+else
+    echo "testing dirname exists checker passed"
+fi
+set -e
+
+
+#------------------------------------------------------------------------------#
+echo_header "setup a new repo from scratch using ANDROID_HOME and do a local sync"
 
 REPOROOT=`create_test_dir`
 cd $REPOROOT
@@ -80,6 +115,13 @@ copy_apks_into_repo $REPOROOT
 $fdroid update --create-metadata
 grep -F '<application id=' repo/index.xml
 
+LOCALCOPYDIR=`create_test_dir`/fdroid
+$fdroid server update --local-copy-dir=$LOCALCOPYDIR
+NEWREPOROOT=`create_test_dir`
+cd $NEWREPOROOT
+$fdroid init
+$fdroid server update --local-copy-dir=$LOCALCOPYDIR --sync-from-local-copy-dir
+
 
 #------------------------------------------------------------------------------#
 # check that --android-home fails when dir does not exist or is not a dir