Merge branch 'support-multiple-serverwebroots-and-fixes' into 'master'

Support multiple serverwebroots and fixes

This adds support for lists of serverwebroots in config.py, and cleans newlines and spaces in the repo_description and archive_description.

examples/config.py

@@ -33,11 +33,12 @@ repo_maxage = 0
 repo_url = "https://MyFirstFDroidRepo.org/fdroid/repo"
 repo_name = "My First FDroid Repo Demo"
 repo_icon = "fdroid-icon.png"
-repo_description = (
+repo_description = """
-    "This is a repository of apps to be used with FDroid. Applications in this "
+This is a repository of apps to be used with FDroid. Applications in this
-    + "repository are either official binaries built by the original application "
+repository are either official binaries built by the original application
-    + "developers, or are binaries built from source by the admin of f-droid.org "
+developers, or are binaries built from source by the admin of f-droid.org
-    + "using the tools on https://gitlab.com/u/fdroid.")
+using the tools on https://gitlab.com/u/fdroid.
+"""
 
 # As above, but for the archive repo.
 # archive_older sets the number of versions kept in the main repo, with all
@@ -47,9 +48,9 @@ archive_older = 3
 archive_url = "https://f-droid.org/archive"
 archive_name = "My First FDroid Archive Demo"
 archive_icon = "fdroid-icon.png"
-archive_description = (
+archive_description = """
-    "The repository of older versions of applications from the main demo "
+The repository of older versions of applications from the main demo repository.
-    + "repository.")
+"""
 
 
 # The ID of a GPG key for making detached signatures for apks. Optional.
@@ -101,9 +102,15 @@ keyaliases['com.example.another.plugin'] = '@com.example.another'
 # rsync/ssh format for a remote host/path. This is used for syncing a locally
 # generated repo to the server that is it hosted on.  It must end in the
 # standard public repo name of "/fdroid", but can be in up to three levels of
-# sub-directories (i.e. /var/www/packagerepos/fdroid).
+# sub-directories (i.e. /var/www/packagerepos/fdroid).  You can include
+# multiple servers to sync to by wrapping the whole thing in {} or [], and
+# including the serverwebroot strings in a comma-separated list.
 #
 # serverwebroot = 'user@example:/var/www/fdroid'
+# serverwebroot = {
+#     'foo.com:/usr/share/nginx/www/fdroid',
+#     'bar.info:/var/www/fdroid',
+#     }
 
 
 # optionally specific which identity file to use when using rsync over SSH

fdroidserver/common.py

@@ -62,11 +62,12 @@ def get_default_config():
         'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
         'repo_name': "My First FDroid Repo Demo",
         'repo_icon': "fdroid-icon.png",
-        'repo_description': (
+        'repo_description': '''
-            "This is a repository of apps to be used with FDroid. Applications in this "
+            This is a repository of apps to be used with FDroid. Applications in this
-            + "repository are either official binaries built by the original application "
+            repository are either official binaries built by the original application
-            + "developers, or are binaries built from source by the admin of f-droid.org "
+            developers, or are binaries built from source by the admin of f-droid.org
-            + "using the tools on https://gitlab.com/u/fdroid."),
+            using the tools on https://gitlab.com/u/fdroid.
+            ''',
         'archive_older': 0,
     }
 
@@ -163,12 +164,25 @@ def read_config(opts, config_file='config.py'):
         if k in config:
             write_password_file(k)
 
-    # since this is used with rsync, where trailing slashes have meaning,
+    for k in ["repo_description", "archive_description"]:
-    # ensure there is always a trailing slash
+        if k in config:
+            config[k] = clean_description(config[k])
+
     if 'serverwebroot' in config:
-        if config['serverwebroot'][-1] != '/':
+        if isinstance(config['serverwebroot'], basestring):
-            config['serverwebroot'] += '/'
+            roots = [config['serverwebroot']]
-        config['serverwebroot'] = config['serverwebroot'].replace('//', '/')
+        elif all(isinstance(item, basestring) for item in config['serverwebroot']):
+            roots = config['serverwebroot']
+        else:
+            raise TypeError('only accepts strings, lists, and tuples')
+        rootlist = []
+        for rootstr in roots:
+            # since this is used with rsync, where trailing slashes have
+            # meaning, ensure there is always a trailing slash
+            if rootstr[-1] != '/':
+                rootstr += '/'
+            rootlist.append(rootstr.replace('//', '/'))
+        config['serverwebroot'] = rootlist
 
     return config
 
@@ -290,6 +304,19 @@ def has_extension(filename, extension):
 apk_regex = None
 
 
+def clean_description(description):
+    'Remove unneeded newlines and spaces from a block of description text'
+    returnstring = ''
+    # this is split up by paragraph to make removing the newlines easier
+    for paragraph in re.split(r'\n\n', description):
+        paragraph = re.sub('\r', '', paragraph)
+        paragraph = re.sub('\n', ' ', paragraph)
+        paragraph = re.sub(' {2,}', ' ', paragraph)
+        paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
+        returnstring += paragraph + '\n\n'
+    return returnstring.rstrip('\n')
+
+
 def apknameinfo(filename):
     global apk_regex
     filename = os.path.basename(filename)

fdroidserver/server.py

@@ -20,6 +20,8 @@
 import sys
 import hashlib
 import os
+import paramiko
+import pwd
 import subprocess
 from optparse import OptionParser
 import logging
@@ -114,7 +116,7 @@ def update_awsbucket(repo_section):
             logging.info(' skipping ' + s3url)
 
 
-def update_serverwebroot(repo_section):
+def update_serverwebroot(serverwebroot, repo_section):
     rsyncargs = ['rsync', '--archive', '--delete']
     if options.verbose:
         rsyncargs += ['--verbose']
@@ -129,11 +131,11 @@ def update_serverwebroot(repo_section):
     # serverwebroot is guaranteed to have a trailing slash in common.py
     if subprocess.call(rsyncargs +
                        ['--exclude', indexxml, '--exclude', indexjar,
-                        repo_section, config['serverwebroot']]) != 0:
+                        repo_section, serverwebroot]) != 0:
         sys.exit(1)
     # use stricter checking on the indexes since they provide the signature
     rsyncargs += ['--checksum']
-    sectionpath = config['serverwebroot'] + repo_section
+    sectionpath = serverwebroot + repo_section
     if subprocess.call(rsyncargs + [indexxml, sectionpath]) != 0:
         sys.exit(1)
     if subprocess.call(rsyncargs + [indexjar, sectionpath]) != 0:
@@ -141,7 +143,8 @@ def update_serverwebroot(repo_section):
 
 
 def _local_sync(fromdir, todir):
-    rsyncargs = ['rsync', '--archive', '--one-file-system', '--delete']
+    rsyncargs = ['rsync', '--recursive', '--links', '--times',
+                 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
     # use stricter rsync checking on all files since people using offline mode
     # are already prioritizing security above ease and speed
     rsyncargs += ['--checksum']
@@ -199,12 +202,11 @@ def main():
     else:
         standardwebroot = True
 
-    if config.get('serverwebroot'):
+    for serverwebroot in config.get('serverwebroot', []):
-        serverwebroot = config['serverwebroot']
         host, fdroiddir = serverwebroot.rstrip('/').split(':')
         repobase = os.path.basename(fdroiddir)
         if standardwebroot and repobase != 'fdroid':
-            logging.error('serverwebroot does not end with "fdroid", '
+            logging.error('serverwebroot path does not end with "fdroid", '
                           + 'perhaps you meant one of these:\n\t'
                           + serverwebroot.rstrip('/') + '/fdroid\n\t'
                           + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
@@ -254,18 +256,27 @@ def main():
             os.mkdir('archive')
 
     if args[0] == 'init':
-        if config.get('serverwebroot'):
+        ssh = paramiko.SSHClient()
-            sshargs = ['ssh']
+        ssh.load_system_host_keys()
-            if options.quiet:
+        for serverwebroot in config.get('serverwebroot', []):
-                sshargs += ['-q']
+            sshstr, remotepath = serverwebroot.rstrip('/').split(':')
+            if sshstr.find('@') >= 0:
+                username, hostname = sshstr.split('@')
+            else:
+                username = pwd.getpwuid(os.getuid())[0]  # get effective uid
+                hostname = sshstr
+            ssh.connect(hostname, username=username)
+            sftp = ssh.open_sftp()
+            if os.path.basename(remotepath) \
+                    not in sftp.listdir(os.path.dirname(remotepath)):
+                sftp.mkdir(remotepath, mode=0755)
             for repo_section in repo_sections:
-                cmd = sshargs + [host, 'mkdir -p', fdroiddir + '/' + repo_section]
+                repo_path = os.path.join(remotepath, repo_section)
-                if options.verbose:
+                if os.path.basename(repo_path) \
-                    # ssh -v produces different output than rsync -v, so this
+                        not in sftp.listdir(remotepath):
-                    # simulates rsync -v
+                    sftp.mkdir(repo_path, mode=0755)
-                    logging.info(' '.join(cmd))
+            sftp.close()
-                if subprocess.call(cmd) != 0:
+            ssh.close()
-                    sys.exit(1)
     elif args[0] == 'update':
         for repo_section in repo_sections:
             if local_copy_dir is not None:
@@ -273,8 +284,8 @@ def main():
                     sync_from_localcopy(repo_section, local_copy_dir)
                 else:
                     update_localcopy(repo_section, local_copy_dir)
-            if config.get('serverwebroot'):
+            for serverwebroot in config.get('serverwebroot', []):
-                update_serverwebroot(repo_section)
+                update_serverwebroot(serverwebroot, repo_section)
             if config.get('awsbucket'):
                 update_awsbucket(repo_section)
 

tests/description-parsing.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+
+sys.path.insert(1, os.path.join(os.getcwd(), '..', 'fdroidserver'))
+import common
+
+config = common.get_default_config()
+
+testtext = '''
+This is a block of text that has been wrapped to fit nicely in PEP8 style:
+
+GnuPrivacyGuard extends the gpgcli command line tool to bring an integrated
+privacy engine to your Android. It gives you command line access to the entire
+GnuPG suite of encryption software. It also serves as the test bed for
+complete Android integration for all of GnuPG's crypto services, including
+OpenPGP, symmetric encryption, and more.
+
+GPG is GNU’s tool for end-to-end secure communication and encrypted data
+storage. This trusted protocol is the free software alternative to PGP. This
+app is built upon GnuPG 2.1, the new modularized version of GnuPG that now
+supports S/MIME.
+
+GPG aims to provide an integrated experience, so clicking on PGP files should
+"just work". You can also share files to GPG to encrypt them. GPG will also
+respond when you click on a PGP fingerprint URL (one that starts with
+openpgp4fpr:).
+
+Before using GPG, be sure to launch the app and let it finish its installation
+process. Once it has completed, then you're ready to use it. The easiest way
+to get started with GPG is to install [[jackpal.androidterm]]. GPG will
+automatically configure Android Terminal Emulator as long as you have the
+"Allow PATH extensions" settings enabled.
+'''
+
+archive_description = """
+The repository of older versions of applications from the main demo repository.
+"""
+
+
+print('\n\n\n----------------------------------------------------')
+print(common.clean_description(testtext))
+print('\n\n\n----------------------------------------------------')
+print(common.clean_description(archive_description))
+print('\n\n\n----------------------------------------------------')
+print(common.clean_description(config['repo_description']))