Merge branch 'buildserver-fixes' into 'master'

random buildserver fixes

Closes #636 and #624

See merge request fdroid/fdroidserver!713

buildserver/provision-apt-get-install

@@ -5,7 +5,7 @@ set -e
 set -x
 
 debian_mirror=$1
-export DEBIAN_FRONTEND=noninteractive 
+export DEBIAN_FRONTEND=noninteractive
 
 printf 'APT::Install-Recommends "0";\nAPT::Install-Suggests "0";\n' \
        > /etc/apt/apt.conf.d/99no-install-recommends
@@ -22,6 +22,11 @@ EOF
 printf 'APT::Get::Assume-Yes "true";\n' \
        > /etc/apt/apt.conf.d/99assumeyes
 
+cat <<EOF > /etc/apt/apt.conf.d/99quiet
+Dpkg::Use-Pty "0";'
+quiet "1";'
+EOF
+
 if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then
     apt-get update || apt-get update
     apt-get install apt-transport-https ca-certificates

buildserver/provision-gradle

@@ -26,3 +26,17 @@ ln -fs /home/vagrant/fdroidserver/gradlew-fdroid /opt/gradle/bin/gradle
 chown -h vagrant.vagrant /opt/gradle/bin/gradle
 chown vagrant.vagrant /opt/gradle/versions
 chmod 0755 /opt/gradle/versions
+
+GRADLE_HOME=/home/vagrant/.gradle
+test -d $GRADLE_HOME/ || mkdir $GRADLE_HOME/
+cat <<EOF > $GRADLE_HOME/gradle.properties
+# builds are not reused, so the daemon is a waste of time
+org.gradle.daemon=false
+
+# fake info to block HTTP repos
+systemProp.http.nonProxyHosts=
+systemProp.http.proxyHost=127.127.127.127
+systemProp.http.proxyPort=12345
+EOF
+chown -R vagrant.vagrant $GRADLE_HOME/
+chmod -R a+rX $GRADLE_HOME/

fdroidserver/build.py

@@ -371,7 +371,8 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
         if build.sudo:
             logging.info("Running 'sudo' commands in %s" % os.getcwd())
 
-            p = FDroidPopen(['sudo', 'bash', '-x', '-c', build.sudo])
+            p = FDroidPopen(['sudo', '--preserve-env=DEBIAN_FRONTEND',
+                             'bash', '-x', '-c', build.sudo])
             if p.returncode != 0:
                 raise BuildException("Error running sudo command for %s:%s" %
                                      (app.id, build.versionName), p.output)

fdroidserver/server.py

@@ -33,6 +33,7 @@ import shutil
 from . import _
 from . import common
 from . import index
+from . import update
 from .exception import FDroidException
 
 config = None
@@ -450,35 +451,48 @@ def upload_to_android_observatory(repo_section):
     import requests
     from lxml.html import fromstring
 
+    if options.verbose:
+        logging.getLogger("requests").setLevel(logging.INFO)
+        logging.getLogger("urllib3").setLevel(logging.INFO)
+    else:
+        logging.getLogger("requests").setLevel(logging.WARNING)
+        logging.getLogger("urllib3").setLevel(logging.WARNING)
+
     if repo_section == 'repo':
-        for f in glob.glob(os.path.join(repo_section, '*.apk')):
+        for f in sorted(glob.glob(os.path.join(repo_section, '*.apk'))):
             fpath = f
             fname = os.path.basename(f)
-            logging.info('Uploading ' + fname + ' to androidobservatory.org')
+            r = requests.post('https://androidobservatory.org/',
+                              data={'q': update.sha256sum(f), 'searchby': 'hash'})
+            if r.status_code == 200:
+                # from now on XPath will be used to retrieve the message in the HTML
+                # androidobservatory doesn't have a nice API to talk with
+                # so we must scrape the page content
+                tree = fromstring(r.text)
+
+                href = None
+                for element in tree.xpath("//html/body/div/div/table/tbody/tr/td/a"):
+                    a = element.attrib.get('href')
+                    if a:
+                        m = re.match(r'^/app/[0-9A-F]{40}$', a)
+                        if m:
+                            href = m.group()
+
+                page = 'https://androidobservatory.org'
+                message = ''
+                if href:
+                    message = (_('Found {apkfilename} at {url}')
+                               .format(apkfilename=fname, url=(page + href)))
+                if message:
+                    logging.debug(message)
+                    continue
 
             # upload the file with a post request
-            r = requests.post('https://androidobservatory.org/upload', files={'apk': (fname, open(fpath, 'rb'))})
-            response = r.text
-            page = r.url
-
-            # from now on XPath will be used to retrieve the message in the HTML
-            # androidobservatory doesn't have a nice API to talk with
-            # so we must scrape the page content
-            tree = fromstring(response)
-            alert = tree.xpath("//html/body/div[@class='container content-container']/div[@class='alert alert-info']")[0]
-
-            message = ""
-            appurl = page
-            for el in alert:
-                # if the application was added successfully we retrive the url
-                # if the application was already uploaded we use the redirect page url
-                if el.attrib.get("href") is not None:
-                    appurl = page + el.attrib["href"][1:]
-                    message += el.text.replace(" here", "") + el.tail
-                else:
-                    message += el.tail
-            message = message.strip() + " " + appurl
-            logging.info(message)
+            logging.info(_('Uploading {apkfilename} to androidobservatory.org')
+                         .format(apkfilename=fname))
+            r = requests.post('https://androidobservatory.org/upload',
+                              files={'apk': (fname, open(fpath, 'rb'))},
+                              allow_redirects=False)
 
 
 def upload_to_virustotal(repo_section, virustotal_apikey):