Merge branch 'apksigcopier' into 'master'

* origin/master:
  gitlab-ci: prevent dualing linters: pyflakes vs mypy
  jenkins-build-all: refocus on building reproducible apps first
  publish: rename vars to match naming in JAR Signature docs
  common.py: update copyrights
  apksigcopier: remove exclude_all_meta logic
  apksigcopier: purge main() to avoid confusion
  force apksigcopier "AUTO"  to select sigs by available files
  publish: add test for reproduble builds with signatures
  vendor & use apksigcopier v0.4.0-12-g93d8e14
  use subclass hack for better ZIP cloning

fdroid/fdroidserver!893

.gitlab-ci.yml

@@ -193,7 +193,8 @@ lint_mypy:
   script:
     - pip install mypy
     - pip install -e .[test]
-    - mypy
+    # exclude vendored file
+    - mypy --exclude fdroidserver/apksigcopier.py
 
 fedora_latest:
   image: fedora:latest

fdroidserver/apksigcopier.py

@@ -0,0 +1,514 @@
+#!/usr/bin/python3
+# encoding: utf-8
+
+# --                                                            ; {{{1
+#
+# File        : apksigcopier
+# Maintainer  : Felix C. Stegerman <flx@obfusk.net>
+# Date        : 2021-04-14
+#
+# Copyright   : Copyright (C) 2021  Felix C. Stegerman
+# Version     : v0.4.0
+# License     : GPLv3+
+#
+# --                                                            ; }}}1
+
+"""
+copy/extract/patch apk signatures
+
+apksigcopier is a tool for copying APK signatures from a signed APK to an
+unsigned one (in order to verify reproducible builds).
+
+
+CLI
+===
+
+$ apksigcopier extract [OPTIONS] SIGNED_APK OUTPUT_DIR
+$ apksigcopier patch [OPTIONS] METADATA_DIR UNSIGNED_APK OUTPUT_APK
+$ apksigcopier copy [OPTIONS] SIGNED_APK UNSIGNED_APK OUTPUT_APK
+
+The following environment variables can be set to 1, yes, or true to
+overide the default behaviour:
+
+* set APKSIGCOPIER_EXCLUDE_ALL_META=1 to exclude all metadata files
+* set APKSIGCOPIER_COPY_EXTRA_BYTES=1 to copy extra bytes after data (e.g. a v2 sig)
+
+
+API
+===
+
+>> from apksigcopier import do_extract, do_patch, do_copy
+>> do_extract(signed_apk, output_dir, v1_only=NO)
+>> do_patch(metadata_dir, unsigned_apk, output_apk, v1_only=NO)
+>> do_copy(signed_apk, unsigned_apk, output_apk, v1_only=NO)
+
+You can use False, None, and True instead of NO, AUTO, and YES respectively.
+
+The following global variables (which default to False), can be set to
+override the default behaviour:
+
+* set exclude_all_meta=True to exclude all metadata files
+* set copy_extra_bytes=True to copy extra bytes after data (e.g. a v2 sig)
+"""
+
+import glob
+import os
+import re
+import struct
+import sys
+import zipfile
+import zlib
+
+from collections import namedtuple
+
+__version__ = "0.4.0"
+NAME = "apksigcopier"
+
+SIGBLOCK, SIGOFFSET = "APKSigningBlock", "APKSigningBlockOffset"
+NOAUTOYES = NO, AUTO, YES = ("no", "auto", "yes")
+APK_META = re.compile(r"^META-INF/([0-9A-Za-z_-]+\.(SF|RSA|DSA|EC)|MANIFEST\.MF)$")
+META_EXT = ("SF", "RSA|DSA|EC", "MF")
+COPY_EXCLUDE = ("META-INF/MANIFEST.MF",)
+DATETIMEZERO = (1980, 0, 0, 0, 0, 0)
+
+ZipData = namedtuple("ZipData", ("cd_offset", "eocd_offset", "cd_and_eocd"))
+
+copy_extra_bytes = False    # copy extra bytes after data in copy_apk()
+
+
+class APKSigCopierError(Exception):
+    """Base class for errors."""
+
+
+class APKSigningBlockError(APKSigCopierError):
+    """Something wrong with the APK Signing Block."""
+
+
+class NoAPKSigningBlock(APKSigningBlockError):
+    """APK Signing Block Missing."""
+
+
+class ZipError(APKSigCopierError):
+    """Something wrong with ZIP file."""
+
+
+# FIXME: is there a better alternative?
+class ReproducibleZipInfo(zipfile.ZipInfo):
+    """Reproducible ZipInfo hack."""
+
+    _override = {}
+
+    def __init__(self, zinfo, **override):
+        if override:
+            self._override = {**self._override, **override}
+        for k in self.__slots__:
+            if hasattr(zinfo, k):
+                setattr(self, k, getattr(zinfo, k))
+
+    def __getattribute__(self, name):
+        if name != "_override":
+            try:
+                return self._override[name]
+            except KeyError:
+                pass
+        return object.__getattribute__(self, name)
+
+
+class APKZipInfo(ReproducibleZipInfo):
+    """Reproducible ZipInfo for APK files."""
+
+    _override = dict(
+        compress_type=8,
+        create_system=0,
+        create_version=20,
+        date_time=DATETIMEZERO,
+        external_attr=0,
+        extract_version=20,
+        flag_bits=0x800,
+    )
+
+
+def noautoyes(value):
+    """
+    Turns False into NO, None into AUTO, and True into YES.
+
+    >>> from apksigcopier import noautoyes, NO, AUTO, YES
+    >>> noautoyes(False) == NO == noautoyes(NO)
+    True
+    >>> noautoyes(None) == AUTO == noautoyes(AUTO)
+    True
+    >>> noautoyes(True) == YES == noautoyes(YES)
+    True
+
+    """
+    if isinstance(value, str):
+        if value not in NOAUTOYES:
+            raise ValueError("expected NO, AUTO, or YES")
+        return value
+    try:
+        return {False: NO, None: AUTO, True: YES}[value]
+    except KeyError:
+        raise ValueError("expected False, None, or True")
+
+
+def is_meta(filename):
+    """
+    Returns whether filename is a v1 (JAR) signature file (.SF), signature block
+    file (.RSA, .DSA, or .EC), or manifest (MANIFEST.MF).
+
+    See https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html
+    """
+    return APK_META.fullmatch(filename) is not None
+
+
+def exclude_from_copying(filename):
+    """fdroidserver always wants JAR Signature files to be excluded"""
+    return is_meta(filename)
+
+
+################################################################################
+#
+# https://en.wikipedia.org/wiki/ZIP_(file_format)
+# https://source.android.com/security/apksigning/v2#apk-signing-block-format
+#
+# =================================
+# | Contents of ZIP entries       |
+# =================================
+# | APK Signing Block             |
+# | ----------------------------- |
+# | | size (w/o this) uint64 LE | |
+# | | ...                       | |
+# | | size (again)    uint64 LE | |
+# | | "APK Sig Block 42" (16B)  | |
+# | ----------------------------- |
+# =================================
+# | ZIP Central Directory         |
+# =================================
+# | ZIP End of Central Directory  |
+# | ----------------------------- |
+# | | 0x06054b50 ( 4B)          | |
+# | | ...        (12B)          | |
+# | | CD Offset  ( 4B)          | |
+# | | ...                       | |
+# | ----------------------------- |
+# =================================
+#
+################################################################################
+
+
+# FIXME: makes certain assumptions and doesn't handle all valid ZIP files!
+def copy_apk(unsigned_apk, output_apk):
+    """
+    Copy APK like apksigner would, excluding files matched by
+    exclude_from_copying().
+
+    Returns max date_time.
+
+    The following global variables (which default to False), can be set to
+    override the default behaviour:
+
+    * set exclude_all_meta=True to exclude all metadata files
+    * set copy_extra_bytes=True to copy extra bytes after data (e.g. a v2 sig)
+    """
+    with zipfile.ZipFile(unsigned_apk, "r") as zf:
+        infos = zf.infolist()
+    zdata = zip_data(unsigned_apk)
+    offsets = {}
+    with open(unsigned_apk, "rb") as fhi, open(output_apk, "w+b") as fho:
+        for info in sorted(infos, key=lambda info: info.header_offset):
+            off_i = fhi.tell()
+            if info.header_offset > off_i:
+                # copy extra bytes
+                fho.write(fhi.read(info.header_offset - off_i))
+            hdr = fhi.read(30)
+            if hdr[:4] != b"\x50\x4b\x03\x04":
+                raise ZipError("Expected local file header signature")
+            n, m = struct.unpack("<HH", hdr[26:30])
+            hdr += fhi.read(n + m)
+            skip = exclude_from_copying(info.filename)
+            if skip:
+                fhi.seek(info.compress_size, os.SEEK_CUR)
+            else:
+                if info.filename in offsets:
+                    raise ZipError("Duplicate ZIP entry: " + info.filename)
+                offsets[info.filename] = off_o = fho.tell()
+                if info.compress_type == 0 and off_o != info.header_offset:
+                    hdr = _realign_zip_entry(info, hdr, n, m, off_o)
+                fho.write(hdr)
+                _copy_bytes(fhi, fho, info.compress_size)
+            if info.flag_bits & 0x08:
+                data_descriptor = fhi.read(12)
+                if data_descriptor[:4] == b"\x50\x4b\x07\x08":
+                    data_descriptor += fhi.read(4)
+                if not skip:
+                    fho.write(data_descriptor)
+        extra_bytes = zdata.cd_offset - fhi.tell()
+        if copy_extra_bytes:
+            _copy_bytes(fhi, fho, extra_bytes)
+        else:
+            fhi.seek(extra_bytes, os.SEEK_CUR)
+        cd_offset = fho.tell()
+        for info in infos:
+            hdr = fhi.read(46)
+            if hdr[:4] != b"\x50\x4b\x01\x02":
+                raise ZipError("Expected central directory file header signature")
+            n, m, k = struct.unpack("<HHH", hdr[28:34])
+            hdr += fhi.read(n + m + k)
+            if not exclude_from_copying(info.filename):
+                off = int.to_bytes(offsets[info.filename], 4, "little")
+                hdr = hdr[:42] + off + hdr[46:]
+                fho.write(hdr)
+        eocd_offset = fho.tell()
+        fho.write(zdata.cd_and_eocd[zdata.eocd_offset - zdata.cd_offset:])
+        fho.seek(eocd_offset + 8)
+        fho.write(struct.pack("<HHLL", len(offsets), len(offsets),
+                              eocd_offset - cd_offset, cd_offset))
+    return max(info.date_time for info in infos)
+
+
+# NB: doesn't sync local & CD headers!
+def _realign_zip_entry(info, hdr, n, m, off_o):
+    align = 4096 if info.filename.endswith(".so") else 4
+    old_off = 30 + n + m + info.header_offset
+    new_off = 30 + n + m + off_o
+    old_xtr = info.extra
+    new_xtr = b""
+    while len(old_xtr) >= 4:
+        hdr_id, size = struct.unpack("<HH", old_xtr[:4])
+        if size > len(old_xtr) - 4:
+            break
+        if not (hdr_id == 0 and size == 0):
+            if hdr_id == 0xd935:
+                if size >= 2:
+                    align = int.from_bytes(old_xtr[4:6], "little")
+            else:
+                new_xtr += old_xtr[:size + 4]
+        old_xtr = old_xtr[size + 4:]
+    if old_off % align == 0 and new_off % align != 0:
+        pad = (align - (new_off - m + len(new_xtr) + 6) % align) % align
+        xtr = new_xtr + struct.pack("<HHH", 0xd935, 2 + pad, align) + pad * b"\x00"
+        m_b = int.to_bytes(len(xtr), 2, "little")
+        hdr = hdr[:28] + m_b + hdr[30:30 + n] + xtr
+    return hdr
+
+
+def _copy_bytes(fhi, fho, size, blocksize=4096):
+    while size > 0:
+        data = fhi.read(min(size, blocksize))
+        if not data:
+            break
+        size -= len(data)
+        fho.write(data)
+    if size != 0:
+        raise ZipError("Unexpected EOF")
+
+
+def extract_meta(signed_apk):
+    """
+    Extract v1 signature metadata files from signed APK.
+
+    Yields (ZipInfo, data) pairs.
+    """
+    with zipfile.ZipFile(signed_apk, "r") as zf_sig:
+        for info in zf_sig.infolist():
+            if is_meta(info.filename):
+                yield info, zf_sig.read(info.filename)
+
+
+def patch_meta(extracted_meta, output_apk, date_time=DATETIMEZERO):
+    """Add v1 signature metadata to APK (removes v2 sig block, if any)."""
+    with zipfile.ZipFile(output_apk, "r") as zf_out:
+        for info in zf_out.infolist():
+            if is_meta(info.filename):
+                raise ZipError("Unexpected metadata")
+    with zipfile.ZipFile(output_apk, "a") as zf_out:
+        info_data = [(APKZipInfo(info, date_time=date_time), data)
+                     for info, data in extracted_meta]
+        _write_to_zip(info_data, zf_out)
+
+
+if sys.version_info >= (3, 7):
+    def _write_to_zip(info_data, zf_out):
+        for info, data in info_data:
+            zf_out.writestr(info, data, compresslevel=9)
+else:
+    def _write_to_zip(info_data, zf_out):
+        old = zipfile._get_compressor
+        zipfile._get_compressor = lambda _: zlib.compressobj(9, 8, -15)
+        try:
+            for info, data in info_data:
+                zf_out.writestr(info, data)
+        finally:
+            zipfile._get_compressor = old
+
+
+def extract_v2_sig(apkfile, expected=True):
+    """
+    Extract APK Signing Block and offset from APK.
+
+    When successful, returns (sb_offset, sig_block); otherwise raises
+    NoAPKSigningBlock when expected is True, else returns None.
+    """
+    cd_offset = zip_data(apkfile).cd_offset
+    with open(apkfile, "rb") as fh:
+        fh.seek(cd_offset - 16)
+        if fh.read(16) != b"APK Sig Block 42":
+            if expected:
+                raise NoAPKSigningBlock("No APK Signing Block")
+            return None
+        fh.seek(-24, os.SEEK_CUR)
+        sb_size2 = int.from_bytes(fh.read(8), "little")
+        fh.seek(-sb_size2 + 8, os.SEEK_CUR)
+        sb_size1 = int.from_bytes(fh.read(8), "little")
+        if sb_size1 != sb_size2:
+            raise APKSigningBlockError("APK Signing Block sizes not equal")
+        fh.seek(-8, os.SEEK_CUR)
+        sb_offset = fh.tell()
+        sig_block = fh.read(sb_size2 + 8)
+    return sb_offset, sig_block
+
+
+def zip_data(apkfile, count=1024):
+    """
+    Extract central directory, EOCD, and offsets from ZIP.
+
+    Returns ZipData.
+    """
+    with open(apkfile, "rb") as fh:
+        fh.seek(-count, os.SEEK_END)
+        data = fh.read()
+        pos = data.rfind(b"\x50\x4b\x05\x06")
+        if pos == -1:
+            raise ZipError("Expected end of central directory record (EOCD)")
+        fh.seek(pos - len(data), os.SEEK_CUR)
+        eocd_offset = fh.tell()
+        fh.seek(16, os.SEEK_CUR)
+        cd_offset = int.from_bytes(fh.read(4), "little")
+        fh.seek(cd_offset)
+        cd_and_eocd = fh.read()
+    return ZipData(cd_offset, eocd_offset, cd_and_eocd)
+
+
+# FIXME: can we determine signed_sb_offset?
+def patch_v2_sig(extracted_v2_sig, output_apk):
+    """Implant extracted v2/v3 signature into APK."""
+    signed_sb_offset, signed_sb = extracted_v2_sig
+    data_out = zip_data(output_apk)
+    if signed_sb_offset < data_out.cd_offset:
+        raise APKSigningBlockError("APK Signing Block offset < central directory offset")
+    padding = b"\x00" * (signed_sb_offset - data_out.cd_offset)
+    offset = len(signed_sb) + len(padding)
+    with open(output_apk, "r+b") as fh:
+        fh.seek(data_out.cd_offset)
+        fh.write(padding)
+        fh.write(signed_sb)
+        fh.write(data_out.cd_and_eocd)
+        fh.seek(data_out.eocd_offset + offset + 16)
+        fh.write(int.to_bytes(data_out.cd_offset + offset, 4, "little"))
+
+
+def patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk):
+    """
+    Patch extracted_meta + extracted_v2_sig (if not None) onto unsigned_apk and
+    save as output_apk.
+    """
+    date_time = copy_apk(unsigned_apk, output_apk)
+    patch_meta(extracted_meta, output_apk, date_time=date_time)
+    if extracted_v2_sig is not None:
+        patch_v2_sig(extracted_v2_sig, output_apk)
+
+
+def do_extract(signed_apk, output_dir, v1_only=NO):
+    """
+    Extract signatures from signed_apk and save in output_dir.
+
+    The v1_only parameter controls whether the absence of a v1 signature is
+    considered an error or not:
+    * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures;
+    * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures;
+    * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures.
+    """
+    v1_only = noautoyes(v1_only)
+    extracted_meta = tuple(extract_meta(signed_apk))
+    if len(extracted_meta) not in (len(META_EXT), 0):
+        raise APKSigCopierError("Unexpected or missing metadata files in signed_apk")
+    for info, data in extracted_meta:
+        name = os.path.basename(info.filename)
+        with open(os.path.join(output_dir, name), "wb") as fh:
+            fh.write(data)
+    if v1_only == YES:
+        if not extracted_meta:
+            raise APKSigCopierError("Expected v1 signature")
+        return
+    expected = v1_only == NO
+    extracted_v2_sig = extract_v2_sig(signed_apk, expected=expected)
+    if extracted_v2_sig is None:
+        if not extracted_meta:
+            raise APKSigCopierError("Expected v1 and/or v2/v3 signature, found neither")
+        return
+    signed_sb_offset, signed_sb = extracted_v2_sig
+    with open(os.path.join(output_dir, SIGOFFSET), "w") as fh:
+        fh.write(str(signed_sb_offset) + "\n")
+    with open(os.path.join(output_dir, SIGBLOCK), "wb") as fh:
+        fh.write(signed_sb)
+
+
+def do_patch(metadata_dir, unsigned_apk, output_apk, v1_only=NO):
+    """
+    Patch signatures from metadata_dir onto unsigned_apk and save as output_apk.
+
+    The v1_only parameter controls whether the absence of a v1 signature is
+    considered an error or not:
+    * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures;
+    * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures;
+    * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures.
+    """
+    v1_only = noautoyes(v1_only)
+    extracted_meta = []
+    for pat in META_EXT:
+        files = [fn for ext in pat.split("|") for fn in
+                 glob.glob(os.path.join(metadata_dir, "*." + ext))]
+        if len(files) != 1:
+            continue
+        info = zipfile.ZipInfo("META-INF/" + os.path.basename(files[0]))
+        with open(files[0], "rb") as fh:
+            extracted_meta.append((info, fh.read()))
+    if len(extracted_meta) not in (len(META_EXT), 0):
+        raise APKSigCopierError("Unexpected or missing files in metadata_dir")
+    if v1_only == YES:
+        extracted_v2_sig = None
+    else:
+        sigoffset_file = os.path.join(metadata_dir, SIGOFFSET)
+        sigblock_file = os.path.join(metadata_dir, SIGBLOCK)
+        if v1_only == AUTO and not os.path.exists(sigblock_file):
+            extracted_v2_sig = None
+        else:
+            with open(sigoffset_file, "r") as fh:
+                signed_sb_offset = int(fh.read())
+            with open(sigblock_file, "rb") as fh:
+                signed_sb = fh.read()
+            extracted_v2_sig = signed_sb_offset, signed_sb
+    if not extracted_meta and extracted_v2_sig is None:
+        raise APKSigCopierError("Expected v1 and/or v2/v3 signature, found neither")
+    patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk)
+
+
+def do_copy(signed_apk, unsigned_apk, output_apk, v1_only=NO):
+    """
+    Copy signatures from signed_apk onto unsigned_apk and save as output_apk.
+
+    The v1_only parameter controls whether the absence of a v1 signature is
+    considered an error or not:
+    * use v1_only=NO (or v1_only=False) to only accept (v1+)v2/v3 signatures;
+    * use v1_only=AUTO (or v1_only=None) to automatically detect v2/v3 signatures;
+    * use v1_only=YES (or v1_only=True) to ignore any v2/v3 signatures.
+    """
+    v1_only = noautoyes(v1_only)
+    extracted_meta = extract_meta(signed_apk)
+    if v1_only == YES:
+        extracted_v2_sig = None
+    else:
+        extracted_v2_sig = extract_v2_sig(signed_apk, expected=v1_only == NO)
+    patch_apk(extracted_meta, extracted_v2_sig, unsigned_apk, output_apk)
+
+# vim: set tw=80 sw=4 sts=4 et fdm=marker :

fdroidserver/common.py

@@ -1,8 +1,16 @@
 #!/usr/bin/env python3
 #
 # common.py - part of the FDroid server tools
-# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
-# Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
+#
+# Copyright (C) 2010-2016, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2013-2017, Daniel Martí <mvdan@mvdan.cc>
+# Copyright (C) 2013-2021, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2017-2018, Torsten Grote <t@grobox.de>
+# Copyright (C) 2017, tobiasKaminsky <tobias@kaminsky.me>
+# Copyright (C) 2017-2021, Michael Pöhn <michael.poehn@fsfe.org>
+# Copyright (C) 2017,2021, mimi89999 <michel@lebihan.pl>
+# Copyright (C) 2019-2021, Jochen Sprickerhof <git@jochen.sprickerhof.de>
+# Copyright (C) 2021, Felix C. Stegerman <flx@obfusk.net>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -67,6 +75,9 @@ from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesEx
     BuildException, VerificationException, MetaDataException
 from .asynchronousfilereader import AsynchronousFileReader
 
+from . import apksigcopier
+
+
 # The path to this fdroidserver distribution
 FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
 
@@ -2932,13 +2943,13 @@ def apk_signer_fingerprint_short(apk_path):
 def metadata_get_sigdir(appid, vercode=None):
     """Get signature directory for app"""
     if vercode:
-        return os.path.join('metadata', appid, 'signatures', vercode)
+        return os.path.join('metadata', appid, 'signatures', str(vercode))
     else:
         return os.path.join('metadata', appid, 'signatures')
 
 
 def metadata_find_developer_signature(appid, vercode=None):
-    """Tires to find the developer signature for given appid.
+    """Tries to find the developer signature for given appid.
 
     This picks the first signature file found in metadata an returns its
     signature.
@@ -2960,45 +2971,63 @@ def metadata_find_developer_signature(appid, vercode=None):
                     appversigdirs.append(appversigdir)
 
     for sigdir in appversigdirs:
-        sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
-            glob.glob(os.path.join(sigdir, '*.EC')) + \
-            glob.glob(os.path.join(sigdir, '*.RSA'))
-        if len(sigs) > 1:
+        signature_block_files = (
+            glob.glob(os.path.join(sigdir, '*.DSA'))
+            + glob.glob(os.path.join(sigdir, '*.EC'))
+            + glob.glob(os.path.join(sigdir, '*.RSA'))
+        )
+        if len(signature_block_files) > 1:
             raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
-        for sig in sigs:
-            with open(sig, 'rb') as f:
+        for signature_block_file in signature_block_files:
+            with open(signature_block_file, 'rb') as f:
                 return signer_fingerprint(get_certificate(f.read()))
     return None
 
 
 def metadata_find_signing_files(appid, vercode):
-    """Gets a list of singed manifests and signatures.
+    """Gets a list of signed manifests and signatures.
+
+    https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html
+    https://source.android.com/security/apksigning/v2
+    https://source.android.com/security/apksigning/v3
 
     :param appid: app id string
     :param vercode: app version code
-    :returns: a list of triplets for each signing key with following paths:
-        (signature_file, singed_file, manifest_file)
+    :returns: a list of 4-tuples for each signing key with following paths:
+        (signature_file, signature_block_file, manifest, v2_files), where v2_files
+        is either a (apk_signing_block_offset_file, apk_signing_block_file) pair or None
+
     """
     ret = []
     sigdir = metadata_get_sigdir(appid, vercode)
-    sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
-        glob.glob(os.path.join(sigdir, '*.EC')) + \
-        glob.glob(os.path.join(sigdir, '*.RSA'))
-    extre = re.compile(r'(\.DSA|\.EC|\.RSA)$')
-    for sig in sigs:
-        sf = extre.sub('.SF', sig)
-        if os.path.isfile(sf):
-            mf = os.path.join(sigdir, 'MANIFEST.MF')
-            if os.path.isfile(mf):
-                ret.append((sig, sf, mf))
+    signature_block_files = (
+        glob.glob(os.path.join(sigdir, '*.DSA'))
+        + glob.glob(os.path.join(sigdir, '*.EC'))
+        + glob.glob(os.path.join(sigdir, '*.RSA'))
+    )
+    signature_block_pat = re.compile(r'(\.DSA|\.EC|\.RSA)$')
+    apk_signing_block = os.path.isfile(os.path.join(sigdir, "APKSigningBlock"))
+    apk_signing_block_offset = os.path.isfile(os.path.join(sigdir, "APKSigningBlockOffset"))
+    if os.path.isfile(apk_signing_block) and os.path.isfile(apk_signing_block_offset):
+        v2_files = apk_signing_block, apk_signing_block_offset
+    else:
+        v2_files = None
+    for signature_block_file in signature_block_files:
+        signature_file = signature_block_pat.sub('.SF', signature_block_file)
+        if os.path.isfile(signature_file):
+            manifest = os.path.join(sigdir, 'MANIFEST.MF')
+            if os.path.isfile(manifest):
+                ret.append((signature_block_file, signature_file, manifest, v2_files))
     return ret
 
 
 def metadata_find_developer_signing_files(appid, vercode):
     """Get developer signature files for specified app from metadata.
 
-    :returns: A triplet of paths for signing files from metadata:
-        (signature_file, singed_file, manifest_file)
+    :returns: a list of 4-tuples for each signing key with following paths:
+        (signature_file, signature_block_file, manifest, v2_files), where v2_files
+        is either a (apk_signing_block_offset_file, apk_signing_block_file) pair or None
+
     """
     allsigningfiles = metadata_find_signing_files(appid, vercode)
     if allsigningfiles and len(allsigningfiles) == 1:
@@ -3007,6 +3036,36 @@ def metadata_find_developer_signing_files(appid, vercode):
         return None
 
 
+class ClonedZipInfo(zipfile.ZipInfo):
+    """Hack to allow fully cloning ZipInfo instances
+
+    The zipfile library has some bugs that prevent it from fully
+    cloning ZipInfo entries.  https://bugs.python.org/issue43547
+
+    """
+    def __init__(self, zinfo):
+        self.original = zinfo
+        for k in self.__slots__:
+            try:
+                setattr(self, k, getattr(zinfo, k))
+            except AttributeError:
+                pass
+
+    def __getattribute__(self, name):
+        if name in ("date_time", "external_attr", "flag_bits"):
+            return getattr(self.original, name)
+        return object.__getattribute__(self, name)
+
+
+def apk_has_v1_signatures(apkfile):
+    """Test whether an APK has v1 signature files."""
+    with ZipFile(apkfile, 'r') as apk:
+        for info in apk.infolist():
+            if APK_SIGNATURE_FILES.match(info.filename):
+                return True
+    return False
+
+
 def apk_strip_v1_signatures(signed_apk, strip_manifest=False):
     """Removes signatures from APK.
 
@@ -3024,10 +3083,10 @@ def apk_strip_v1_signatures(signed_apk, strip_manifest=False):
                         if strip_manifest:
                             if info.filename != 'META-INF/MANIFEST.MF':
                                 buf = in_apk.read(info.filename)
-                                out_apk.writestr(info, buf)
+                                out_apk.writestr(ClonedZipInfo(info), buf)
                         else:
                             buf = in_apk.read(info.filename)
-                            out_apk.writestr(info, buf)
+                            out_apk.writestr(ClonedZipInfo(info), buf)
 
 
 def _zipalign(unsigned_apk, aligned_apk):
@@ -3042,49 +3101,36 @@ def _zipalign(unsigned_apk, aligned_apk):
         raise BuildException("Failed to align application")
 
 
-def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
+def apk_implant_signatures(apkpath, outpath, manifest):
     """Implants a signature from metadata into an APK.
 
     Note: this changes there supplied APK in place. So copy it if you
     need the original to be preserved.
 
-    :param apkpath: location of the apk
+    https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html
+    https://source.android.com/security/apksigning/v2
+    https://source.android.com/security/apksigning/v3
+
+    :param apkpath: location of the unsigned apk
+    :param outpath: location of the output apk
     """
-    # get list of available signature files in metadata
-    with tempfile.TemporaryDirectory() as tmpdir:
-        apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
-        with ZipFile(apkpath, 'r') as in_apk:
-            with ZipFile(apkwithnewsig, 'w') as out_apk:
-                for sig_file in [signaturefile, signedfile, manifest]:
-                    with open(sig_file, 'rb') as fp:
-                        buf = fp.read()
-                    info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
-                    info.compress_type = zipfile.ZIP_DEFLATED
-                    info.create_system = 0  # "Windows" aka "FAT", what Android SDK uses
-                    out_apk.writestr(info, buf)
-                for info in in_apk.infolist():
-                    if not APK_SIGNATURE_FILES.match(info.filename):
-                        if info.filename != 'META-INF/MANIFEST.MF':
-                            buf = in_apk.read(info.filename)
-                            out_apk.writestr(info, buf)
-        os.remove(apkpath)
-        _zipalign(apkwithnewsig, apkpath)
+
+    sigdir = os.path.dirname(manifest)  # FIXME
+    apksigcopier.do_patch(sigdir, apkpath, outpath, v1_only=None)
 
 
-def apk_extract_signatures(apkpath, outdir, manifest=True):
+def apk_extract_signatures(apkpath, outdir):
     """Extracts a signature files from APK and puts them into target directory.
 
+    https://docs.oracle.com/javase/tutorial/deployment/jar/intro.html
+    https://source.android.com/security/apksigning/v2
+    https://source.android.com/security/apksigning/v3
+
     :param apkpath: location of the apk
     :param outdir: folder where the extracted signature files will be stored
-    :param manifest: (optionally) disable extracting manifest file
+
     """
-    with ZipFile(apkpath, 'r') as in_apk:
-        for f in in_apk.infolist():
-            if APK_SIGNATURE_FILES.match(f.filename) or \
-                    (manifest and f.filename == 'META-INF/MANIFEST.MF'):
-                newpath = os.path.join(outdir, os.path.basename(f.filename))
-                with open(newpath, 'wb') as out_file:
-                    out_file.write(in_apk.read(f.filename))
+    apksigcopier.do_extract(apkpath, outdir, v1_only=None)
 
 
 def get_min_sdk_version(apk):
@@ -3147,11 +3193,11 @@ def sign_apk(unsigned_path, signed_path, keyalias):
     os.remove(unsigned_path)
 
 
-def verify_apks(signed_apk, unsigned_apk, tmp_dir):
+def verify_apks(signed_apk, unsigned_apk, tmp_dir, v1_only=None):
     """Verify that two apks are the same
 
     One of the inputs is signed, the other is unsigned. The signature metadata
-    is transferred from the signed to the unsigned apk, and then jarsigner is
+    is transferred from the signed to the unsigned apk, and then apksigner is
     used to verify that the signature from the signed APK is also valid for
     the unsigned one.  If the APK given as unsigned actually does have a
     signature, it will be stripped out and ignored.
@@ -3159,53 +3205,38 @@ def verify_apks(signed_apk, unsigned_apk, tmp_dir):
     :param signed_apk: Path to a signed APK file
     :param unsigned_apk: Path to an unsigned APK file expected to match it
     :param tmp_dir: Path to directory for temporary files
+    :param v1_only: True for v1-only signatures, False for v1 and v2 signatures,
+                    or None for autodetection
     :returns: None if the verification is successful, otherwise a string
               describing what went wrong.
     """
 
+    if not verify_apk_signature(signed_apk):
+        logging.info('...NOT verified - {0}'.format(signed_apk))
+        return 'verification of signed APK failed'
+
     if not os.path.isfile(signed_apk):
         return 'can not verify: file does not exists: {}'.format(signed_apk)
-
     if not os.path.isfile(unsigned_apk):
         return 'can not verify: file does not exists: {}'.format(unsigned_apk)
 
-    with ZipFile(signed_apk, 'r') as signed:
-        meta_inf_files = ['META-INF/MANIFEST.MF']
-        for f in signed.namelist():
-            if APK_SIGNATURE_FILES.match(f):
-                meta_inf_files.append(f)
-        if len(meta_inf_files) < 3:
-            return "Signature files missing from {0}".format(signed_apk)
+    tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
 
-        tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
-        with ZipFile(unsigned_apk, 'r') as unsigned:
-            # only read the signature from the signed APK, everything else from unsigned
-            with ZipFile(tmp_apk, 'w') as tmp:
-                for filename in meta_inf_files:
-                    tmp.writestr(signed.getinfo(filename), signed.read(filename))
-                for info in unsigned.infolist():
-                    if info.filename in meta_inf_files:
-                        logging.warning('Ignoring %s from %s', info.filename, unsigned_apk)
-                        continue
-                    if info.filename in tmp.namelist():
-                        return "duplicate filename found: " + info.filename
-                    tmp.writestr(info, unsigned.read(info.filename))
-
-    # Use jarsigner to verify the v1 signature on the reproduced APK, as
-    # apksigner will reject the reproduced APK if the original also had a v2
-    # signature
     try:
-        verify_jar_signature(tmp_apk)
-        verified = True
-    except Exception:
-        verified = False
+        apksigcopier.do_copy(signed_apk, unsigned_apk, tmp_apk, v1_only=v1_only)
+    except apksigcopier.APKSigCopierError as e:
+        logging.info('...NOT verified - {0}'.format(tmp_apk))
+        return 'signature copying failed: {}'.format(str(e))
 
-    if not verified:
-        logging.info("...NOT verified - {0}".format(tmp_apk))
-        return compare_apks(signed_apk, tmp_apk, tmp_dir,
-                            os.path.dirname(unsigned_apk))
+    if not verify_apk_signature(tmp_apk):
+        logging.info('...NOT verified - {0}'.format(tmp_apk))
+        result = compare_apks(signed_apk, tmp_apk, tmp_dir,
+                              os.path.dirname(unsigned_apk))
+        if result is not None:
+            return result
+        return 'verification of APK with copied signature failed'
 
-    logging.info("...successfully verified")
+    logging.info('...successfully verified')
     return None
 
 

fdroidserver/publish.py

@@ -3,6 +3,7 @@
 # publish.py - part of the FDroid server tools
 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
 # Copyright (C) 2013-2014 Daniel Martí <mvdan@mvdan.cc>
+# Copyright (C) 2021 Felix C. Stegerman <flx@obfusk.net>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -345,16 +346,14 @@ def main():
                 # metadata. This means we're going to prepare both a locally
                 # signed APK and a version signed with the developers key.
 
-                signaturefile, signedfile, manifest = signingfiles
+                signature_file, _ignored, manifest, v2_files = signingfiles
 
-                with open(signaturefile, 'rb') as f:
+                with open(signature_file, 'rb') as f:
                     devfp = common.signer_fingerprint_short(common.get_certificate(f.read()))
                 devsigned = '{}_{}_{}.apk'.format(appid, vercode, devfp)
                 devsignedtmp = os.path.join(tmp_dir, devsigned)
-                shutil.copy(apkfile, devsignedtmp)
 
-                common.apk_implant_signatures(devsignedtmp, signaturefile,
-                                              signedfile, manifest)
+                common.apk_implant_signatures(apkfile, devsignedtmp, manifest=manifest)
                 if common.verify_apk_signature(devsignedtmp):
                     shutil.move(devsignedtmp, os.path.join(output_dir, devsigned))
                 else:

jenkins-build-all

@@ -95,10 +95,25 @@ else
     sed -i '/^wiki_/d' config.yml
 fi
 
+printf '\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\nbuild all with reproducible signatures\n'
+for f in metadata/*/signatures/*; do
+    appid=$(basename $(dirname $(dirname $f)))
+    versionCode=$(basename $f)
+    rm -f repo/${appid}_* archive/${appid}_* unsigned/${appid}_*
+    $WORKSPACE/fdroid build --verbose --latest --no-tarball ${appid}:$versionCode
+done
+
+printf '\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\nbuild all with Binaries:\n'
+for appid in `grep '^Binaries: ' metadata/*.yml --files-with-match | sed 's,^metadata/\(.*\)\.yml$,\1,'`; do
+    rm -f repo/${appid}_* archive/${appid}_* unsigned/${appid}_*
+    $WORKSPACE/fdroid build --verbose --latest --no-tarball ${appid}
+done
+
 # force global timeout to 6 hours
 sed -Ei 's,^(\s+endtime\s*=\s*time\.time\(\))\s*.*,\1 + 6 * 60 * 60  # 6 hours,' \
     $WORKSPACE/fdroidserver/build.py
 
+printf '\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\nbuild all\n'
 $WORKSPACE/fdroid build --verbose --latest --no-tarball --all $wikiflag
 
 vagrant global-status

tests/common.TestCase

@@ -473,7 +473,7 @@ class CommonTest(unittest.TestCase):
                 for info in apk.infolist():
                     testapk.writestr(info, apk.read(info.filename))
                     if info.filename.startswith('META-INF/'):
-                        testapk.writestr(info, otherapk.read(info.filename))
+                        testapk.writestr(info.filename, otherapk.read(info.filename))
         otherapk.close()
         self.assertFalse(fdroidserver.common.verify_apk_signature(twosigapk))
         self.assertIsNone(fdroidserver.common.verify_apks(sourceapk, twosigapk, self.tmpdir))
@@ -1754,6 +1754,42 @@ class CommonTest(unittest.TestCase):
             fdroidserver.common.read_pkg_args(appid_versionCode_pairs, allow_vercodes)
         )
 
+    def test_apk_strip_v1_signatures(self):
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        before = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk')
+        after = os.path.join(testdir, 'after.apk')
+        shutil.copy(before, after)
+        fdroidserver.common.apk_strip_v1_signatures(after, strip_manifest=False)
+
+    def test_metadata_find_developer_signing_files(self):
+        appid = 'org.smssecure.smssecure'
+
+        self.assertIsNone(
+            fdroidserver.common.metadata_find_developer_signing_files(appid, 133)
+        )
+
+        vc = '135'
+        self.assertEqual(
+            (
+                os.path.join('metadata', appid, 'signatures', vc, '28969C09.RSA'),
+                os.path.join('metadata', appid, 'signatures', vc, '28969C09.SF'),
+                os.path.join('metadata', appid, 'signatures', vc, 'MANIFEST.MF'),
+                None
+            ),
+            fdroidserver.common.metadata_find_developer_signing_files(appid, vc)
+        )
+
+        vc = '134'
+        self.assertEqual(
+            (
+                os.path.join('metadata', appid, 'signatures', vc, '28969C09.RSA'),
+                os.path.join('metadata', appid, 'signatures', vc, '28969C09.SF'),
+                os.path.join('metadata', appid, 'signatures', vc, 'MANIFEST.MF'),
+                None
+            ),
+            fdroidserver.common.metadata_find_developer_signing_files(appid, vc)
+        )
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/publish.TestCase

@@ -30,6 +30,8 @@ if localmodule not in sys.path:
 
 from fdroidserver import publish
 from fdroidserver import common
+from fdroidserver import metadata
+from fdroidserver import signatures
 from fdroidserver.exception import FDroidException
 
 
@@ -250,6 +252,71 @@ class PublishTest(unittest.TestCase):
                 self.assertEqual(publish.config['jarsigner'], data['jarsigner'])
                 self.assertEqual(publish.config['keytool'], data['keytool'])
 
+    def test_sign_then_implant_signature(self):
+
+        class Options:
+            verbose = False
+
+        testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
+        os.chdir(testdir)
+
+        config = common.read_config(Options)
+        if 'apksigner' not in config:
+            self.skipTest('SKIPPING test_sign_then_implant_signature, apksigner not installed!')
+        config['repo_keyalias'] = 'sova'
+        config['keystorepass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        config['keypass'] = 'r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI='
+        shutil.copy(os.path.join(self.basedir, 'keystore.jks'), testdir)
+        config['keystore'] = 'keystore.jks'
+        config['keydname'] = 'CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US'
+        publish.config = config
+        common.config = config
+
+        app = metadata.App()
+        app.id = 'org.fdroid.ci'
+        versionCode = 1
+        build = metadata.Build(
+            {
+                'versionCode': versionCode,
+                'versionName': '1.0',
+            }
+        )
+        app.Builds = [build]
+        os.mkdir('metadata')
+        metadata.write_metadata(os.path.join('metadata', '%s.yml' % app.id), app)
+
+        os.mkdir('unsigned')
+        testapk = os.path.join(self.basedir, 'no_targetsdk_minsdk1_unsigned.apk')
+        unsigned = os.path.join('unsigned', common.get_release_filename(app, build))
+        signed = os.path.join('repo', common.get_release_filename(app, build))
+        shutil.copy(testapk, unsigned)
+
+        # sign the unsigned APK
+        self.assertTrue(os.path.exists(unsigned))
+        self.assertFalse(os.path.exists(signed))
+        with mock.patch('sys.argv', ['fdroid publish', '%s:%d' % (app.id, versionCode)]):
+            publish.main()
+        self.assertFalse(os.path.exists(unsigned))
+        self.assertTrue(os.path.exists(signed))
+
+        with mock.patch('sys.argv', ['fdroid signatures', signed]):
+            signatures.main()
+        self.assertTrue(
+            os.path.exists(
+                os.path.join('metadata', 'org.fdroid.ci', 'signatures', '1', 'MANIFEST.MF')
+            )
+        )
+        os.remove(signed)
+
+        # implant the signature into the unsigned APK
+        shutil.copy(testapk, unsigned)
+        self.assertTrue(os.path.exists(unsigned))
+        self.assertFalse(os.path.exists(signed))
+        with mock.patch('sys.argv', ['fdroid publish', '%s:%d' % (app.id, versionCode)]):
+            publish.main()
+        self.assertFalse(os.path.exists(unsigned))
+        self.assertTrue(os.path.exists(signed))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))