Add support for self-contained update packages

Task-number: QTBUG-54500
Change-Id: Ia735e85c0c92cbbebd0098b9021ff7d04b3e246b
Reviewed-by: Gatis Paeglis <gatis.paeglis@qt.io>

3 files changed, 107 insertions, 4 deletions

diff --git a/qt-ostree/build-ostree.sh b/qt-ostree/build-ostree.sh
index d01bf35..cfcf621 100755
--- a/qt-ostree/build-ostree.sh
+++ b/qt-ostree/build-ostree.sh
@@ -50,6 +50,7 @@ git am "${DIR}"/patches/Support-for-booting-without-initramfs.patch
 git am "${DIR}"/patches/Allow-updating-files-in-the-boot-directory.patch
 git am "${DIR}"/patches/u-boot-add-bootdir-to-the-generated-uEnv.txt.patch
 git am "${DIR}"/patches/Create-firmware-convenience-symlinks.patch
+git am "${DIR}"/patches/deltas-Expose-the-filename-parameter.patch
 
 ./autogen.sh \
     --with-soup \
diff --git a/qt-ostree/patches/deltas-Expose-the-filename-parameter.patch b/qt-ostree/patches/deltas-Expose-the-filename-parameter.patch
new file mode 100644
index 0000000..cb8fa8b
--- /dev/null
+++ b/qt-ostree/patches/deltas-Expose-the-filename-parameter.patch
@@ -0,0 +1,45 @@
+From 4537b1d2c89d3c9b24f0c08874ec6a1f720add0a Mon Sep 17 00:00:00 2001
+From: Gatis Paeglis <gatis.paeglis@qt.io>
+Date: Thu, 24 Nov 2016 15:59:36 +0100
+Subject: [PATCH] deltas: Expose the filename parameter
+
+The C API (ostree_repo_static_delta_generate) knows what to do
+with it, but this parameter was never exposed via command line
+tool.
+---
+ src/ostree/ot-builtin-static-delta.c | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c
+index a1c220b..57799d4 100644
+--- a/src/ostree/ot-builtin-static-delta.c
++++ b/src/ostree/ot-builtin-static-delta.c
+@@ -33,6 +33,7 @@ static char *opt_min_fallback_size;
+ static char *opt_max_bsdiff_size;
+ static char *opt_max_chunk_size;
+ static char *opt_endianness;
++static char *opt_filename;
+ static gboolean opt_empty;
+ static gboolean opt_swap_endianness;
+ static gboolean opt_inline;
+@@ -71,6 +72,7 @@ static GOptionEntry generate_options[] = {
+   { "min-fallback-size", 0, 0, G_OPTION_ARG_STRING, &opt_min_fallback_size, "Minimum uncompressed size in megabytes for individual HTTP request", NULL},
+   { "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to consider bsdiff compression for input files", NULL},
+   { "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in megabytes", NULL},
++  { "filename", 0, 0, G_OPTION_ARG_STRING, &opt_filename, "Sets where to store the delta. By default gets stored in the ostree repository", NULL},
+   { NULL }
+ };
+ 
+@@ -328,6 +330,9 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
+       if (opt_inline)
+         g_variant_builder_add (parambuilder, "{sv}",
+                                "inline-parts", g_variant_new_boolean (TRUE));
++      if (opt_filename)
++        g_variant_builder_add (parambuilder, "{sv}",
++                               "filename", g_variant_new_bytestring (opt_filename));
+ 
+       g_variant_builder_add (parambuilder, "{sv}", "verbose", g_variant_new_boolean (TRUE));
+       if (opt_endianness || opt_swap_endianness)
+-- 
+2.7.4
+
diff --git a/qt-ostree/qt-ostree b/qt-ostree/qt-ostree
index 7ab3709..318936a 100755
--- a/qt-ostree/qt-ostree
+++ b/qt-ostree/qt-ostree
@@ -63,6 +63,12 @@ FIRST_COMMIT=false
 OSTREE_REPO=${WORKDIR}/ostree-repo
 OSTREE_BRANCH=linux/qt
 OSTREE_COMMIT_SUBJECT=""
+# STATIC DELTA
+STATIC_DELTA_ARGS=""
+SELF_CONTAINED_PACKAGE=false
+SUPERBLOCK=${WORKDIR}/superblock
+DELTA_FROM=${OSTREE_BRANCH}^
+DELTA_TO=${OSTREE_BRANCH}
 # TLS
 USE_CLIENT_TLS=false
 SERVER_CERT=""
@@ -85,7 +91,7 @@ RECOVERY_PARTITION=false
 RECOVERYFS_TYPE="ext2"
 RECOVERYFS_OPT="-F -L recovery"
 RECOVERYFS_SIZE="65536"         # Recovery partition size [in KiB]
-ROOT_OVERHEAD_FACTOR="1.3"      # Add 30% extra space in rootfs
+ROOT_OVERHEAD_FACTOR="3"
 PARTITION_ALIGNMENT="4096"      # Set alignment to 4MB [in KiB]
 # LOG COLORS
 COLOR_DEFAULT='\e[0m'
@@ -189,6 +195,19 @@ usage()
     echo "    exist, one is created in the specified location. If this argument is not"
     echo "    provided, a default repository is created in the working directory."
     echo
+    echo "--create-self-contained-package"
+    echo
+    echo "    Creates a self-contained (superblock) update package. This package is saved in the"
+    echo "    current working directory."
+    echo
+    echo "--disable-bsdiff"
+    echo
+    echo "    The bsdiff algorithm produces smaller updates by taking advantage of how executable"
+    echo "    files change. Smaller updates mean a better download efficiency. Applying bsdiff deltas"
+    echo "    on the client devices requires extra CPU and memory resources, as well as I/O requests."
+    echo "    When generating an update for resource-constrained devices it might be desirable"
+    echo "    to disable bsdiff."
+    echo
 
     # NOTE: Disabling, as we don't really use them at the moment.
     #echo "--ostree-branch os/branch-name        Commits the generated update in the specified OSTree branch. A default branch is linux/qt."
@@ -347,6 +366,12 @@ parse_args()
               OSTREE_REPO=$(realpath -ms ${2})
               shift 1
               ;;
+          --create-self-contained-package)
+              SELF_CONTAINED_PACKAGE=true
+              ;;
+          --disable-bsdiff)
+              STATIC_DELTA_ARGS="--disable-bsdiff"
+              ;;
           --uboot-env-file)
               UBOOT_ENV_FILE=$(realpath -ms ${2})
               shift 1
@@ -476,6 +501,13 @@ parse_args()
         validation_error "Must specify both --tls-client-cert-path and --tls-client-key-path for TLS client authentication feature."
     fi
 
+    if [ ! -d ${OSTREE_REPO}/objects ] ; then
+        FIRST_COMMIT=true
+    fi
+    if [[ $FIRST_COMMIT = true && $SELF_CONTAINED_PACKAGE = true ]] ; then
+        validation_error "Can not generate a self-contained package (--create-self-contained-package), when --ostree-repo points to a non-existing repository."
+    fi
+
     if [ $INVALID_ARGS = true ] ; then
         usage
         qt_ostree_exit 1
@@ -732,12 +764,25 @@ convert_to_ostree_sysroot()
     fi
 }
 
+create_self_contained_package()
+{
+    qt_ostree_info "Generating a self-contained update package ..."
+    # Disable fallback objects, so all objects would be included in the generated
+    # delta and applying the delta would not require an Internet connection.
+    "${OSTREE}" --repo=${OSTREE_REPO} static-delta generate  ${STATIC_DELTA_ARGS} \
+                --min-fallback-size=0 --inline --filename=${SUPERBLOCK}
+
+    if [ ! -e "${SUPERBLOCK}" ] ; then
+        qt_ostree_error "Something failed, ${SUPERBLOCK} does not exist"
+    fi
+    qt_ostree_info "Generated a self-contained update package: ${SUPERBLOCK}"
+}
+
 commit_generated_tree()
 {
     # Commit the generated tree into OSTree repository.
-    mkdir -p ${OSTREE_REPO}
-    if [ ! -d ${OSTREE_REPO}/objects ] ; then
-        FIRST_COMMIT=true
+    if [ $FIRST_COMMIT = true ] ; then
+        mkdir -p ${OSTREE_REPO}
         qt_ostree_info "Initializing new OSTree repository in ${OSTREE_REPO}"
         "${OSTREE}" --repo=${OSTREE_REPO} init --mode=archive-z2
     fi
@@ -766,6 +811,18 @@ commit_generated_tree()
         qt_ostree_exit 0
     fi
 
+    STATIC_DELTA_ARGS="${STATIC_DELTA_ARGS} --from=${DELTA_FROM} --to=${DELTA_TO}"
+    if [ $SELF_CONTAINED_PACKAGE = true ] ; then
+        create_self_contained_package
+    fi
+    # Pulling static deltas via HTTP is on hold due to several UX issues: https://github.com/ostreedev/ostree/issues/475
+    # if [ $FIRST_COMMIT = false ] ; then
+    #     qt_ostree_info "Generating static delta  ..."
+    #     "${OSTREE}" --repo=${OSTREE_REPO} static-delta generate ${STATIC_DELTA_ARGS}
+    # fi
+
+    "${OSTREE}" --repo=${OSTREE_REPO} summary -u ${GPG_ARGS}
+
     qt_ostree_info "Checking the repository for consistency ..."
     if [ $DEVELOPER = false ] ; then
         "${OSTREE}" --repo=${OSTREE_REPO} fsck