QBSP: add support for using containers

Update QBSP scripts so that it can use Docker containers if needed.
A linux host QBSP can now be installed also on Windows and macOS hosts,
where it will create a Docker container and register that to QtCreator.
The QBSP can also be set to use container toolchain on linux host with
variable QBSP_FORCE_CONTAINER_TOOLCHAIN = "true".

For ARM hosts, aarch64 toolchain can be used when building with
SDKMACHINE=aarch64. Runnig x86 toolchain on ARM host would use
QEMU which make it too slow for practical use.

Task-number: QTBUG-106692
Change-Id: Iff3f60ce7ad34c4652e1a90aaa28b178e21b3a8b
Reviewed-by: Ari Parkkila <ari.parkkila@qt.io>

7 files changed, 136 insertions, 84 deletions

diff --git a/meta-boot2qt/classes/qbsp.bbclass b/meta-boot2qt/classes/qbsp.bbclass
index 194db6bd..71452618 100644
--- a/meta-boot2qt/classes/qbsp.bbclass
+++ b/meta-boot2qt/classes/qbsp.bbclass
@@ -1,6 +1,6 @@
 ############################################################################
 ##
-## Copyright (C) 2021 The Qt Company Ltd.
+## Copyright (C) 2022 The Qt Company Ltd.
 ## Contact: https://www.qt.io/licensing/
 ##
 ## This file is part of the Boot to Qt meta layer.
@@ -44,8 +44,8 @@ INHIBIT_DEFAULT_DEPS = "1"
 do_qbsp[depends] += "\
     p7zip-native:do_populate_sysroot \
     installer-framework-native:do_populate_sysroot \
-    ${QBSP_SDK_TASK}:do_populate_sdk \
-    ${QBSP_IMAGE_TASK}:do_image_complete \
+    ${@d.getVar('QBSP_SDK_TASK', True) + ':do_populate_sdk' if d.getVar('QBSP_SDK_TASK', True) else ''}  \
+    ${@d.getVar('QBSP_IMAGE_TASK', True) + ':do_image_complete' if d.getVar('QBSP_IMAGE_TASK', True) else ''}  \
     "
 
 QBSP_VERSION ?= "${PV}${VERSION_AUTO_INCREMENT}"
@@ -55,6 +55,12 @@ QBSP_INSTALL_PATH ?= "/Extras/${MACHINE}"
 QBSP_LICENSE_FILE ??= ""
 QBSP_LICENSE_NAME ??= ""
 
+QBSP_FORCE_CONTAINER_TOOLCHAIN ?= "false"
+QBSP_FORCE_CONTAINER_TOOLCHAIN:sdkmingw32 = "false"
+
+TOOLCHAIN_HOST_TYPE = "linux"
+TOOLCHAIN_HOST_TYPE:sdkmingw32 = "windows"
+
 VERSION_AUTO_INCREMENT = "-${DATETIME}"
 VERSION_AUTO_INCREMENT[vardepsexclude] = "DATETIME"
 
@@ -78,7 +84,7 @@ patch_installer_files() {
 
     sed -e "s#@NAME@#${QBSP_NAME}#" \
         -e "s#@TARGET@#${DEPLOY_CONF_NAME}#" \
-        -e "s#@VERSION@#${QBSP_VERSION}#" \
+        -e "s#@QBSP_VERSION@#${QBSP_VERSION}#" \
         -e "s#@RELEASEDATE@#${RELEASEDATE}#" \
         -e "s#@MACHINE@#${MACHINE}#" \
         -e "s#@SYSROOT@#${REAL_MULTIMACH_TARGET_SYS}#" \
@@ -92,23 +98,29 @@ patch_installer_files() {
         -e "s#@LICENSEFILE@#$(basename ${QBSP_LICENSE_FILE})#" \
         -e "s#@LICENSENAME@#${QBSP_LICENSE_NAME}#" \
         -e "s#@TOOLCHAIN_HOST_SYSROOT@#${SDK_SYS}#" \
+        -e "s#@FORCE_CONTAINER_TOOLCHAIN@#${QBSP_FORCE_CONTAINER_TOOLCHAIN}#" \
+        -e "s#@TOOLCHAIN_HOST_TYPE@#${TOOLCHAIN_HOST_TYPE}#" \
+        -e "s#@DOCKER_ARCH@#${@'arm64' if d.getVar('SDKMACHINE') == 'aarch64' else 'amd64'}#" \
+        -e "s#@VERSION@#${PV}#" \
         -i ${1}/*
 }
 
 prepare_qbsp() {
     # Toolchain component
-    COMPONENT_PATH="${B}/pkg/${QBSP_INSTALLER_COMPONENT}.toolchain"
-    mkdir -p ${COMPONENT_PATH}/meta
-    mkdir -p ${COMPONENT_PATH}/data
+    if [ -e ${DEPLOY_DIR}/sdk/${SDK_NAME} ]; then
+        COMPONENT_PATH="${B}/pkg/${QBSP_INSTALLER_COMPONENT}.toolchain"
+        mkdir -p ${COMPONENT_PATH}/meta
+        mkdir -p ${COMPONENT_PATH}/data
 
-    cp ${WORKDIR}/toolchain_package.xml ${COMPONENT_PATH}/meta/package.xml
-    cp ${WORKDIR}/toolchain_installscript.qs ${COMPONENT_PATH}/meta/installscript.qs
-    patch_installer_files ${COMPONENT_PATH}/meta
+        cp ${WORKDIR}/toolchain_package.xml ${COMPONENT_PATH}/meta/package.xml
+        cp ${WORKDIR}/toolchain_installscript.qs ${COMPONENT_PATH}/meta/installscript.qs
+        patch_installer_files ${COMPONENT_PATH}/meta
 
-    if [ "${SDK_POSTFIX}" = "${SDK_POSTFIX:sdkmingw32}" ]; then
-        cp ${DEPLOY_DIR}/sdk/${SDK_NAME} ${COMPONENT_PATH}/data/toolchain.${SDK_POSTFIX}
-    else
-        7za a -mx=0 ${COMPONENT_PATH}/data/toolchain.7z ${DEPLOY_DIR}/sdk/${SDK_NAME}
+        if [ "${SDK_POSTFIX}" = "${SDK_POSTFIX:sdkmingw32}" ]; then
+            cp ${DEPLOY_DIR}/sdk/${SDK_NAME} ${COMPONENT_PATH}/data/toolchain.${SDK_POSTFIX}
+        else
+            7za a -mx=0 ${COMPONENT_PATH}/data/toolchain.7z ${DEPLOY_DIR}/sdk/${SDK_NAME}
+        fi
     fi
 
     # Image component, only if we have the qbsp-image
diff --git a/meta-boot2qt/files/qbsp/base_installscript.qs b/meta-boot2qt/files/qbsp/base_installscript.qs
index e6c5fbce..65d68399 100644
--- a/meta-boot2qt/files/qbsp/base_installscript.qs
+++ b/meta-boot2qt/files/qbsp/base_installscript.qs
@@ -1,6 +1,6 @@
 /****************************************************************************
 **
-** Copyright (C) 2018 The Qt Company Ltd.
+** Copyright (C) 2022 The Qt Company Ltd.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the Boot to Qt meta layer.
@@ -27,32 +27,23 @@
 **
 ****************************************************************************/
 
-var targetHost = "Linux";
-var currentHost = "Linux";
-
 function Component()
 {
-    if ("@SDKFILE@".indexOf("mingw32") >= 0)
-        targetHost = "Windows";
-
-    if (systemInfo.kernelType === "winnt")
-        currentHost = "Windows";
-    else if (systemInfo.kernelType === "darwin")
-        currentHost = "macOS";
-
-    if (currentHost != targetHost) {
+    if ("@TOOLCHAIN_HOST_TYPE@" == "windows" && systemInfo.kernelType !== "winnt") {
         component.enabled = false;
         component.setValue("Default", false);
-        installer.componentByName(component.name + ".toolchain").setValue("Default", false);
-        installer.componentByName(component.name + ".toolchain").enabled = false;
-        installer.componentByName(component.name + ".system").setValue("Default", false);
-        installer.componentByName(component.name + ".system").enabled = false;
 
-        gui.currentPageWidget().completeChanged.connect(this, Component.prototype.completeChanged)
-    }
-}
+        var toolchain = installer.componentByName(component.name + ".toolchain")
+        if (toolchain) {
+            toolchain.setValue("Default", false);
+            toolchain.enabled = false;
+        }
+        var system = installer.componentByName(component.name + ".system")
+        if (system) {
+            system.setValue("Default", false);
+            system.enabled = false;
+        }
 
-Component.prototype.completeChanged = function ()
-{
-    QMessageBox.critical("error", "Invalid QBSP package", "The selected QBSP supports only " + targetHost + " host platform.\nPlease restart the installer before continuing.");
+        QMessageBox.critical("error", "Invalid QBSP package", "The selected QBSP supports only Windows host platform.\nPlease restart the installer before continuing.");
+    }
 }
diff --git a/meta-boot2qt/files/qbsp/base_package.xml b/meta-boot2qt/files/qbsp/base_package.xml
index 57b50764..3c64c66d 100644
--- a/meta-boot2qt/files/qbsp/base_package.xml
+++ b/meta-boot2qt/files/qbsp/base_package.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 <Package>
-  <Version>@VERSION@</Version>
+  <Version>@QBSP_VERSION@</Version>
   <ReleaseDate>@RELEASEDATE@</ReleaseDate>
   <DisplayName>@TARGET@</DisplayName>
   <Description>@NAME@ for @TARGET@</Description>
diff --git a/meta-boot2qt/files/qbsp/image_package.xml b/meta-boot2qt/files/qbsp/image_package.xml
index e7d25cbb..467423b7 100644
--- a/meta-boot2qt/files/qbsp/image_package.xml
+++ b/meta-boot2qt/files/qbsp/image_package.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 <Package>
-  <Version>@VERSION@</Version>
+  <Version>@QBSP_VERSION@</Version>
   <ReleaseDate>@RELEASEDATE@</ReleaseDate>
   <DisplayName>@TARGET@ Linux System</DisplayName>
   <Description>System image for @TARGET@ hardware</Description>
diff --git a/meta-boot2qt/files/qbsp/license_package.xml b/meta-boot2qt/files/qbsp/license_package.xml
index 405ee5c0..a700f976 100644
--- a/meta-boot2qt/files/qbsp/license_package.xml
+++ b/meta-boot2qt/files/qbsp/license_package.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <Package>
     <DisplayName>@LICENSENAME@</DisplayName>
-    <Version>@VERSION@</Version>
+    <Version>@QBSP_VERSION@</Version>
     <ReleaseDate>@RELEASEDATE@</ReleaseDate>
     <Licenses>
         <License name="@LICENSENAME@" file="@LICENSEFILE@"/>
diff --git a/meta-boot2qt/files/qbsp/toolchain_installscript.qs b/meta-boot2qt/files/qbsp/toolchain_installscript.qs
index acf96682..aee2563a 100644
--- a/meta-boot2qt/files/qbsp/toolchain_installscript.qs
+++ b/meta-boot2qt/files/qbsp/toolchain_installscript.qs
@@ -1,6 +1,6 @@
 /****************************************************************************
 **
-** Copyright (C) 2021 The Qt Company Ltd.
+** Copyright (C) 2022 The Qt Company Ltd.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the Boot to Qt meta layer.
@@ -36,34 +36,57 @@ Component.prototype.createOperations = function()
     component.createOperations();
 
     var device = "@MACHINE@";
-    var platform = "@NAME@";
+    var platform = "@NAME@ @TARGET@";
     var sysroot = "@SYSROOT@";
     var target_sys = "@TARGET_SYS@";
-    var target = "@TARGET@";
     var abi = "@ABI@-linux-poky-elf-@BITS@bit";
     var installPath = "@INSTALLPATH@/toolchain";
     var sdkPath = "@SDKPATH@";
     var sdkFile = "@SDKFILE@";
     var hostSysroot = "@TOOLCHAIN_HOST_SYSROOT@";
+    var imageTag = "boot2qt-@MACHINE@:@VERSION@"
+    var dockerPrefix = "";
+
+    var container = false;
+    if ( "@TOOLCHAIN_HOST_TYPE@" == "linux" && systemInfo.kernelType !== "linux" || @FORCE_CONTAINER_TOOLCHAIN@)
+       container = true;
 
     var path = installer.value("TargetDir") + installPath;
-    if (systemInfo.kernelType !== "winnt") {
-        var script = path + "/" + sdkFile;
-        component.addOperation("Execute", "{0}", "chmod", "+x", script);
-        component.addOperation("Execute", "{0}", script, "-y", "-d", path, "UNDOEXECUTE", "rm", "-rf", path);
-        component.addOperation("Execute", "{0}", "/bin/rm", script);
+    if (!container) {
+        if (systemInfo.kernelType !== "winnt") {
+            var script = path + "/" + sdkFile;
+            component.addOperation("Execute", "{0}", "sh", script, "-y", "-d", path, "UNDOEXECUTE", "rm", "-rf", path);
+            component.addOperation("Execute", "{0}", "/bin/rm", script);
+        } else {
+            // workaround for QTIFW-2344
+            path = path.replace(/\\/g, "/");
+        }
     } else {
-        // workaround for QTIFW-2344
-        path = path.replace(/\\/g ,"/");
+        component.addOperation("AppendFile", path + "/Dockerfile",
+            "\
+FROM --platform=linux/@DOCKER_ARCH@ ubuntu:20.04\n\
+ENV LANG C.UTF-8\n\
+RUN apt-get update && DEBIAN_FRONTEND=\"noninteractive\" apt-get install -y --no-install-recommends python3 xz-utils file make && rm -rf /var/lib/apt/lists/*\n\
+COPY *.sh /\n\
+RUN sh *.sh -d /opt/toolchain -y && rm *.sh\n");
+
+        component.addOperation("Execute", [
+            "docker",
+            "build", path,
+            "-t", imageTag,
+            "errormessage=Installer was unable to run docker. " +
+            "Make sure Docker is installed and running before continuing.\n\n" +
+            "The toolchain Docker container can also be created manually by running command:\n" +
+            "docker build " + path + " -t " + imageTag,
+            "UNDOEXECUTE",
+            "docker", "image", "rm", "-f", imageTag]);
+        path = "/opt/toolchain";
+        dockerPrefix = "docker://" + imageTag;
     }
-    var basecomponent = component.name.substring(0, component.name.lastIndexOf("."));
+
     var toolchainId = "ProjectExplorer.ToolChain.Gcc:" + component.name;
-    var debuggerId = basecomponent + ".gdb";
-    var qtId = basecomponent + ".qt";
-    var cmakeId = basecomponent + ".cmake";
-    var icon = installer.value("B2QtDeviceIcon");
     var executableExt = "";
-    if (systemInfo.kernelType === "winnt") {
+    if (!container && systemInfo.kernelType === "winnt") {
         executableExt = ".exe";
         toolchainId = "ProjectExplorer.ToolChain.Mingw:" + component.name;
     }
@@ -76,8 +99,8 @@ Component.prototype.createOperations = function()
     component.addOperation("Execute",
         ["@SDKToolBinary@", "addTC",
         "--id", toolchainId + ".gcc",
-        "--name", "GCC (" + platform + " " + target + ")",
-        "--path", path + "/sysroots/" + hostSysroot + "/usr/bin/" + target_sys + "/" + target_sys + "-gcc" + executableExt,
+        "--name", "GCC (" + platform + ")",
+        "--path", dockerPrefix + path + "/sysroots/" + hostSysroot + "/usr/bin/" + target_sys + "/" + target_sys + "-gcc" + executableExt,
         "--abi", abi,
         "--language", "C",
         "UNDOEXECUTE",
@@ -86,8 +109,8 @@ Component.prototype.createOperations = function()
     component.addOperation("Execute",
         ["@SDKToolBinary@", "addTC",
         "--id", toolchainId + ".g++",
-        "--name", "G++ (" + platform + " " + target + ")",
-        "--path", path + "/sysroots/" + hostSysroot + "/usr/bin/" + target_sys + "/" + target_sys + "-g++" + executableExt,
+        "--name", "G++ (" + platform + ")",
+        "--path", dockerPrefix + path + "/sysroots/" + hostSysroot + "/usr/bin/" + target_sys + "/" + target_sys + "-g++" + executableExt,
         "--abi", abi,
         "--language", "Cxx",
         "UNDOEXECUTE",
@@ -95,52 +118,78 @@ Component.prototype.createOperations = function()
 
     component.addOperation("Execute",
         ["@SDKToolBinary@", "addDebugger",
-        "--id", debuggerId,
-        "--name", "GDB (" + platform + " " + target + ")",
+        "--id", component.name,
+        "--name", "GDB (" + platform + ")",
         "--engine", "1",
-        "--binary", path + "/sysroots/" + hostSysroot + "/usr/bin/" + target_sys + "/" + target_sys + "-gdb" + executableExt,
+        "--binary", dockerPrefix + path + "/sysroots/" + hostSysroot + "/usr/bin/" + target_sys + "/" + target_sys + "-gdb" + executableExt,
         "--abis", abi,
         "UNDOEXECUTE",
-        "@SDKToolBinary@", "rmDebugger", "--id", debuggerId]);
+        "@SDKToolBinary@", "rmDebugger", "--id", component.name]);
 
     component.addOperation("Execute",
         ["@SDKToolBinary@", "addQt",
-         "--id", qtId,
-         "--name", platform + " " + target,
+         "--id", component.name,
+         "--name", platform,
          "--type", "Qdb.EmbeddedLinuxQt",
-         "--qmake", path + "/sysroots/" + hostSysroot + "/usr/bin/qmake" + executableExt,
+         "--qmake", dockerPrefix + path + "/sysroots/" + hostSysroot + "/usr/bin/qmake" + executableExt,
          "--abis", abi,
          "UNDOEXECUTE",
-         "@SDKToolBinary@", "rmQt", "--id", qtId]);
+         "@SDKToolBinary@", "rmQt", "--id", component.name]);
 
     component.addOperation("Execute",
         ["@SDKToolBinary@", "addCMake",
-        "--id", cmakeId,
-        "--name", "CMake (" + platform + " " + target + ")",
-        "--path", path + "/sysroots/" + hostSysroot + "/usr/bin/cmake" + executableExt,
+        "--id", component.name,
+        "--name", "CMake (" + platform + ")",
+        "--path", dockerPrefix + path + "/sysroots/" + hostSysroot + "/usr/bin/cmake" + executableExt,
         "UNDOEXECUTE",
-        "@SDKToolBinary@", "rmCMake", "--id", cmakeId]);
+        "@SDKToolBinary@", "rmCMake", "--id", component.name]);
 
-    component.addOperation("Execute",
+    if (container) {
+        component.addOperation("Execute",
+            ["@SDKToolBinary@", "addDev",
+            "--id", component.name,
+            "--name", "Docker Image (" + platform + ")",
+            "--type", "0",
+            "--osType", "DockerDeviceType",
+            "--origin", "1",
+            "--dockerRepo", "boot2qt-@MACHINE@",
+            "--dockerTag", "@VERSION@",
+            "--dockerMappedPaths", installer.value("TargetDir"),
+            "UNDOEXECUTE",
+            "@SDKToolBinary@", "rmDev", "--id", component.name]);
+    }
+
+    var addKitOperations =
         ["@SDKToolBinary@", "addKit",
-         "--id", basecomponent,
-         "--name", platform + " " + target,
-         "--mkspec", "linux-oe-g++",
-         "--qt", qtId,
-         "--debuggerid", debuggerId,
+         "--id", component.name,
+         "--name", platform,
+         "--qt", component.name,
+         "--debuggerid", component.name,
          "--sysroot", path + "/sysroots/" + sysroot,
          "--devicetype", "QdbLinuxOsType",
          "--Ctoolchain", toolchainId + ".gcc",
          "--Cxxtoolchain", toolchainId + ".g++",
-         "--icon", icon,
-         "--cmake", cmakeId,
+         "--cmake", component.name,
          "--cmake-generator", "Ninja",
          "--cmake-config", "CMAKE_CXX_COMPILER:STRING=%{Compiler:Executable:Cxx}",
          "--cmake-config", "CMAKE_C_COMPILER:STRING=%{Compiler:Executable:C}",
          "--cmake-config", "CMAKE_PREFIX_PATH:STRING=%{Qt:QT_INSTALL_PREFIX}",
          "--cmake-config", "QT_QMAKE_EXECUTABLE:STRING=%{Qt:qmakeExecutable}",
          "--cmake-config", "CMAKE_TOOLCHAIN_FILE:FILEPATH=" + path + "/sysroots/" + hostSysroot + "/usr/lib/cmake/Qt6/qt.toolchain.cmake",
-         "--cmake-config", "CMAKE_MAKE_PROGRAM:FILEPATH=" + path + "/sysroots/" + hostSysroot + "/usr/bin/ninja" + executableExt,
-         "UNDOEXECUTE",
-         "@SDKToolBinary@", "rmKit", "--id", basecomponent]);
+         "--cmake-config", "CMAKE_MAKE_PROGRAM:FILEPATH=" + path + "/sysroots/"+ hostSysroot + "/usr/bin/ninja" + executableExt];
+
+    if (container) {
+        addKitOperations.push("--builddevice", component.name);
+    }
+    if (!container) {
+        addKitOperations.push("--mkspec", "linux-oe-g++");
+    }
+
+    addKitOperations.push("UNDOEXECUTE", "@SDKToolBinary@", "rmKit", "--id", component.name);
+    component.addOperation("Execute", addKitOperations);
+
+    if (container) {
+        var settingsFile = installer.value("QtCreatorInstallerSettingsFile");
+        component.addOperation("Settings", "path="+settingsFile, "method=add_array_value", "key=Plugins/ForceEnabled", "value=Docker");
+    }
 }
diff --git a/meta-boot2qt/files/qbsp/toolchain_package.xml b/meta-boot2qt/files/qbsp/toolchain_package.xml
index fc1667dc..8e2f36bd 100644
--- a/meta-boot2qt/files/qbsp/toolchain_package.xml
+++ b/meta-boot2qt/files/qbsp/toolchain_package.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 <Package>
-  <Version>@VERSION@</Version>
+  <Version>@QBSP_VERSION@</Version>
   <ReleaseDate>@RELEASEDATE@</ReleaseDate>
   <DisplayName>@TARGET@ Linux Toolchain</DisplayName>
   <Description>Toolchain and system root for @TARGET@</Description>