Use a dedicated launcher process to run all ProcessCommands

On Linux (and very likely other Unices as well), QProcess::start()
incurs a certain overhead, because forking off the child process
duplicates some of the parent process' resources. This overhead appears
to be proportional to the amount of memory allocated by the parent
process, presumably due to page table entries getting copied. This has
consequences for qbs, particularly when being used from an IDE such as
Qt Creator, which has a higher memory footprint than the command line
tool. When using a high job count, as is typical on machines with lots
of CPU cores or in a distributed compilation environment, the following
problems were observed:
    - High CPU load in the starting process (Qt Creator). Profiling
      showed that most of the time was spent in fork() and related
      functions.
    - As a result, the number of parallel jobs stalled at a value
      well below the requested one, slowing down the build.
    - In some cases, QProcess::start() failed altogether, emitting
      a message such as "fork() failed: Could not allocate memory".
We solve these issues by outsourcing the starting of ProcessCommands to
a dedicated launcher tool with modest memory requirements. For each qbs
process, we have one instance of this tool running while a build job is
going on. Communication with qbs happens via QLocalSocket. The protocol
is encapsulated in a QProcess replacement, so almost no changes to
existing code were necessary.
No performance regressions were observed when using lower job counts.
This patch will also enable us to properly support the incredibuild tool
in IDEs such as Qt Creator, which you do not want to start via
ibconsole.

[ChangeLog] Improved scalability of parallel builds on Linux by starting
Process commands via a dedicated launcher process.

Change-Id: I8966c00a2d67a94c3a380f0e089d61eda122209e
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@qt.io>
Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>

1 files changed, 175 insertions, 0 deletions

diff --git a/src/lib/corelib/tools/launcherpackets.cpp b/src/lib/corelib/tools/launcherpackets.cpp
new file mode 100644
index 000000000..9c131bd69
--- /dev/null
+++ b/src/lib/corelib/tools/launcherpackets.cpp
@@ -0,0 +1,175 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qbs.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "launcherpackets.h"
+
+#include <QtCore/qbytearray.h>
+#include <QtCore/qcoreapplication.h>
+#include <QtCore/qendian.h>
+
+namespace qbs {
+namespace Internal {
+
+LauncherPacket::~LauncherPacket() { }
+
+QByteArray LauncherPacket::serialize() const
+{
+    QByteArray data;
+    QDataStream stream(&data, QIODevice::WriteOnly);
+    stream << static_cast<int>(0) << static_cast<quint8>(type) << token;
+    doSerialize(stream);
+    stream.device()->reset();
+    stream << static_cast<int>(data.size() - sizeof(int));
+    return data;
+}
+
+void LauncherPacket::deserialize(const QByteArray &data)
+{
+    QDataStream stream(data);
+    doDeserialize(stream);
+}
+
+
+StartProcessPacket::StartProcessPacket(quintptr token)
+    : LauncherPacket(LauncherPacketType::StartProcess, token)
+{
+}
+
+void StartProcessPacket::doSerialize(QDataStream &stream) const
+{
+    stream << command << arguments << workingDir << env;
+}
+
+void StartProcessPacket::doDeserialize(QDataStream &stream)
+{
+    stream >> command >> arguments >> workingDir >> env;
+}
+
+
+StopProcessPacket::StopProcessPacket(quintptr token)
+    : LauncherPacket(LauncherPacketType::StopProcess, token)
+{
+}
+
+void StopProcessPacket::doSerialize(QDataStream &stream) const
+{
+    Q_UNUSED(stream);
+}
+
+void StopProcessPacket::doDeserialize(QDataStream &stream)
+{
+    Q_UNUSED(stream);
+}
+
+
+ProcessErrorPacket::ProcessErrorPacket(quintptr token)
+    : LauncherPacket(LauncherPacketType::ProcessError, token)
+{
+}
+
+void ProcessErrorPacket::doSerialize(QDataStream &stream) const
+{
+    stream << static_cast<quint8>(error) << errorString;
+}
+
+void ProcessErrorPacket::doDeserialize(QDataStream &stream)
+{
+    quint8 e;
+    stream >> e;
+    error = static_cast<QProcess::ProcessError>(e);
+    stream >> errorString;
+}
+
+
+ProcessFinishedPacket::ProcessFinishedPacket(quintptr token)
+    : LauncherPacket(LauncherPacketType::ProcessFinished, token)
+{
+}
+
+void ProcessFinishedPacket::doSerialize(QDataStream &stream) const
+{
+    stream << errorString << stdOut << stdErr
+           << static_cast<quint8>(exitStatus) << static_cast<quint8>(error)
+           << exitCode;
+}
+
+void ProcessFinishedPacket::doDeserialize(QDataStream &stream)
+{
+    stream >> errorString >> stdOut >> stdErr;
+    quint8 val;
+    stream >> val;
+    exitStatus = static_cast<QProcess::ExitStatus>(val);
+    stream >> val;
+    error = static_cast<QProcess::ProcessError>(val);
+    stream >> exitCode;
+}
+
+ShutdownPacket::ShutdownPacket() : LauncherPacket(LauncherPacketType::Shutdown, 0) { }
+void ShutdownPacket::doSerialize(QDataStream &stream) const { Q_UNUSED(stream); }
+void ShutdownPacket::doDeserialize(QDataStream &stream) { Q_UNUSED(stream); }
+
+void PacketParser::setDevice(QIODevice *device)
+{
+    m_stream.setDevice(device);
+    m_sizeOfNextPacket = -1;
+}
+
+bool PacketParser::parse()
+{
+    static const int commonPayloadSize = static_cast<int>(1 + sizeof LauncherPacket::token);
+    if (m_sizeOfNextPacket == -1) {
+        if (m_stream.device()->bytesAvailable() < static_cast<int>(sizeof m_sizeOfNextPacket))
+            return false;
+        m_stream >> m_sizeOfNextPacket;
+        if (m_sizeOfNextPacket < commonPayloadSize)
+            throw InvalidPacketSizeException(m_sizeOfNextPacket);
+    }
+    if (m_stream.device()->bytesAvailable() < m_sizeOfNextPacket)
+        return false;
+    quint8 type;
+    m_stream >> type;
+    m_type = static_cast<LauncherPacketType>(type);
+    m_stream >> m_token;
+    m_packetData = m_stream.device()->read(m_sizeOfNextPacket - commonPayloadSize);
+    m_sizeOfNextPacket = -1;
+    return true;
+}
+
+} // namespace Internal
+} // namespace qbs