QProcess/Unix: add setUnixProcessParameters()

This commit adds those three flags that are either frequent enough or
difficult to do: close all file descriptors above stderr and reset the
signal handlers. Setting SIGPIPE to be ignored isn't critical, but is
required when the ResetSignalHandlers flag is used, as this is run
after the user child process modifier.

[ChangeLog][QtCore][QProcess] Added setUnixProcessParameters() function
that can be used to modify certain settings of the child process,
without the need to provide a callback using setChildProcessModifier().

Change-Id: Icfe44ecf285a480fafe4fffd174d0d1d63840403
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

9 files changed, 404 insertions, 8 deletions

diff --git a/src/corelib/configure.cmake b/src/corelib/configure.cmake
index 8e1b9dc37b..dd89b8ac99 100644
--- a/src/corelib/configure.cmake
+++ b/src/corelib/configure.cmake
@@ -98,6 +98,18 @@ clock_gettime(CLOCK_MONOTONIC, &ts);
 }
 ")
 
+# close_range
+qt_config_compile_test(close_range
+    LABEL "close_range()"
+    CODE
+"#include <unistd.h>
+
+int main()
+{
+    return close_range(3, 1024, 0) != 0;
+}
+")
+
 # cloexec
 qt_config_compile_test(cloexec
     LABEL "O_CLOEXEC"
@@ -551,6 +563,11 @@ qt_feature("clock-monotonic" PUBLIC
     CONDITION QT_FEATURE_clock_gettime AND TEST_clock_monotonic
 )
 qt_feature_definition("clock-monotonic" "QT_NO_CLOCK_MONOTONIC" NEGATE VALUE "1")
+qt_feature("close_range" PRIVATE
+    LABEL "close_range()"
+    CONDITION QT_FEATURE_process AND TEST_close_range
+    AUTODETECT UNIX
+)
 qt_feature("doubleconversion" PRIVATE
     LABEL "DoubleConversion"
 )
diff --git a/src/corelib/io/qprocess.cpp b/src/corelib/io/qprocess.cpp
index 7f3cc7bd89..ddcd7a4768 100644
--- a/src/corelib/io/qprocess.cpp
+++ b/src/corelib/io/qprocess.cpp
@@ -805,6 +805,58 @@ void QProcessPrivate::Channel::clear()
 */
 
 /*!
+    \class QProcess::UnixProcessParameters
+    \inmodule QtCore
+    \note This struct is only available on Unix platforms
+    \since 6.6
+
+    This struct can be used to pass extra, Unix-specific configuration for the
+    child process using QProcess::setUnixProcessParameters().
+
+    Its members are:
+    \list
+    \li UnixProcessParameters::flags    Flags, see QProcess::UnixProcessFlags
+    \endlist
+
+    All of the settings above can also be manually achieved by calling the
+    respective POSIX function from a handler set with
+    QProcess::setChildProcessModifier(). This structure allows QProcess to deal
+    with any platform-specific differences, benefit from certain optimizations,
+    and reduces code duplication. Moreover, if any of those functions fail,
+    QProcess will enter QProcess::FailedToStart state, while the child process
+    modifier callback is not allowed to fail.
+
+    \sa QProcess::setUnixProcessParameters(), QProcess::setChildProcessModifier()
+*/
+
+/*!
+    \enum QProcess::UnixProcessFlags
+    \since 6.6
+
+    These flags can be used in the \c flags field of \l UnixProcessParameters.
+
+    \value CloseNonStandardFileDescriptors  Close all file descriptors besides
+           \c stdin, \c stdout, and \c stderr, preventing any currently open
+           descriptor in the parent process from accidentally leaking to the
+           child.
+
+    \value IgnoreSigPipe    Always sets the \c SIGPIPE signal to ignored
+           (\c SIG_IGN), even if the \c ResetSignalHandlers flag was set. By
+           default, if the child attempts to write to its standard output or
+           standard error after the respective channel was closed with
+           QProcess::closeReadChannel(), it would get the \c SIGPIPE signal and
+           terminate immediately; with this flag, the write operation fails
+           without a signal and the child may continue executing.
+
+    \value ResetSignalHandlers  Resets all Unix signal handlers back to their
+           default state (that is, pass \c SIG_DFL to \c{signal(2)}). This flag
+           is useful to ensure any ignored (\c SIG_IGN) signal does not affect
+           the child's behavior.
+
+    \sa setUnixProcessParameters(), unixProcessParameters()
+*/
+
+/*!
     \fn void QProcess::errorOccurred(QProcess::ProcessError error)
     \since 5.6
 
@@ -1553,7 +1605,7 @@ void QProcess::setCreateProcessArgumentsModifier(CreateProcessArgumentModifier m
 
     \note This function is only available on Unix platforms.
 
-    \sa setChildProcessModifier()
+    \sa setChildProcessModifier(), unixProcessParameters()
 */
 std::function<void(void)> QProcess::childProcessModifier() const
 {
@@ -1567,12 +1619,9 @@ std::function<void(void)> QProcess::childProcessModifier() const
     Sets the \a modifier function for the child process, for Unix systems
     (including \macos; for Windows, see setCreateProcessArgumentsModifier()).
     The function contained by the \a modifier argument will be invoked in the
-    child process after \c{fork()} or \c{vfork()} is completed and QProcess has set up the
-    standard file descriptors for the child process, but before \c{execve()},
-    inside start(). The modifier is useful to change certain properties of the
-    child process, such as setting up additional file descriptors or closing
-    others, changing the nice level, disconnecting from the controlling TTY,
-    etc.
+    child process after \c{fork()} or \c{vfork()} is completed and QProcess has
+    set up the standard file descriptors for the child process, but before
+    \c{execve()}, inside start().
 
     The following shows an example of setting up a child process to run without
     privileges:
@@ -1582,13 +1631,22 @@ std::function<void(void)> QProcess::childProcessModifier() const
     If the modifier function needs to exit the process, remember to use
     \c{_exit()}, not \c{exit()}.
 
+    Certain properties of the child process, such as closing all extraneous
+    file descriptors or disconnecting from the controlling TTY, can be more
+    readily achieved by using setUnixProcessParameters(), which can detect
+    failure and report a \l{QProcess::}{FailedToStart} condition. The modifier
+    is useful to change certain uncommon properties of the child process, such
+    as setting up additional file descriptors. If both a child process modifier
+    and Unix process parameters are set, the modifier is run before these
+    parameters are applied.
+
     \note In multithreaded applications, this function must be careful not to
     call any functions that may lock mutexes that may have been in use in
     other threads (in general, using only functions defined by POSIX as
     "async-signal-safe" is advised). Most of the Qt API is unsafe inside this
     callback, including qDebug(), and may lead to deadlocks.
 
-    \sa childProcessModifier()
+    \sa childProcessModifier(), setUnixProcessParameters()
 */
 void QProcess::setChildProcessModifier(const std::function<void(void)> &modifier)
 {
@@ -1597,6 +1655,67 @@ void QProcess::setChildProcessModifier(const std::function<void(void)> &modifier
         d->unixExtras.reset(new QProcessPrivate::UnixExtras);
     d->unixExtras->childProcessModifier = modifier;
 }
+
+/*!
+    \since 6.6
+    Returns the \l UnixProcessParameters object describing extra flags and
+    settings that will be applied to the child process on Unix systems. The
+    default settings correspond to a default-constructed UnixProcessParameters.
+
+    \note This function is only available on Unix platforms.
+
+    \sa childProcessModifier()
+*/
+auto QProcess::unixProcessParameters() const noexcept -> UnixProcessParameters
+{
+    Q_D(const QProcess);
+    return d->unixExtras ? d->unixExtras->processParameters : UnixProcessParameters{};
+}
+
+/*!
+    \since 6.6
+    Sets the extra settings and parameters for the child process on Unix
+    systems to be \a params. This function can be used to ask QProcess to
+    modify the child process before launching the target executable.
+
+    This function can be used to change certain properties of the child
+    process, such as closing all extraneous file descriptors, changing the nice
+    level of the child, or disconnecting from the controlling TTY. For more
+    fine-grained control of the child process or to modify it in other ways,
+    use the setChildProcessModifier() function. If both a child process
+    modifier and Unix process parameters are set, the modifier is run before
+    these parameters are applied.
+
+    \note This function is only available on Unix platforms.
+
+    \sa unixProcessParameters(), setChildProcessModifier()
+*/
+void QProcess::setUnixProcessParameters(const UnixProcessParameters &params)
+{
+    Q_D(QProcess);
+    if (!d->unixExtras)
+        d->unixExtras.reset(new QProcessPrivate::UnixExtras);
+    d->unixExtras->processParameters = params;
+}
+
+/*!
+    \since 6.6
+    \overload
+
+    Sets the extra settings for the child process on Unix systems to \a
+    flagsOnly. This is the same as the overload with just the \c flags field
+    set.
+    \note This function is only available on Unix platforms.
+
+    \sa unixProcessParameters(), setChildProcessModifier()
+*/
+void QProcess::setUnixProcessParameters(UnixProcessFlags flagsOnly)
+{
+    Q_D(QProcess);
+    if (!d->unixExtras)
+        d->unixExtras.reset(new QProcessPrivate::UnixExtras);
+    d->unixExtras->processParameters = { flagsOnly };
+}
 #endif
 
 /*!
diff --git a/src/corelib/io/qprocess.h b/src/corelib/io/qprocess.h
index 4fa1037b38..710b5e2f78 100644
--- a/src/corelib/io/qprocess.h
+++ b/src/corelib/io/qprocess.h
@@ -1,4 +1,5 @@
 // Copyright (C) 2016 The Qt Company Ltd.
+// Copyright (C) 2023 Intel Corporation.
 // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 #ifndef QPROCESS_H
@@ -173,6 +174,23 @@ public:
 #if defined(Q_OS_UNIX) || defined(Q_QDOC)
     std::function<void(void)> childProcessModifier() const;
     void setChildProcessModifier(const std::function<void(void)> &modifier);
+
+    enum UnixProcessFlag : quint32 {
+        ResetSignalHandlers                 = 0x0001, // like POSIX_SPAWN_SETSIGDEF
+        IgnoreSigPipe                       = 0x0002,
+        // some room if we want to add IgnoreSigHup or so
+        CloseNonStandardFileDescriptors     = 0x0010,
+    };
+    Q_DECLARE_FLAGS(UnixProcessFlags, UnixProcessFlag)
+    struct UnixProcessParameters
+    {
+        UnixProcessFlags flags = {};
+
+        quint32 _reserved[7] {};
+    };
+    UnixProcessParameters unixProcessParameters() const noexcept;
+    void setUnixProcessParameters(const UnixProcessParameters &params);
+    void setUnixProcessParameters(UnixProcessFlags flagsOnly);
 #endif
 
     QString workingDirectory() const;
@@ -256,6 +274,10 @@ private:
     Q_PRIVATE_SLOT(d_func(), void _q_processDied())
 };
 
+#ifdef Q_OS_UNIX
+Q_DECLARE_OPERATORS_FOR_FLAGS(QProcess::UnixProcessFlags)
+#endif
+
 #endif // QT_CONFIG(process)
 
 QT_END_NAMESPACE
diff --git a/src/corelib/io/qprocess_p.h b/src/corelib/io/qprocess_p.h
index dacf41b77a..3b862bc5f3 100644
--- a/src/corelib/io/qprocess_p.h
+++ b/src/corelib/io/qprocess_p.h
@@ -282,6 +282,7 @@ public:
 #else
     struct UnixExtras {
         std::function<void(void)> childProcessModifier;
+        QProcess::UnixProcessParameters processParameters;
     };
     std::unique_ptr<UnixExtras> unixExtras;
     QSocketNotifier *stateNotifier = nullptr;
diff --git a/src/corelib/io/qprocess_unix.cpp b/src/corelib/io/qprocess_unix.cpp
index 2de5cded52..c0b10c130e 100644
--- a/src/corelib/io/qprocess_unix.cpp
+++ b/src/corelib/io/qprocess_unix.cpp
@@ -36,6 +36,13 @@
 #include <limits.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#if __has_include(<linux/close_range.h>)
+// FreeBSD's is in <unistd.h>
+#  include <linux/close_range.h>
+#endif
 
 #if QT_CONFIG(process)
 #include <forkfd.h>
@@ -576,6 +583,46 @@ static constexpr int FakeErrnoForThrow =
 #endif
         ;
 
+// See IMPORTANT notice below
+static void applyProcessParameters(const QProcess::UnixProcessParameters &params)
+{
+    // Apply Unix signal handler parameters.
+    // We don't expect signal() to fail, so we ignore its return value
+    bool ignore_sigpipe = params.flags.testFlag(QProcess::UnixProcessFlag::IgnoreSigPipe);
+    if (ignore_sigpipe)
+        signal(SIGPIPE, SIG_IGN);                   // don't use qt_ignore_sigpipe!
+    if (params.flags.testFlag(QProcess::UnixProcessFlag::ResetSignalHandlers)) {
+        for (int sig = 1; sig < NSIG; ++sig) {
+            if (!ignore_sigpipe || sig != SIGPIPE)
+                signal(sig, SIG_DFL);
+        }
+    }
+
+    // Close all file descriptors above stderr.
+    // This isn't expected to fail, so we ignore close()'s return value.
+    if (params.flags.testFlag(QProcess::UnixProcessFlag::CloseNonStandardFileDescriptors)) {
+        int r = -1;
+        int fd = STDERR_FILENO + 1;
+#if QT_CONFIG(close_range)
+        // On FreeBSD, this probably won't fail.
+        // On Linux, this will fail with ENOSYS before kernel 5.9.
+        r = close_range(fd, INT_MAX, 0);
+#endif
+        if (r == -1) {
+            // We *could* read /dev/fd to find out what file descriptors are
+            // open, but we won't. We CANNOT use opendir() here because it
+            // allocates memory. Using getdents(2) plus either strtoul() or
+            // std::from_chars() would be acceptable.
+            int max_fd = INT_MAX;
+            if (struct rlimit limit; getrlimit(RLIMIT_NOFILE, &limit) == 0)
+                max_fd = limit.rlim_cur;
+            for ( ; fd < max_fd; ++fd)
+                close(fd);
+        }
+    }
+}
+
+// the noexcept here adds an extra layer of protection
 static const char *callChildProcessModifier(const QProcessPrivate::UnixExtras *unixExtras) noexcept
 {
     QT_TRY {
@@ -597,8 +644,13 @@ static const char *doExecChild(char **argv, char **envp, int workingDirFd,
         return "fchdir";
 
     if (unixExtras) {
+        // FIRST we call the user modifier function, before we dropping
+        // privileges or closing non-standard file descriptors
         if (const char *what = callChildProcessModifier(unixExtras))
             return what;
+
+        // then we apply our other user-provided parameters
+        applyProcessParameters(unixExtras->processParameters);
     }
 
     // execute the process
diff --git a/tests/auto/corelib/io/qprocess/CMakeLists.txt b/tests/auto/corelib/io/qprocess/CMakeLists.txt
index 016fcf6666..9fbf7657fd 100644
--- a/tests/auto/corelib/io/qprocess/CMakeLists.txt
+++ b/tests/auto/corelib/io/qprocess/CMakeLists.txt
@@ -28,3 +28,6 @@ if(WIN32)
     add_subdirectory(testProcessEchoGui)
     add_subdirectory(testSetNamedPipeHandleState)
 endif()
+if(UNIX)
+    add_subdirectory(testUnixProcessParameters)
+endif()
diff --git a/tests/auto/corelib/io/qprocess/testUnixProcessParameters/CMakeLists.txt b/tests/auto/corelib/io/qprocess/testUnixProcessParameters/CMakeLists.txt
new file mode 100644
index 0000000000..9b6c48933c
--- /dev/null
+++ b/tests/auto/corelib/io/qprocess/testUnixProcessParameters/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Copyright (C) 2023 Intel Corporation.
+# SPDX-License-Identifier: BSD-3-Clause
+
+#####################################################################
+## testProcessNormal Binary:
+#####################################################################
+
+qt_internal_add_executable(testUnixProcessParameters
+    OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/"
+    CORE_LIBRARY None
+    SOURCES
+        main.cpp
+)
diff --git a/tests/auto/corelib/io/qprocess/testUnixProcessParameters/main.cpp b/tests/auto/corelib/io/qprocess/testUnixProcessParameters/main.cpp
new file mode 100644
index 0000000000..e701677403
--- /dev/null
+++ b/tests/auto/corelib/io/qprocess/testUnixProcessParameters/main.cpp
@@ -0,0 +1,62 @@
+// Copyright (C) 2023 Intel Corporation.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <string_view>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+int main(int argc, char **argv)
+{
+    if (argc < 2) {
+        printf("Usage: %s command [extra]\nSee source code for commands\n",
+               argv[0]);
+        return EXIT_FAILURE;
+    }
+
+    std::string_view cmd = argv[1];
+    errno = 0;
+
+    if (cmd.size() == 0) {
+        // just checking that we did get here
+        return EXIT_SUCCESS;
+    }
+
+    if (cmd == "reset-sighand") {
+        // confirm it was not ignored
+        struct sigaction action;
+        sigaction(SIGUSR1, nullptr, &action);
+        if (action.sa_handler == SIG_DFL)
+            return EXIT_SUCCESS;
+        fprintf(stderr, "SIGUSR1 is SIG_IGN\n");
+        return EXIT_FAILURE;
+    }
+
+    if (cmd == "ignore-sigpipe") {
+        // confirm it was ignored
+        struct sigaction action;
+        sigaction(SIGPIPE, nullptr, &action);
+        if (action.sa_handler == SIG_IGN)
+            return EXIT_SUCCESS;
+        fprintf(stderr, "SIGPIPE is SIG_DFL\n");
+        return EXIT_FAILURE;
+    }
+
+    if (cmd == "std-file-descriptors") {
+        int fd = atoi(argv[2]);
+        if (close(fd) < 0 && errno == EBADF)
+            return EXIT_SUCCESS;
+        fprintf(stderr, "%d is a valid file descriptor\n", fd);
+        return EXIT_FAILURE;
+    }
+
+    fprintf(stderr, "Unknown command \"%s\"", cmd.data());
+    return EXIT_FAILURE;
+}
diff --git a/tests/auto/corelib/io/qprocess/tst_qprocess.cpp b/tests/auto/corelib/io/qprocess/tst_qprocess.cpp
index 285126b826..069440c49b 100644
--- a/tests/auto/corelib/io/qprocess/tst_qprocess.cpp
+++ b/tests/auto/corelib/io/qprocess/tst_qprocess.cpp
@@ -114,6 +114,9 @@ private slots:
     void setChildProcessModifier_data();
     void setChildProcessModifier();
     void throwInChildProcessModifier();
+    void unixProcessParameters_data();
+    void unixProcessParameters();
+    void unixProcessParametersAndChildModifier();
 #endif
     void exitCodeTest();
     void systemEnvironment();
@@ -1439,6 +1442,110 @@ void tst_QProcess::createProcessArgumentsModifier()
 #endif // Q_OS_WIN
 
 #ifdef Q_OS_UNIX
+void tst_QProcess::unixProcessParameters_data()
+{
+    QTest::addColumn<QProcess::UnixProcessParameters>("params");
+    QTest::addColumn<QString>("cmd");
+    QTest::newRow("defaults") << QProcess::UnixProcessParameters{} << QString();
+
+    auto addRow = [](const char *cmd, QProcess::UnixProcessFlags flags) {
+        QProcess::UnixProcessParameters params = {};
+        params.flags = flags;
+        QTest::addRow("%s", cmd) << params << cmd;
+    };
+    using P = QProcess::UnixProcessFlag;
+    addRow("reset-sighand", P::ResetSignalHandlers);
+    addRow("ignore-sigpipe", P::IgnoreSigPipe);
+    addRow("std-file-descriptors", P::CloseNonStandardFileDescriptors);
+}
+
+void tst_QProcess::unixProcessParameters()
+{
+    QFETCH(QProcess::UnixProcessParameters, params);
+    QFETCH(QString, cmd);
+
+    // set up a few things
+    struct Scope {
+        int devnull;
+        struct sigaction old_sigusr1, old_sigpipe;
+        Scope()
+        {
+            int fd = open("/dev/null", O_RDONLY);
+            devnull = fcntl(fd, F_DUPFD, 100);
+            close(fd);
+
+            // we ignore SIGUSR1 and reset SIGPIPE to Terminate
+            struct sigaction act = {};
+            sigemptyset(&act.sa_mask);
+            act.sa_handler = SIG_IGN;
+            sigaction(SIGUSR1, &act, &old_sigusr1);
+            act.sa_handler = SIG_DFL;
+            sigaction(SIGPIPE, &act, &old_sigpipe);
+        }
+        ~Scope()
+        {
+            if (devnull != -1)
+                dismiss();
+        }
+        void dismiss()
+        {
+            close(devnull);
+            sigaction(SIGUSR1, &old_sigusr1, nullptr);
+            sigaction(SIGPIPE, &old_sigpipe, nullptr);
+            devnull = -1;
+        }
+    } scope;
+
+    QProcess process;
+    process.setUnixProcessParameters(params);
+    process.setStandardInputFile(QProcess::nullDevice());   // so we can't mess with SIGPIPE
+    process.setProgram("testUnixProcessParameters/testUnixProcessParameters");
+    process.setArguments({ cmd, QString::number(scope.devnull) });
+    process.start();
+    QVERIFY2(process.waitForStarted(5000), qPrintable(process.errorString()));
+    QVERIFY(process.waitForFinished(5000));
+    QCOMPARE(process.readAllStandardError(), QString());
+    QCOMPARE(process.readAll(), QString());
+    QCOMPARE(process.exitCode(), 0);
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+}
+
+void tst_QProcess::unixProcessParametersAndChildModifier()
+{
+    static constexpr char message[] = "Message from the handler function\n";
+    static_assert(std::char_traits<char>::length(message) <= PIPE_BUF);
+    QProcess process;
+    int pipes[2];
+
+    QVERIFY2(pipe(pipes) == 0, qPrintable(qt_error_string()));
+    auto pipeGuard0 = qScopeGuard([=] { close(pipes[0]); });
+    {
+        auto pipeGuard1 = qScopeGuard([=] { close(pipes[1]); });
+
+        // verify that our modifier runs before the parameters are applied
+        process.setChildProcessModifier([=] {
+            write(pipes[1], message, strlen(message));
+        });
+        auto flags = QProcess::UnixProcessFlag::CloseNonStandardFileDescriptors;
+        process.setUnixProcessParameters({ flags });
+        process.setProgram("testUnixProcessParameters/testUnixProcessParameters");
+        process.setArguments({ "std-file-descriptors", QString::number(pipes[1]) });
+        process.start();
+        QVERIFY2(process.waitForStarted(5000), qPrintable(process.errorString()));
+    } // closes the writing end of the pipe
+
+    QVERIFY(process.waitForFinished(5000));
+    QCOMPARE(process.readAllStandardError(), QString());
+    QCOMPARE(process.readAll(), QString());
+
+    char buf[2 * sizeof(message)];
+    int r = read(pipes[0], buf, sizeof(buf));
+    QVERIFY2(r >= 0, qPrintable(qt_error_string()));
+    QCOMPARE(QByteArrayView(buf, r), message);
+}
+#endif
+
+#ifdef Q_OS_UNIX
 static constexpr char messageFromChildProcess[] = "Message from the child process";
 static_assert(std::char_traits<char>::length(messageFromChildProcess) <= PIPE_BUF);
 static void childProcessModifier(int fd)