QProcess: set the error message and state if qt_create_pipe() fails

It can only happen on resource exhaustion, which is an unlikely
situation. That's probably why we didn't handle this case. But let's do
it now.

This necessitated that qt_create_pipe() return with errno/GetLastError()
properly set, so save it and restore around functions that may change it
(like qErrnoWarning()).

Drive-by update of some warning messages.

Testing this is not practical and is fragile, because it requires
causing the resource exhaustion problem.

Pick-to: 6.6 6.7
Change-Id: I76ffba14ece04f24b43efffd17aafe4f11f54c21
Reviewed-by: Oswald Buddenhagen <oswald.buddenhagen@gmx.de>
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

2 files changed, 29 insertions, 8 deletions

diff --git a/src/corelib/io/qprocess_unix.cpp b/src/corelib/io/qprocess_unix.cpp
index 9093cf1e75..1179373cf3 100644
--- a/src/corelib/io/qprocess_unix.cpp
+++ b/src/corelib/io/qprocess_unix.cpp
@@ -423,7 +423,8 @@ static int qt_create_pipe(int *pipe)
         qt_safe_close(pipe[1]);
     int pipe_ret = qt_safe_pipe(pipe);
     if (pipe_ret != 0) {
-        qErrnoWarning("QProcessPrivate::createPipe: Cannot create pipe %p", pipe);
+        QScopedValueRollback rollback(errno);
+        qErrnoWarning("QProcess: Cannot create pipe");
     }
     return pipe_ret;
 }
@@ -472,8 +473,10 @@ bool QProcessPrivate::openChannel(Channel &channel)
 
     if (channel.type == Channel::Normal) {
         // we're piping this channel to our own process
-        if (qt_create_pipe(channel.pipe) != 0)
+        if (qt_create_pipe(channel.pipe) != 0) {
+            setErrorAndEmit(QProcess::FailedToStart, "pipe: "_L1 + qt_error_string(errno));
             return false;
+        }
 
         // create the socket notifiers
         if (threadData.loadRelaxed()->hasEventDispatcher()) {
@@ -552,8 +555,10 @@ bool QProcessPrivate::openChannel(Channel &channel)
             Q_ASSERT(sink->pipe[0] == INVALID_Q_PIPE && sink->pipe[1] == INVALID_Q_PIPE);
 
             Q_PIPE pipe[2] = { -1, -1 };
-            if (qt_create_pipe(pipe) != 0)
+            if (qt_create_pipe(pipe) != 0) {
+                setErrorAndEmit(QProcess::FailedToStart, "pipe: "_L1 + qt_error_string(errno));
                 return false;
+            }
             sink->pipe[0] = pipe[0];
             source->pipe[1] = pipe[1];
 
diff --git a/src/corelib/io/qprocess_win.cpp b/src/corelib/io/qprocess_win.cpp
index 9b1ac1ca3a..e64b133815 100644
--- a/src/corelib/io/qprocess_win.cpp
+++ b/src/corelib/io/qprocess_win.cpp
@@ -140,6 +140,7 @@ static bool qt_create_pipe(Q_PIPE *pipe, bool isInputPipe, BOOL defInheritFlag)
         DWORD dwError = GetLastError();
         if (dwError != ERROR_PIPE_BUSY || !--attempts) {
             qErrnoWarning(dwError, "QProcess: CreateNamedPipe failed.");
+            SetLastError(dwError);
             return false;
         }
     }
@@ -154,8 +155,10 @@ static bool qt_create_pipe(Q_PIPE *pipe, bool isInputPipe, BOOL defInheritFlag)
                                       FILE_FLAG_OVERLAPPED,
                                       NULL);
     if (hClient == INVALID_HANDLE_VALUE) {
+        DWORD dwError = GetLastError();
         qErrnoWarning("QProcess: CreateFile failed.");
         CloseHandle(hServer);
+        SetLastError(dwError);
         return false;
     }
 
@@ -172,10 +175,12 @@ static bool qt_create_pipe(Q_PIPE *pipe, bool isInputPipe, BOOL defInheritFlag)
             WaitForSingleObject(overlapped.hEvent, INFINITE);
             break;
         default:
+            dwError = GetLastError();
             qErrnoWarning(dwError, "QProcess: ConnectNamedPipe failed.");
             CloseHandle(overlapped.hEvent);
             CloseHandle(hClient);
             CloseHandle(hServer);
+            SetLastError(dwError);
             return false;
         }
     }
@@ -201,8 +206,13 @@ bool QProcessPrivate::openChannel(Channel &channel)
     switch (channel.type) {
     case Channel::Normal: {
         // we're piping this channel to our own process
-        if (&channel == &stdinChannel)
-            return qt_create_pipe(channel.pipe, true, FALSE);
+        if (&channel == &stdinChannel) {
+            if (!qt_create_pipe(channel.pipe, true, FALSE)) {
+                setErrorAndEmit(QProcess::FailedToStart, "pipe: "_L1 + qt_error_string(errno));
+                return false;
+            }
+            return true;
+        }
 
         if (&channel == &stdoutChannel) {
             if (!stdoutChannel.reader) {
@@ -215,8 +225,10 @@ bool QProcessPrivate::openChannel(Channel &channel)
                 q->connect(stderrChannel.reader, SIGNAL(readyRead()), SLOT(_q_canReadStandardError()));
             }
         }
-        if (!qt_create_pipe(channel.pipe, false, FALSE))
+        if (!qt_create_pipe(channel.pipe, false, FALSE)) {
+            setErrorAndEmit(QProcess::FailedToStart, "pipe: "_L1 + qt_error_string(errno));
             return false;
+        }
 
         channel.reader->setHandle(channel.pipe[0]);
         channel.reader->startAsyncRead();
@@ -281,8 +293,10 @@ bool QProcessPrivate::openChannel(Channel &channel)
         Q_ASSERT(source == &stdoutChannel);
         Q_ASSERT(sink->process == this && sink->type == Channel::PipeSink);
 
-        if (!qt_create_pipe(source->pipe, /* in = */ false, TRUE))  // source is stdout
+        if (!qt_create_pipe(source->pipe, /* in = */ false, TRUE)) { // source is stdout
+            setErrorAndEmit(QProcess::FailedToStart, "pipe: "_L1 + qt_error_string(errno));
             return false;
+        }
 
         sink->pipe[0] = source->pipe[0];
         source->pipe[0] = INVALID_Q_PIPE;
@@ -301,8 +315,10 @@ bool QProcessPrivate::openChannel(Channel &channel)
         Q_ASSERT(sink == &stdinChannel);
         Q_ASSERT(source->process == this && source->type == Channel::PipeSource);
 
-        if (!qt_create_pipe(sink->pipe, /* in = */ true, TRUE))  // sink is stdin
+        if (!qt_create_pipe(sink->pipe, /* in = */ true, TRUE)) { // sink is stdin
+            setErrorAndEmit(QProcess::FailedToStart, "pipe: "_L1 + qt_error_string(errno));
             return false;
+        }
 
         source->pipe[1] = sink->pipe[1];
         sink->pipe[1] = INVALID_Q_PIPE;