logging: Clarify and document how we look for the presence of a console

The privately exported qt_logging_to_console() function has been renamed
to shouldLogToStderr, and exported in the QtPrivate namespace for QtTestLib.

[ChangeLog][Important behavior changes][Logging (including qDebug and
qWarning)] The QT_LOGGING_TO_CONSOLE environment variable has been
deprecated. Use the more specific QT_ASSUME_STDERR_HAS_CONSOLE or
QT_FORCE_STDERR_LOGGING, depending on your usecase.

Change-Id: Ie29228eeac3b700c3de94fee022d5d66d9b5c210
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

3 files changed, 164 insertions, 53 deletions

diff --git a/src/corelib/configure.json b/src/corelib/configure.json
index 7c3b513b0b..5e48024def 100644
--- a/src/corelib/configure.json
+++ b/src/corelib/configure.json
@@ -909,7 +909,7 @@ Mozilla License) is included. The data is then also used in QNetworkCookieJar::v
             "condition": "features.journald || features.syslog || (config.qnx && features.slog2)",
             "message": "journald, syslog or slog2 integration is enabled.
 If your users intend to develop applications against this build,
-ensure that the IDEs they use either set QT_LOGGING_TO_CONSOLE to 1
+ensure that the IDEs they use either set QT_FORCE_STDERR_LOGGING to 1
 or are able to read the logged output from journald, syslog or slog2."
         },
         {
diff --git a/src/corelib/global/qlogging.cpp b/src/corelib/global/qlogging.cpp
index 9f1fe7cabb..99c57c3b7a 100644
--- a/src/corelib/global/qlogging.cpp
+++ b/src/corelib/global/qlogging.cpp
@@ -41,6 +41,7 @@
 
 #include "qglobal_p.h"
 #include "qlogging.h"
+#include "qlogging_p.h"
 #include "qlist.h"
 #include "qbytearray.h"
 #include "qstring.h"
@@ -204,56 +205,102 @@ static bool isDefaultCategory(const char *category)
     return !category || strcmp(category, "default") == 0;
 }
 
-static bool willLogToConsole()
+/*!
+    Returns true if writing to \c stderr is supported.
+
+    \sa stderrHasConsoleAttached()
+*/
+static bool systemHasStderr()
 {
 #if defined(Q_OS_WINRT)
-    // these systems have no stderr, so always log to the system log
-    return false;
-#else
-    // rules to determine if we'll log preferably to the console:
-    //  1) if QT_LOGGING_TO_CONSOLE is set, it determines behavior:
-    //    - if it's set to 0, we will not log to console
-    //    - if it's set to 1, we will log to console
-    //  2) otherwise, we will log to console if we have a console window (Windows)
-    //     or a controlling TTY (Unix). This is done even if stderr was redirected
-    //     to the blackhole device (NUL or /dev/null).
-
-    bool ok = true;
-    uint envcontrol = qgetenv("QT_LOGGING_TO_CONSOLE").toUInt(&ok);
-    if (ok)
-        return envcontrol;
-
-#  ifdef Q_OS_WIN
-    return GetConsoleWindow();
-#  elif defined(Q_OS_UNIX)
-#    ifndef _PATH_TTY
-#    define _PATH_TTY "/dev/tty"
-#    endif
-    // if /dev/tty exists, we can only open it if we have a controlling TTY
-    int devtty = qt_safe_open(_PATH_TTY, O_RDONLY);
-    if (devtty == -1 && (errno == ENOENT || errno == EPERM || errno == ENXIO)) {
-        // no /dev/tty, fall back to isatty on stderr
-        return isatty(STDERR_FILENO);
-    } else if (devtty != -1) {
-        // there is a /dev/tty and we could open it: we have a controlling TTY
-        qt_safe_close(devtty);
-        return true;
-    }
+    return false; // WinRT has no stderr
+#endif
 
-    // no controlling TTY
-    return false;
-#  else
-#    error "Not Unix and not Windows?"
-#  endif
+    return true;
+}
+
+/*!
+    Returns true if writing to \c stderr will end up in a console/terminal visible to the user.
+
+    This is typically the case if the application was started from the command line.
+
+    If the application is started without a controlling console/terminal, but the parent
+    process reads \c stderr and presents it to the user in some other way, the parent process
+    may override the detection in this function by setting the QT_ASSUME_STDERR_HAS_CONSOLE
+    environment variable to \c 1.
+
+    \note Qt Creator does not implement a pseudo TTY, nor does it launch apps with
+    the override environment variable set, but it will read stderr and print it to
+    the user, so in effect this function can not be used to conclude that stderr
+    output will _not_ be visible to the user, as even if this function returns false,
+    the output might still end up visible to the user. For this reason, we don't guard
+    the stderr output in the default message handler with stderrHasConsoleAttached().
+
+    \sa systemHasStderr()
+*/
+bool stderrHasConsoleAttached()
+{
+    static const bool stderrHasConsoleAttached = []() -> bool {
+        if (!systemHasStderr())
+            return false;
+
+        if (qEnvironmentVariableIntValue("QT_LOGGING_TO_CONSOLE")) {
+            fprintf(stderr, "warning: Environment variable QT_LOGGING_TO_CONSOLE is deprecated, use\n"
+                            "QT_ASSUME_STDERR_HAS_CONSOLE and/or QT_FORCE_STDERR_LOGGING instead.\n");
+            return true;
+        }
+
+        if (qEnvironmentVariableIntValue("QT_ASSUME_STDERR_HAS_CONSOLE"))
+            return true;
+
+#if defined(Q_OS_WIN) && !defined(Q_OS_WINRT)
+        return GetConsoleWindow();
+#elif defined(Q_OS_UNIX)
+#       ifndef _PATH_TTY
+#       define _PATH_TTY "/dev/tty"
+#       endif
+
+        // If we can open /dev/tty, we have a controlling TTY
+        int ttyDevice = -1;
+        if ((ttyDevice = qt_safe_open(_PATH_TTY, O_RDONLY)) >= 0) {
+            qt_safe_close(ttyDevice);
+            return true;
+        } else if (errno == ENOENT || errno == EPERM || errno == ENXIO) {
+            // Fall back to isatty for some non-critical errors
+            return isatty(STDERR_FILENO);
+        } else {
+            return false;
+        }
+#else
+        return false; // No way to detect if stderr has a console attached
 #endif
+    }();
+
+    return stderrHasConsoleAttached;
 }
 
-Q_CORE_EXPORT bool qt_logging_to_console()
+
+namespace QtPrivate {
+
+/*!
+    Returns true if logging \c stderr should be ensured.
+
+    This is normally the case if \c stderr has a console attached, but may be overridden
+    by the user by setting the QT_FORCE_STDERR_LOGGING environment variable to \c 1.
+
+    \sa stderrHasConsoleAttached()
+*/
+bool shouldLogToStderr()
 {
-    static const bool logToConsole = willLogToConsole();
-    return logToConsole;
+    static bool forceStderrLogging = qEnvironmentVariableIntValue("QT_FORCE_STDERR_LOGGING");
+    return forceStderrLogging || stderrHasConsoleAttached();
 }
 
+
+} // QtPrivate
+
+using namespace QtPrivate;
+
 /*!
     \class QMessageLogContext
     \inmodule QtCore
@@ -1472,8 +1519,8 @@ static QBasicAtomicPointer<void (QtMsgType, const QMessageLogContext &, const QS
 
 static bool slog2_default_handler(QtMsgType type, const QMessageLogContext &context, const QString &message)
 {
-    if (qt_logging_to_console())
-        return false;
+    if (shouldLogToStderr())
+        return false; // Leave logging up to stderr handler
 
     QString formattedMessage = qFormatLogMessage(type, context, message);
     formattedMessage.append(QLatin1Char('\n'));
@@ -1528,8 +1575,8 @@ static bool systemd_default_message_handler(QtMsgType type,
                                             const QMessageLogContext &context,
                                             const QString &message)
 {
-    if (qt_logging_to_console())
-        return false;
+    if (shouldLogToStderr())
+        return false; // Leave logging up to stderr handler
 
     QString formattedMessage = qFormatLogMessage(type, context, message);
 
@@ -1567,8 +1614,8 @@ static bool systemd_default_message_handler(QtMsgType type,
 #if QT_CONFIG(syslog)
 static bool syslog_default_message_handler(QtMsgType type, const QMessageLogContext &context, const QString &message)
 {
-    if (qt_logging_to_console())
-        return false;
+    if (shouldLogToStderr())
+        return false; // Leave logging up to stderr handler
 
     QString formattedMessage = qFormatLogMessage(type, context, message);
 
@@ -1602,8 +1649,8 @@ static bool android_default_message_handler(QtMsgType type,
                                   const QMessageLogContext &context,
                                   const QString &message)
 {
-    if (qt_logging_to_console())
-        return false;
+    if (shouldLogToStderr())
+        return false; // Leave logging up to stderr handler
 
     QString formattedMessage = qFormatLogMessage(type, context, message);
 
@@ -1627,8 +1674,8 @@ static bool android_default_message_handler(QtMsgType type,
 #ifdef Q_OS_WIN
 static bool win_message_handler(QtMsgType type, const QMessageLogContext &context, const QString &message)
 {
-    if (qt_logging_to_console())
-        return false;
+    if (shouldLogToStderr())
+        return false; // Leave logging up to stderr handler
 
     QString formattedMessage = qFormatLogMessage(type, context, message);
     formattedMessage.append(QLatin1Char('\n'));
@@ -1756,7 +1803,7 @@ static void qt_message_print(const QString &message)
     OutputDebugString(reinterpret_cast<const wchar_t*>(message.utf16()));
     return;
 #elif defined(Q_OS_WIN) && !defined(QT_BOOTSTRAPPED)
-    if (!qt_logging_to_console()) {
+    if (!shouldLogToStderr()) {
         OutputDebugString(reinterpret_cast<const wchar_t*>(message.utf16()));
         return;
     }
diff --git a/src/corelib/global/qlogging_p.h b/src/corelib/global/qlogging_p.h
new file mode 100644
index 0000000000..3d0c097ffe
--- /dev/null
+++ b/src/corelib/global/qlogging_p.h
@@ -0,0 +1,64 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or (at your option) the GNU General
+** Public license version 3 or any later version approved by the KDE Free
+** Qt Foundation. The licenses are as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-2.0.html and
+** https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QLOGGING_P_H
+#define QLOGGING_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists for the convenience
+// of a number of Qt sources files.  This header file may change from
+// version to version without notice, or even be removed.
+//
+// We mean it.
+//
+
+QT_BEGIN_NAMESPACE
+
+namespace QtPrivate {
+
+Q_CORE_EXPORT bool shouldLogToStderr();
+
+}
+
+QT_END_NAMESPACE
+
+#endif // QLOGGING_P_H