Add XCTest logger backend to QtTestLib

Will be active when running test apps through Xcode's 'test' action,
and reports QtTestLib test objects and functions to Xcode as XCTest
cases.

This allows running tests on both iOS Simulator and iOS devices from
the command line, through xcodebuild, without relying on any 3rd party
tools. It also integrates Qt test failures and passes into the Xcode
IDE, which may be useful for closer investigation of test failures.

The feature is limited to Xcode 6.x.

Change-Id: I33d39edbabdbaebef48d2d0eb7e08a1ffb72c397
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@theqtcompany.com>

1 files changed, 501 insertions, 0 deletions

diff --git a/src/testlib/qxctestlogger.mm b/src/testlib/qxctestlogger.mm
new file mode 100644
index 0000000000..576f46acea
--- /dev/null
+++ b/src/testlib/qxctestlogger.mm
@@ -0,0 +1,501 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd.
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the QtTest module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL21$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 or version 3 as published by the Free
+** Software Foundation and appearing in the file LICENSE.LGPLv21 and
+** LICENSE.LGPLv3 included in the packaging of this file. Please review the
+** following information to ensure the GNU Lesser General Public License
+** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** As a special exception, The Qt Company gives you certain additional
+** rights. These rights are described in The Qt Company LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qxctestlogger_p.h"
+
+#include <QtCore/qstring.h>
+
+#include <QtTest/private/qtestlog_p.h>
+#include <QtTest/private/qtestresult_p.h>
+
+#import <XCTest/XCTest.h>
+
+// ---------------------------------------------------------
+
+@interface XCTestProbe (Private)
++ (BOOL)isTesting;
++ (void)runTests:(id)unusedArgument;
++ (NSString*)testScope;
++ (BOOL)isInverseTestScope;
+@end
+
+@interface XCTestDriver : NSObject
++ (XCTestDriver*)sharedTestDriver;
+@property (readonly, assign) NSObject *IDEConnection;
+@end
+
+@interface XCTest (Private)
+- (NSString *)nameForLegacyLogging;
+@end
+
+#pragma GCC diagnostic push // Ignore XCTestProbe deprecation
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+
+// ---------------------------------------------------------
+
+@interface QtTestLibWrapper : XCTestCase
+@end
+
+@interface QtTestLibTests : XCTestSuite
++ (XCTestSuiteRun*)testRun;
+@end
+
+@interface QtTestLibTest : XCTestCase
+@property (nonatomic, retain) NSString* testObjectName;
+@property (nonatomic, retain) NSString* testFunctionName;
+@end
+
+// ---------------------------------------------------------
+
+class ThreadBarriers
+{
+public:
+    enum Barrier {
+        XCTestCanStartTesting,
+        XCTestHaveStarted,
+        QtTestsCanStartTesting,
+        QtTestsHaveCompleted,
+        XCTestsHaveCompleted,
+        BarrierCount
+    };
+
+    static ThreadBarriers *get()
+    {
+        static ThreadBarriers instance;
+        return &instance;
+    }
+
+    static void initialize() { get(); }
+
+    void wait(Barrier barrier) { dispatch_semaphore_wait(barriers[barrier], DISPATCH_TIME_FOREVER); }
+    void signal(Barrier barrier) { dispatch_semaphore_signal(barriers[barrier]); }
+
+private:
+    #define FOREACH_BARRIER(cmd) for (int i = 0; i < BarrierCount; ++i) { cmd }
+
+    ThreadBarriers() { FOREACH_BARRIER(barriers[i] = dispatch_semaphore_create(0);) }
+    ~ThreadBarriers() { FOREACH_BARRIER(dispatch_release(barriers[i]);) }
+
+    dispatch_semaphore_t barriers[BarrierCount];
+};
+
+#define WAIT_FOR_BARRIER(b) ThreadBarriers::get()->wait(ThreadBarriers::b);
+#define SIGNAL_BARRIER(b) ThreadBarriers::get()->signal(ThreadBarriers::b);
+
+// ---------------------------------------------------------
+
+@implementation QtTestLibWrapper
+
++ (void)load
+{
+    NSAutoreleasePool *autoreleasepool = [[NSAutoreleasePool alloc] init];
+
+    if (![XCTestProbe isTesting])
+        return;
+
+    if (!([NSDate timeIntervalSinceReferenceDate] > 0))
+        qFatal("error: Device date '%s' is bad, likely set to update automatically. Please correct.",
+            [NSDate date].description.UTF8String);
+
+    XCTestDriver *testDriver = nil;
+    if ([QtTestLibWrapper usingTestManager])
+        testDriver = [XCTestDriver sharedTestDriver];
+
+    // Spawn off task to run test infrastructure on separate thread so that we can
+    // let main() execute like normal on the main thread. The queue will never be
+    // destroyed, so there's no point in trying to keep a proper retain count.
+    dispatch_async(dispatch_queue_create("io.qt.QTestLib.xctest-wrapper", DISPATCH_QUEUE_SERIAL), ^{
+        Q_ASSERT(![NSThread isMainThread]);
+        [XCTestProbe runTests:nil];
+        Q_UNREACHABLE();
+    });
+
+    // Initialize barriers before registering exit handler so that the
+    // semaphores stay alive until after the exit handler completes.
+    ThreadBarriers::initialize();
+
+    // We register an exit handler so that we can intercept when main() completes
+    // and let the XCTest thread finish up. For main() functions that never started
+    // testing using QtTestLib we also need to signal that xcTestsCanStart.
+    atexit_b(^{
+        Q_ASSERT([NSThread isMainThread]);
+
+        // In case not started by startLogging
+        SIGNAL_BARRIER(XCTestCanStartTesting);
+
+        // [XCTestProbe runTests:] ends up calling [XCTestProbe exitTests:] after
+        // all test suites have completed, which calls exit(). We use that to signal
+        // to the main thread that it's free to continue its exit handler.
+        atexit_b(^{
+            Q_ASSERT(![NSThread isMainThread]);
+            SIGNAL_BARRIER(XCTestsHaveCompleted);
+
+            // Block forever so that the main thread does all the cleanup
+            dispatch_semaphore_wait(dispatch_semaphore_create(0), DISPATCH_TIME_FOREVER);
+        });
+
+        SIGNAL_BARRIER(QtTestsHaveCompleted);
+
+        // Ensure XCTest complets the top level tests suite
+        WAIT_FOR_BARRIER(XCTestsHaveCompleted);
+    });
+
+    // Let test driver (Xcode) connection setup complete before continuing
+    if ([[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"]) {
+        while (!testDriver.IDEConnection)
+            [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]];
+    }
+
+    // Wait for our QtTestLib test suite to run before running main
+    WAIT_FOR_BARRIER(QtTestsCanStartTesting);
+
+    // Prevent XCTestProbe from re-launching runTests on application startup
+    [[NSNotificationCenter defaultCenter] removeObserver:[XCTestProbe class]
+        name:[NSString stringWithFormat:@"%@DidFinishLaunchingNotification",
+                #if defined(Q_OS_OSX)
+                    @"NSApplication"
+                #elif defined(Q_OS_IOS)
+                    @"UIApplication"
+                #endif
+             ]
+        object:nil];
+
+    [autoreleasepool release];
+}
+
++ (id)defaultTestSuite
+{
+    return [[QtTestLibTests alloc] initWithName:@"QtTestLib"];
+}
+
++ (BOOL)usingTestManager
+{
+    return [[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"];
+}
+
+@end
+
+// ---------------------------------------------------------
+
+static XCTestSuiteRun *s_qtTestSuiteRun = 0;
+
+@implementation QtTestLibTests
+
+- (void)performTest:(XCTestSuiteRun *)testSuiteRun
+{
+    Q_ASSERT(![NSThread isMainThread]);
+
+    Q_ASSERT(!s_qtTestSuiteRun);
+    s_qtTestSuiteRun = testSuiteRun;
+
+    SIGNAL_BARRIER(QtTestsCanStartTesting);
+
+    // Wait for main() to complete, or a QtTestLib test to start, so we
+    // know if we should start the QtTestLib test suite.
+    WAIT_FOR_BARRIER(XCTestCanStartTesting);
+
+    if (QXcodeTestLogger::isActive())
+        [testSuiteRun start];
+
+    SIGNAL_BARRIER(XCTestHaveStarted);
+
+    // All test reporting happens on main thread from now on. Wait until
+    // main() completes before allowing the XCTest thread to continue.
+    WAIT_FOR_BARRIER(QtTestsHaveCompleted);
+
+    if ([testSuiteRun startDate])
+        [testSuiteRun stop];
+}
+
++ (XCTestSuiteRun*)testRun
+{
+    return s_qtTestSuiteRun;
+}
+
+@end
+
+// ---------------------------------------------------------
+
+@implementation QtTestLibTest
+
+- (id)initWithInvocation:(NSInvocation *)invocation
+{
+    if (self = [super initWithInvocation:invocation]) {
+        // The test object name and function name are used by XCTest after QtTestLib has
+        // reset them, so we need to store them up front for each XCTestCase.
+        self.testObjectName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()];
+        self.testFunctionName = [NSString stringWithUTF8String:QTestResult::currentTestFunction()];
+    }
+
+    return self;
+}
+
+- (NSString *)testClassName
+{
+    return self.testObjectName;
+}
+
+- (NSString *)testMethodName
+{
+    return self.testFunctionName;
+}
+
+- (NSString *)nameForLegacyLogging
+{
+    NSString *name = [NSString stringWithFormat:@"%@::%@", [self testClassName], [self testMethodName]];
+    if (QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag()) {
+        const char *currentDataTag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : "";
+        const char *globalDataTag = QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : "";
+        const char *filler = (currentDataTag[0] && globalDataTag[0]) ? ":" : "";
+        name = [name stringByAppendingString:[NSString stringWithFormat:@"(%s%s%s)",
+            globalDataTag, filler, currentDataTag]];
+    }
+
+    return name;
+}
+
+@end
+
+// ---------------------------------------------------------
+
+bool QXcodeTestLogger::canLogTestProgress()
+{
+    return [XCTestProbe isTesting]; // FIXME: Exclude xctool
+}
+
+int QXcodeTestLogger::parseCommandLineArgument(const char *argument)
+{
+    if (strncmp(argument, "-NS", 3) == 0 || strncmp(argument, "-Apple", 6) == 0)
+        return 2; // -NSTreatUnknownArgumentsAsOpen, -ApplePersistenceIgnoreState, etc, skip argument
+    else if (strcmp(argument, "--use-testmanagerd") == 0)
+        return 2; // Skip UID argument
+    else if (strncmp(argument, "-XCTest", 7) == 0)
+        return 2; // -XCTestInvertScope, -XCTest scope, etc, skip argument
+    else if (strcmp(argument + (strlen(argument) - 7), ".xctest") == 0)
+        return 1; // Skip test bundle
+    else
+        return 0;
+}
+
+// ---------------------------------------------------------
+
+QXcodeTestLogger *QXcodeTestLogger::s_currentTestLogger = 0;
+
+// ---------------------------------------------------------
+
+QXcodeTestLogger::QXcodeTestLogger()
+    : QAbstractTestLogger(0)
+    , m_testRuns([[NSMutableArray arrayWithCapacity:2] retain])
+
+{
+    Q_ASSERT(!s_currentTestLogger);
+    s_currentTestLogger = this;
+}
+
+QXcodeTestLogger::~QXcodeTestLogger()
+{
+    s_currentTestLogger = 0;
+    [m_testRuns release];
+}
+
+void QXcodeTestLogger::startLogging()
+{
+    SIGNAL_BARRIER(XCTestCanStartTesting);
+
+    static dispatch_once_t onceToken;
+    dispatch_once (&onceToken, ^{
+        WAIT_FOR_BARRIER(XCTestHaveStarted);
+    });
+
+    // Scope test object suite under top level QtTestLib test run
+    [m_testRuns addObject:[QtTestLibTests testRun]];
+
+    NSString *suiteName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()];
+    pushTestRunForTest([XCTestSuite testSuiteWithName:suiteName], true);
+}
+
+void QXcodeTestLogger::stopLogging()
+{
+    popTestRun();
+}
+
+static bool isTestFunctionInActiveScope(const char *function)
+{
+    static NSString *testScope = [XCTestProbe testScope];
+
+    enum TestScope { Unknown, All, None, Self, Selected };
+    static TestScope activeScope = Unknown;
+
+    if (activeScope == Unknown) {
+        if ([testScope isEqualToString:@"All"])
+            activeScope = All;
+        else if ([testScope isEqualToString:@"None"])
+            activeScope = None;
+        else if ([testScope isEqualToString:@"Self"])
+            activeScope = Self;
+        else
+            activeScope = Selected;
+    }
+
+    if (activeScope == All)
+        return true;
+    else if (activeScope == None)
+        return false;
+    else if (activeScope == Self)
+        return true; // Investigate
+
+    Q_ASSERT(activeScope == Selected);
+
+    static NSArray *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain];
+    if ([forcedTests containsObject:[NSString stringWithUTF8String:function]])
+        return true;
+
+    static NSArray *testsInScope = [[testScope componentsSeparatedByString:@","] retain];
+    bool inScope = [testsInScope containsObject:[NSString stringWithFormat:@"%s/%s",
+                        QTestResult::currentTestObjectName(), function]];
+
+    if ([XCTestProbe isInverseTestScope])
+        inScope = !inScope;
+
+    return inScope;
+}
+
+void QXcodeTestLogger::enterTestFunction(const char *function)
+{
+    if (!isTestFunctionInActiveScope(function))
+        QTestResult::setSkipCurrentTest(true);
+
+    XCTest *test = [QtTestLibTest testCaseWithInvocation:nil];
+    pushTestRunForTest(test, !QTestResult::skipCurrentTest());
+}
+
+void QXcodeTestLogger::leaveTestFunction()
+{
+    popTestRun();
+}
+
+void QXcodeTestLogger::addIncident(IncidentTypes type, const char *description,
+                                   const char *file, int line)
+{
+    XCTestRun *testRun = [m_testRuns lastObject];
+
+    // The 'expected' argument to recordFailureWithDescription refers to whether
+    // the failure was a regular failed assertion, or an unexpected exception,
+    // so in our case it's always 'YES', and we need to explicitly ignore XFail.
+    if (type == QAbstractTestLogger::XFail) {
+        QTestCharBuffer buf;
+        NSString *testCaseName = [[testRun test] nameForLegacyLogging];
+        QTest::qt_asprintf(&buf, "Test Case '%s' failed expectedly (%s).\n",
+            [testCaseName UTF8String], description);
+        outputString(buf.constData());
+        return;
+    }
+
+    if (type == QAbstractTestLogger::Pass) {
+        // We ignore non-data passes, as we're already reporting that as part of the
+        // normal test case start/stop cycle.
+        if (!(QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag()))
+            return;
+
+        QTestCharBuffer buf;
+        NSString *testCaseName = [[testRun test] nameForLegacyLogging];
+        QTest::qt_asprintf(&buf, "Test Case '%s' passed.\n", [testCaseName UTF8String]);
+        outputString(buf.constData());
+        return;
+    }
+
+    // FIXME: Handle blacklisted tests
+
+    if (!file || !description)
+        return; // Or report?
+
+    [testRun recordFailureWithDescription:[NSString stringWithUTF8String:description]
+        inFile:[NSString stringWithUTF8String:file] atLine:line expected:YES];
+}
+
+void QXcodeTestLogger::addMessage(MessageTypes type, const QString &message,
+                                  const char *file, int line)
+{
+    QTestCharBuffer buf;
+
+    if (QTestLog::verboseLevel() > 0 && (file && line)) {
+        QTest::qt_asprintf(&buf, "%s:%d: ", file, line);
+        outputString(buf.constData());
+    }
+
+    if (type == QAbstractTestLogger::Skip) {
+        XCTestRun *testRun = [m_testRuns lastObject];
+        NSString *testCaseName = [[testRun test] nameForLegacyLogging];
+        QTest::qt_asprintf(&buf, "Test Case '%s' skipped (%s).\n",
+            [testCaseName UTF8String], message.toUtf8().constData());
+    } else {
+        QTest::qt_asprintf(&buf, "%s\n", message.toUtf8().constData());
+    }
+
+    outputString(buf.constData());
+}
+
+void QXcodeTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
+{
+    Q_UNUSED(result);
+}
+
+void QXcodeTestLogger::pushTestRunForTest(XCTest *test, bool start)
+{
+    XCTestRun *testRun = [[test testRunClass] testRunWithTest:test];
+    [m_testRuns addObject:testRun];
+
+    if (start)
+        [testRun start];
+}
+
+XCTestRun *QXcodeTestLogger::popTestRun()
+{
+    XCTestRun *testRun = [[m_testRuns lastObject] retain];
+    [m_testRuns removeLastObject];
+
+    if ([testRun startDate])
+        [testRun stop];
+
+    [[m_testRuns lastObject] addTestRun:testRun];
+    [testRun release];
+
+    return testRun;
+}
+
+bool QXcodeTestLogger::isActive()
+{
+    return s_currentTestLogger;
+}
+
+#pragma GCC diagnostic pop