diff options
Diffstat (limited to 'src/testlib/qxctestlogger.mm')
| -rw-r--r-- | src/testlib/qxctestlogger.mm | 501 |
1 files changed, 501 insertions, 0 deletions
diff --git a/src/testlib/qxctestlogger.mm b/src/testlib/qxctestlogger.mm new file mode 100644 index 0000000000..576f46acea --- /dev/null +++ b/src/testlib/qxctestlogger.mm @@ -0,0 +1,501 @@ +/**************************************************************************** +** +** Copyright (C) 2015 The Qt Company Ltd. +** Contact: http://www.qt.io/licensing/ +** +** This file is part of the QtTest module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL21$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see http://www.qt.io/terms-conditions. For further +** information use the contact form at http://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 or version 3 as published by the Free +** Software Foundation and appearing in the file LICENSE.LGPLv21 and +** LICENSE.LGPLv3 included in the packaging of this file. Please review the +** following information to ensure the GNU Lesser General Public License +** requirements will be met: https://www.gnu.org/licenses/lgpl.html and +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** As a special exception, The Qt Company gives you certain additional +** rights. These rights are described in The Qt Company LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qxctestlogger_p.h" + +#include <QtCore/qstring.h> + +#include <QtTest/private/qtestlog_p.h> +#include <QtTest/private/qtestresult_p.h> + +#import <XCTest/XCTest.h> + +// --------------------------------------------------------- + +@interface XCTestProbe (Private) ++ (BOOL)isTesting; ++ (void)runTests:(id)unusedArgument; ++ (NSString*)testScope; ++ (BOOL)isInverseTestScope; +@end + +@interface XCTestDriver : NSObject ++ (XCTestDriver*)sharedTestDriver; +@property (readonly, assign) NSObject *IDEConnection; +@end + +@interface XCTest (Private) +- (NSString *)nameForLegacyLogging; +@end + +#pragma GCC diagnostic push // Ignore XCTestProbe deprecation +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + +// --------------------------------------------------------- + +@interface QtTestLibWrapper : XCTestCase +@end + +@interface QtTestLibTests : XCTestSuite ++ (XCTestSuiteRun*)testRun; +@end + +@interface QtTestLibTest : XCTestCase +@property (nonatomic, retain) NSString* testObjectName; +@property (nonatomic, retain) NSString* testFunctionName; +@end + +// --------------------------------------------------------- + +class ThreadBarriers +{ +public: + enum Barrier { + XCTestCanStartTesting, + XCTestHaveStarted, + QtTestsCanStartTesting, + QtTestsHaveCompleted, + XCTestsHaveCompleted, + BarrierCount + }; + + static ThreadBarriers *get() + { + static ThreadBarriers instance; + return &instance; + } + + static void initialize() { get(); } + + void wait(Barrier barrier) { dispatch_semaphore_wait(barriers[barrier], DISPATCH_TIME_FOREVER); } + void signal(Barrier barrier) { dispatch_semaphore_signal(barriers[barrier]); } + +private: + #define FOREACH_BARRIER(cmd) for (int i = 0; i < BarrierCount; ++i) { cmd } + + ThreadBarriers() { FOREACH_BARRIER(barriers[i] = dispatch_semaphore_create(0);) } + ~ThreadBarriers() { FOREACH_BARRIER(dispatch_release(barriers[i]);) } + + dispatch_semaphore_t barriers[BarrierCount]; +}; + +#define WAIT_FOR_BARRIER(b) ThreadBarriers::get()->wait(ThreadBarriers::b); +#define SIGNAL_BARRIER(b) ThreadBarriers::get()->signal(ThreadBarriers::b); + +// --------------------------------------------------------- + +@implementation QtTestLibWrapper + ++ (void)load +{ + NSAutoreleasePool *autoreleasepool = [[NSAutoreleasePool alloc] init]; + + if (![XCTestProbe isTesting]) + return; + + if (!([NSDate timeIntervalSinceReferenceDate] > 0)) + qFatal("error: Device date '%s' is bad, likely set to update automatically. Please correct.", + [NSDate date].description.UTF8String); + + XCTestDriver *testDriver = nil; + if ([QtTestLibWrapper usingTestManager]) + testDriver = [XCTestDriver sharedTestDriver]; + + // Spawn off task to run test infrastructure on separate thread so that we can + // let main() execute like normal on the main thread. The queue will never be + // destroyed, so there's no point in trying to keep a proper retain count. + dispatch_async(dispatch_queue_create("io.qt.QTestLib.xctest-wrapper", DISPATCH_QUEUE_SERIAL), ^{ + Q_ASSERT(![NSThread isMainThread]); + [XCTestProbe runTests:nil]; + Q_UNREACHABLE(); + }); + + // Initialize barriers before registering exit handler so that the + // semaphores stay alive until after the exit handler completes. + ThreadBarriers::initialize(); + + // We register an exit handler so that we can intercept when main() completes + // and let the XCTest thread finish up. For main() functions that never started + // testing using QtTestLib we also need to signal that xcTestsCanStart. + atexit_b(^{ + Q_ASSERT([NSThread isMainThread]); + + // In case not started by startLogging + SIGNAL_BARRIER(XCTestCanStartTesting); + + // [XCTestProbe runTests:] ends up calling [XCTestProbe exitTests:] after + // all test suites have completed, which calls exit(). We use that to signal + // to the main thread that it's free to continue its exit handler. + atexit_b(^{ + Q_ASSERT(![NSThread isMainThread]); + SIGNAL_BARRIER(XCTestsHaveCompleted); + + // Block forever so that the main thread does all the cleanup + dispatch_semaphore_wait(dispatch_semaphore_create(0), DISPATCH_TIME_FOREVER); + }); + + SIGNAL_BARRIER(QtTestsHaveCompleted); + + // Ensure XCTest complets the top level tests suite + WAIT_FOR_BARRIER(XCTestsHaveCompleted); + }); + + // Let test driver (Xcode) connection setup complete before continuing + if ([[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"]) { + while (!testDriver.IDEConnection) + [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]]; + } + + // Wait for our QtTestLib test suite to run before running main + WAIT_FOR_BARRIER(QtTestsCanStartTesting); + + // Prevent XCTestProbe from re-launching runTests on application startup + [[NSNotificationCenter defaultCenter] removeObserver:[XCTestProbe class] + name:[NSString stringWithFormat:@"%@DidFinishLaunchingNotification", + #if defined(Q_OS_OSX) + @"NSApplication" + #elif defined(Q_OS_IOS) + @"UIApplication" + #endif + ] + object:nil]; + + [autoreleasepool release]; +} + ++ (id)defaultTestSuite +{ + return [[QtTestLibTests alloc] initWithName:@"QtTestLib"]; +} + ++ (BOOL)usingTestManager +{ + return [[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"]; +} + +@end + +// --------------------------------------------------------- + +static XCTestSuiteRun *s_qtTestSuiteRun = 0; + +@implementation QtTestLibTests + +- (void)performTest:(XCTestSuiteRun *)testSuiteRun +{ + Q_ASSERT(![NSThread isMainThread]); + + Q_ASSERT(!s_qtTestSuiteRun); + s_qtTestSuiteRun = testSuiteRun; + + SIGNAL_BARRIER(QtTestsCanStartTesting); + + // Wait for main() to complete, or a QtTestLib test to start, so we + // know if we should start the QtTestLib test suite. + WAIT_FOR_BARRIER(XCTestCanStartTesting); + + if (QXcodeTestLogger::isActive()) + [testSuiteRun start]; + + SIGNAL_BARRIER(XCTestHaveStarted); + + // All test reporting happens on main thread from now on. Wait until + // main() completes before allowing the XCTest thread to continue. + WAIT_FOR_BARRIER(QtTestsHaveCompleted); + + if ([testSuiteRun startDate]) + [testSuiteRun stop]; +} + ++ (XCTestSuiteRun*)testRun +{ + return s_qtTestSuiteRun; +} + +@end + +// --------------------------------------------------------- + +@implementation QtTestLibTest + +- (id)initWithInvocation:(NSInvocation *)invocation +{ + if (self = [super initWithInvocation:invocation]) { + // The test object name and function name are used by XCTest after QtTestLib has + // reset them, so we need to store them up front for each XCTestCase. + self.testObjectName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()]; + self.testFunctionName = [NSString stringWithUTF8String:QTestResult::currentTestFunction()]; + } + + return self; +} + +- (NSString *)testClassName +{ + return self.testObjectName; +} + +- (NSString *)testMethodName +{ + return self.testFunctionName; +} + +- (NSString *)nameForLegacyLogging +{ + NSString *name = [NSString stringWithFormat:@"%@::%@", [self testClassName], [self testMethodName]]; + if (QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag()) { + const char *currentDataTag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""; + const char *globalDataTag = QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : ""; + const char *filler = (currentDataTag[0] && globalDataTag[0]) ? ":" : ""; + name = [name stringByAppendingString:[NSString stringWithFormat:@"(%s%s%s)", + globalDataTag, filler, currentDataTag]]; + } + + return name; +} + +@end + +// --------------------------------------------------------- + +bool QXcodeTestLogger::canLogTestProgress() +{ + return [XCTestProbe isTesting]; // FIXME: Exclude xctool +} + +int QXcodeTestLogger::parseCommandLineArgument(const char *argument) +{ + if (strncmp(argument, "-NS", 3) == 0 || strncmp(argument, "-Apple", 6) == 0) + return 2; // -NSTreatUnknownArgumentsAsOpen, -ApplePersistenceIgnoreState, etc, skip argument + else if (strcmp(argument, "--use-testmanagerd") == 0) + return 2; // Skip UID argument + else if (strncmp(argument, "-XCTest", 7) == 0) + return 2; // -XCTestInvertScope, -XCTest scope, etc, skip argument + else if (strcmp(argument + (strlen(argument) - 7), ".xctest") == 0) + return 1; // Skip test bundle + else + return 0; +} + +// --------------------------------------------------------- + +QXcodeTestLogger *QXcodeTestLogger::s_currentTestLogger = 0; + +// --------------------------------------------------------- + +QXcodeTestLogger::QXcodeTestLogger() + : QAbstractTestLogger(0) + , m_testRuns([[NSMutableArray arrayWithCapacity:2] retain]) + +{ + Q_ASSERT(!s_currentTestLogger); + s_currentTestLogger = this; +} + +QXcodeTestLogger::~QXcodeTestLogger() +{ + s_currentTestLogger = 0; + [m_testRuns release]; +} + +void QXcodeTestLogger::startLogging() +{ + SIGNAL_BARRIER(XCTestCanStartTesting); + + static dispatch_once_t onceToken; + dispatch_once (&onceToken, ^{ + WAIT_FOR_BARRIER(XCTestHaveStarted); + }); + + // Scope test object suite under top level QtTestLib test run + [m_testRuns addObject:[QtTestLibTests testRun]]; + + NSString *suiteName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()]; + pushTestRunForTest([XCTestSuite testSuiteWithName:suiteName], true); +} + +void QXcodeTestLogger::stopLogging() +{ + popTestRun(); +} + +static bool isTestFunctionInActiveScope(const char *function) +{ + static NSString *testScope = [XCTestProbe testScope]; + + enum TestScope { Unknown, All, None, Self, Selected }; + static TestScope activeScope = Unknown; + + if (activeScope == Unknown) { + if ([testScope isEqualToString:@"All"]) + activeScope = All; + else if ([testScope isEqualToString:@"None"]) + activeScope = None; + else if ([testScope isEqualToString:@"Self"]) + activeScope = Self; + else + activeScope = Selected; + } + + if (activeScope == All) + return true; + else if (activeScope == None) + return false; + else if (activeScope == Self) + return true; // Investigate + + Q_ASSERT(activeScope == Selected); + + static NSArray *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain]; + if ([forcedTests containsObject:[NSString stringWithUTF8String:function]]) + return true; + + static NSArray *testsInScope = [[testScope componentsSeparatedByString:@","] retain]; + bool inScope = [testsInScope containsObject:[NSString stringWithFormat:@"%s/%s", + QTestResult::currentTestObjectName(), function]]; + + if ([XCTestProbe isInverseTestScope]) + inScope = !inScope; + + return inScope; +} + +void QXcodeTestLogger::enterTestFunction(const char *function) +{ + if (!isTestFunctionInActiveScope(function)) + QTestResult::setSkipCurrentTest(true); + + XCTest *test = [QtTestLibTest testCaseWithInvocation:nil]; + pushTestRunForTest(test, !QTestResult::skipCurrentTest()); +} + +void QXcodeTestLogger::leaveTestFunction() +{ + popTestRun(); +} + +void QXcodeTestLogger::addIncident(IncidentTypes type, const char *description, + const char *file, int line) +{ + XCTestRun *testRun = [m_testRuns lastObject]; + + // The 'expected' argument to recordFailureWithDescription refers to whether + // the failure was a regular failed assertion, or an unexpected exception, + // so in our case it's always 'YES', and we need to explicitly ignore XFail. + if (type == QAbstractTestLogger::XFail) { + QTestCharBuffer buf; + NSString *testCaseName = [[testRun test] nameForLegacyLogging]; + QTest::qt_asprintf(&buf, "Test Case '%s' failed expectedly (%s).\n", + [testCaseName UTF8String], description); + outputString(buf.constData()); + return; + } + + if (type == QAbstractTestLogger::Pass) { + // We ignore non-data passes, as we're already reporting that as part of the + // normal test case start/stop cycle. + if (!(QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag())) + return; + + QTestCharBuffer buf; + NSString *testCaseName = [[testRun test] nameForLegacyLogging]; + QTest::qt_asprintf(&buf, "Test Case '%s' passed.\n", [testCaseName UTF8String]); + outputString(buf.constData()); + return; + } + + // FIXME: Handle blacklisted tests + + if (!file || !description) + return; // Or report? + + [testRun recordFailureWithDescription:[NSString stringWithUTF8String:description] + inFile:[NSString stringWithUTF8String:file] atLine:line expected:YES]; +} + +void QXcodeTestLogger::addMessage(MessageTypes type, const QString &message, + const char *file, int line) +{ + QTestCharBuffer buf; + + if (QTestLog::verboseLevel() > 0 && (file && line)) { + QTest::qt_asprintf(&buf, "%s:%d: ", file, line); + outputString(buf.constData()); + } + + if (type == QAbstractTestLogger::Skip) { + XCTestRun *testRun = [m_testRuns lastObject]; + NSString *testCaseName = [[testRun test] nameForLegacyLogging]; + QTest::qt_asprintf(&buf, "Test Case '%s' skipped (%s).\n", + [testCaseName UTF8String], message.toUtf8().constData()); + } else { + QTest::qt_asprintf(&buf, "%s\n", message.toUtf8().constData()); + } + + outputString(buf.constData()); +} + +void QXcodeTestLogger::addBenchmarkResult(const QBenchmarkResult &result) +{ + Q_UNUSED(result); +} + +void QXcodeTestLogger::pushTestRunForTest(XCTest *test, bool start) +{ + XCTestRun *testRun = [[test testRunClass] testRunWithTest:test]; + [m_testRuns addObject:testRun]; + + if (start) + [testRun start]; +} + +XCTestRun *QXcodeTestLogger::popTestRun() +{ + XCTestRun *testRun = [[m_testRuns lastObject] retain]; + [m_testRuns removeLastObject]; + + if ([testRun startDate]) + [testRun stop]; + + [[m_testRuns lastObject] addTestRun:testRun]; + [testRun release]; + + return testRun; +} + +bool QXcodeTestLogger::isActive() +{ + return s_currentTestLogger; +} + +#pragma GCC diagnostic pop |