From 4f7eb4e6b8fe6ec8612c9ddf9172954db03ec32e Mon Sep 17 00:00:00 2001 From: Nikolai Kosjar Date: Thu, 15 Jan 2015 13:14:35 +0100 Subject: [PATCH] Plugin Tests: Support additional test objects/classes So far tests running within Qt Creator could be implemented with a private slot in the plugin class starting with "test". Binding the test functions to the plugin object/class is fine for test functions without side effects. But as soon as side effects come into play we need proper initialization and cleanup as it's provided by init(), cleanup(), initTestCase() and cleanupTestCase(). However, implementing these functions in the plugin class is not appropriate since they would affect (potentially quite diverse) test functions. This patch enables us to provide 'ordinary' test classes in which we can handle initialization and clean up the usual way. In addition to the current test invocations, e.g.: # (1) Run all test functions of the plugin ./qtcreator -test CppTools # (2) Run selected test functions of the plugin by stating them ./qtcreator -test CppTools,test_completion,test_builtinsymbolsearcher # (3) Run selected test functions of the plugin by a wild card # expression ./qtcreator -test "CppTools,*pointerdeclaration*" # (4) Run a test function of the plugin with certain test data ./qtcreator -test CppTools,test_completion:template_1 it's now also possible to state the test class in order to execute all test functions of that class: # Run all test functions of a certain class: ./qtcreator -test CppTools,SomeClassWithTests As long as the test class does not start with "test", there should not be any problems. Further, an invocation like (1) now additionally execute all test functions of all test classes. For invocations of type (2), (3) and (4) all test functions of all test classes are considered, too. Change-Id: Ief08a6e9e451c599fd0109b8b8e57f92e3ee19f2 Reviewed-by: Eike Ziller Reviewed-by: Erik Verbruggen --- src/libs/extensionsystem/iplugin.cpp | 14 ++ src/libs/extensionsystem/iplugin.h | 1 + src/libs/extensionsystem/pluginmanager.cpp | 251 +++++++++++++++------ src/libs/extensionsystem/pluginmanager_p.h | 6 +- 4 files changed, 206 insertions(+), 66 deletions(-) diff --git a/src/libs/extensionsystem/iplugin.cpp b/src/libs/extensionsystem/iplugin.cpp index 6df04c64064..66e1e9ce0ad 100644 --- a/src/libs/extensionsystem/iplugin.cpp +++ b/src/libs/extensionsystem/iplugin.cpp @@ -204,6 +204,20 @@ IPlugin::~IPlugin() d = 0; } +/*! + \fn QList IPlugin::createTestObjects() const + + Returns objects that are meant to be passed on to QTest::qExec(). + + This function will be called if the user starts \QC with '-test PluginName' or '-test all'. + + The ownership of returned objects is transferred to caller. +*/ +QList IPlugin::createTestObjects() const +{ + return QList(); +} + /*! \fn PluginSpec *IPlugin::pluginSpec() const Returns the PluginSpec corresponding to this plugin. diff --git a/src/libs/extensionsystem/iplugin.h b/src/libs/extensionsystem/iplugin.h index 04c6d3cc0c6..a7da247bd9c 100644 --- a/src/libs/extensionsystem/iplugin.h +++ b/src/libs/extensionsystem/iplugin.h @@ -64,6 +64,7 @@ public: virtual bool delayedInitialize() { return false; } virtual ShutdownFlag aboutToShutdown() { return SynchronousShutdown; } virtual QObject *remoteCommand(const QStringList & /* options */, const QStringList & /* arguments */) { return 0; } + virtual QList createTestObjects() const; PluginSpec *pluginSpec() const; diff --git a/src/libs/extensionsystem/pluginmanager.cpp b/src/libs/extensionsystem/pluginmanager.cpp index 3548adf4805..cd248abcb1e 100644 --- a/src/libs/extensionsystem/pluginmanager.cpp +++ b/src/libs/extensionsystem/pluginmanager.cpp @@ -51,10 +51,15 @@ #include #include +#include +#include + #ifdef WITH_TESTS #include #endif +#include + Q_LOGGING_CATEGORY(pluginLog, "qtc.extensionsystem") const char C_IGNORED_PLUGINS[] = "Plugins/Ignored"; @@ -870,77 +875,201 @@ void PluginManagerPrivate::deleteAll() } #ifdef WITH_TESTS + +typedef QMap TestPlan; // Object -> selected test functions +typedef QMapIterator TestPlanIterator; + +static bool isTestFunction(const QMetaMethod &metaMethod) +{ + static const QByteArrayList blackList = QByteArrayList() + << "initTestCase()" << "cleanupTestCase()" << "init()" << "cleanup()"; + + if (metaMethod.methodType() != QMetaMethod::Slot) + return false; + + if (metaMethod.access() != QMetaMethod::Private) + return false; + + const QByteArray signature = metaMethod.methodSignature(); + if (blackList.contains(signature)) + return false; + + if (!signature.startsWith("test")) + return false; + + if (signature.endsWith("_data()")) + return false; + + return true; +} + static QStringList testFunctions(const QMetaObject *metaObject) { - QStringList testFunctions; + + QStringList functions; for (int i = metaObject->methodOffset(); i < metaObject->methodCount(); ++i) { - const QByteArray signature = metaObject->method(i).methodSignature(); - if (signature.startsWith("test") && !signature.endsWith("_data()")) { + const QMetaMethod metaMethod = metaObject->method(i); + if (isTestFunction(metaMethod)) { + const QByteArray signature = metaMethod.methodSignature(); const QString method = QString::fromLatin1(signature); const QString methodName = method.left(method.size() - 2); - testFunctions.append(methodName); + functions.append(methodName); } } - return testFunctions; + return functions; } -static QStringList testFunctionsWantedByUser(const PluginSpec *pluginSpec, - const QStringList &availableTestFunctions, - const QStringList &testFunctionsSpecs) +static QStringList matchingTestFunctions(const QStringList &testFunctions, + const QString &matchText) { - QStringList testFunctions; + // There might be a test data suffix like in "testfunction:testdata1". + QString testFunctionName = matchText; + QString testDataSuffix; + const int index = testFunctionName.indexOf(QLatin1Char(':')); + if (index != -1) { + testDataSuffix = testFunctionName.mid(index); + testFunctionName = testFunctionName.left(index); + } - foreach (const QString &userTestFunction, testFunctionsSpecs) { - // There might be a test data suffix like in "testfunction:testdata1". - QString testFunctionName = userTestFunction; - QString testDataSuffix; - const int index = testFunctionName.indexOf(QLatin1Char(':')); - if (index != -1) { - testDataSuffix = testFunctionName.mid(index); - testFunctionName = testFunctionName.left(index); - } - - const QRegExp regExp(testFunctionName, Qt::CaseSensitive, QRegExp::Wildcard); - QStringList matchingFunctions; - foreach (const QString &testFunction, availableTestFunctions) { - if (regExp.exactMatch(testFunction)) - matchingFunctions.append(testFunction); - } - if (!matchingFunctions.isEmpty()) { + const QRegExp regExp(testFunctionName, Qt::CaseSensitive, QRegExp::Wildcard); + QStringList matchingFunctions; + foreach (const QString &testFunction, testFunctions) { + if (regExp.exactMatch(testFunction)) { // If the specified test data is invalid, the QTest framework will // print a reasonable error message for us. - foreach (const QString &matchingFunction, matchingFunctions) - testFunctions.append(matchingFunction + testDataSuffix); - } else { - QTextStream out(stdout); - out << "No test function matches \"" << testFunctionName - << "\" for plugin \"" << pluginSpec->name() << "\"." << endl - << " Available test functions for plugin \"" << pluginSpec->name() - << "\" are:" << endl; - foreach (const QString &testFunction, availableTestFunctions) - out << " " << testFunction << endl; + matchingFunctions.append(testFunction + testDataSuffix); } } - return testFunctions; + return matchingFunctions; } -static int executeTestFunctions(QObject *testObject, const QStringList &functions) +static QObject *objectWithClassName(const QList &objects, const QString &className) { - // Don't run QTest::qExec without any test functions, that'd run *all* slots as tests. - if (functions.isEmpty()) - return 0; - - // QTest::qExec() expects basically QCoreApplication::arguments(), - QStringList qExecArguments = QStringList() - << QLatin1String("arg0") // fake application name - << QLatin1String("-maxwarnings") << QLatin1String("0"); // unlimit output - qExecArguments << functions; - return QTest::qExec(testObject, qExecArguments); + return Utils::findOr(objects, 0, [className] (QObject *object) { + QString candidate = QString::fromUtf8(object->metaObject()->className()); + const int colonIndex = candidate.lastIndexOf(QLatin1Char(':')); + if (colonIndex != -1 && colonIndex < candidate.size() - 1) + candidate = candidate.mid(colonIndex + 1); + return candidate == className; + }); } +static int executeTestPlan(const TestPlan &testPlan) +{ + int failedTests = 0; + + TestPlanIterator it(testPlan); + while (it.hasNext()) { + it.next(); + QObject *testObject = it.key(); + QStringList functions = it.value(); + + // Don't run QTest::qExec without any test functions, that'd run *all* slots as tests. + if (functions.isEmpty()) + continue; + + functions.removeDuplicates(); + + // QTest::qExec() expects basically QCoreApplication::arguments(), + QStringList qExecArguments = QStringList() + << QLatin1String("arg0") // fake application name + << QLatin1String("-maxwarnings") << QLatin1String("0"); // unlimit output + qExecArguments << functions; + failedTests += QTest::qExec(testObject, qExecArguments); + } + + return failedTests; +} + +/// Resulting plan consists of all test functions of the plugin object and +/// all test functions of all test objects of the plugin. +static TestPlan generateCompleteTestPlan(IPlugin *plugin, const QList &testObjects) +{ + TestPlan testPlan; + + testPlan.insert(plugin, testFunctions(plugin->metaObject())); + foreach (QObject *testObject, testObjects) { + const QStringList allFunctions = testFunctions(testObject->metaObject()); + testPlan.insert(testObject, allFunctions); + } + + return testPlan; +} + +/// Resulting plan consists of all matching test functions of the plugin object +/// and all matching functions of all test objects of the plugin. However, if a +/// match text denotes a test class, all test functions of that will be +/// included and the class will not be considered further. +/// +/// Since multiple match texts can match the same function, a test function might +/// be included multiple times for a test object. +static TestPlan generateCustomTestPlan(IPlugin *plugin, const QList &testObjects, + const QStringList &matchTexts) +{ + TestPlan testPlan; + + const QStringList testFunctionsOfPluginObject = testFunctions(plugin->metaObject()); + QStringList matchedTestFunctionsOfPluginObject; + QStringList remainingMatchTexts = matchTexts; + QList remainingTestObjectsOfPlugin = testObjects; + + while (!remainingMatchTexts.isEmpty()) { + const QString matchText = remainingMatchTexts.takeFirst(); + bool matched = false; + + if (QObject *testObject = objectWithClassName(remainingTestObjectsOfPlugin, matchText)) { + // Add all functions of the matching test object + matched = true; + testPlan.insert(testObject, testFunctions(testObject->metaObject())); + remainingTestObjectsOfPlugin.removeAll(testObject); + + } else { + // Add all matching test functions of all remaining test objects + foreach (QObject *testObject, remainingTestObjectsOfPlugin) { + const QStringList allFunctions = testFunctions(testObject->metaObject()); + const QStringList matchingFunctions = matchingTestFunctions(allFunctions, + matchText); + if (!matchingFunctions.isEmpty()) { + matched = true; + testPlan[testObject] += matchingFunctions; + } + } + } + + const QStringList currentMatchedTestFunctionsOfPluginObject + = matchingTestFunctions(testFunctionsOfPluginObject, matchText); + if (!currentMatchedTestFunctionsOfPluginObject.isEmpty()) { + matched = true; + matchedTestFunctionsOfPluginObject += currentMatchedTestFunctionsOfPluginObject; + } + + if (!matched) { + QTextStream out(stdout); + out << "No test function or class matches \"" << matchText + << "\" in plugin \"" << plugin->metaObject()->className() << "\"." << endl; + } + } + + // Add all matching test functions of plugin + if (!matchedTestFunctionsOfPluginObject.isEmpty()) + testPlan.insert(plugin, matchedTestFunctionsOfPluginObject); + + return testPlan; +} + +class ExecuteOnDestruction +{ +public: + ExecuteOnDestruction(std::function code) : destructionCode(code) {} + ~ExecuteOnDestruction() { if (destructionCode) destructionCode(); } + +private: + const std::function destructionCode; +}; + void PluginManagerPrivate::startTests() { if (PluginManager::hasError()) { @@ -951,25 +1080,21 @@ void PluginManagerPrivate::startTests() } foreach (const PluginManagerPrivate::TestSpec &testSpec, testSpecs) { - const PluginSpec * const pluginSpec = testSpec.pluginSpec; - if (!pluginSpec->plugin()) - continue; + IPlugin *plugin = testSpec.pluginSpec->plugin(); + QTC_ASSERT(plugin, continue); - // Collect all test functions of the plugin. - const QStringList allTestFunctions = testFunctions(pluginSpec->plugin()->metaObject()); + const QList testObjects = plugin->createTestObjects(); + ExecuteOnDestruction deleteTestObjects([&]() { qDeleteAll(testObjects); }); - QStringList testFunctionsToExecute; - // User did not specify any test functions, so add every test function. - if (testSpec.testFunctions.isEmpty()) { - testFunctionsToExecute = allTestFunctions; + const bool hasDuplicateTestObjects = testObjects.size() != testObjects.toSet().size(); + QTC_ASSERT(!hasDuplicateTestObjects, continue); + QTC_ASSERT(!testObjects.contains(plugin), continue); - // User specified test functions. Add them if they are valid. - } else { - testFunctionsToExecute = testFunctionsWantedByUser(pluginSpec, allTestFunctions, - testSpec.testFunctions); - } + const TestPlan testPlan = testSpec.testFunctionsOrObjects.isEmpty() + ? generateCompleteTestPlan(plugin, testObjects) + : generateCustomTestPlan(plugin, testObjects, testSpec.testFunctionsOrObjects); - m_failedTests += executeTestFunctions(pluginSpec->plugin(), testFunctionsToExecute); + m_failedTests += executeTestPlan(testPlan); } if (!testSpecs.isEmpty()) QTimer::singleShot(1, this, SLOT(exitWithNumberOfFailedTests())); diff --git a/src/libs/extensionsystem/pluginmanager_p.h b/src/libs/extensionsystem/pluginmanager_p.h index bb456c86bed..dcc69e92ab8 100644 --- a/src/libs/extensionsystem/pluginmanager_p.h +++ b/src/libs/extensionsystem/pluginmanager_p.h @@ -83,10 +83,10 @@ public: class TestSpec { public: - TestSpec(PluginSpec *pluginSpec, const QStringList &testFunctions = QStringList()) - : pluginSpec(pluginSpec), testFunctions(testFunctions) {} + TestSpec(PluginSpec *pluginSpec, const QStringList &testFunctionsOrObjects = QStringList()) + : pluginSpec(pluginSpec), testFunctionsOrObjects(testFunctionsOrObjects) {} PluginSpec *pluginSpec; - QStringList testFunctions; + QStringList testFunctionsOrObjects; }; bool containsTestSpec(PluginSpec *pluginSpec) const