Plugin tests: Allow to specify which test functions to run

This simplifies debugging single test functions.

Test data can be specified the same way as for QTest executables.

New syntax for the -test option:
    -test <plugin> [testfunction[:testdata]]...

Examples:
    ./qtcreator -test Git testDiffFileResolving
    ./qtcreator -test Git testDiffFileResolving testStatusParsing:"DU"

Change-Id: Ifea6b114bfc0fabe3e9ddffcc2fd90af157052ec
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@digia.com>
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>
Reviewed-by: Daniel Teske <daniel.teske@digia.com>

src/libs/extensionsystem/optionsparser.cpp

@@ -107,8 +107,8 @@ bool OptionsParser::checkForTestOption()
     if (nextToken(RequiredToken)) {
         if (m_currentArg == QLatin1String("all")) {
             foreach (PluginSpec *spec, m_pmPrivate->pluginSpecs) {
-                if (spec && !m_pmPrivate->testSpecs.contains(spec))
+                if (spec && !m_pmPrivate->containsTestSpec(spec))
-                    m_pmPrivate->testSpecs.append(spec);
+                    m_pmPrivate->testSpecs.append(PluginManagerPrivate::TestSpec(spec));
             }
         } else {
             PluginSpec *spec = m_pmPrivate->pluginByName(m_currentArg);
@@ -117,8 +117,20 @@ bool OptionsParser::checkForTestOption()
                     *m_errorString = QCoreApplication::translate("PluginManager",
                                                                  "The plugin '%1' does not exist.").arg(m_currentArg);
                 m_hasError = true;
-            } else if (!m_pmPrivate->testSpecs.contains(spec)) {
+            } else if (!m_pmPrivate->containsTestSpec(spec)) {
-                m_pmPrivate->testSpecs.append(spec);
+                    // Collect optional test functions. Everything following the plugin
+                    // name until the next option is interpreted as a test function. E.g.
+                    // in './qtcreator -test Git myFile' the argument 'myFile' will be
+                    // be interpreted as an function name and not a file to open.
+                    const QStringList::const_iterator current(m_it);
+                    QStringList testFunctions;
+                    while (nextToken() && !m_currentArg.startsWith(QLatin1Char('-')))
+                           testFunctions.append(m_currentArg);
+                    // Make sure a following nextToken() call will get the current/next option.
+                    if (current != m_it && m_it != m_end)
+                        --m_it;
+                    m_pmPrivate->testSpecs.append(
+                        PluginManagerPrivate::TestSpec(spec, testFunctions));
             }
         }
     }

src/libs/extensionsystem/pluginmanager.cpp

@@ -616,8 +616,11 @@ void PluginManager::formatOptions(QTextStream &str, int optionIndentation, int d
                  QString(), QLatin1String("Profile plugin loading"),
                  optionIndentation, descriptionIndentation);
 #ifdef WITH_TESTS
-    formatOption(str, QLatin1String(OptionsParser::TEST_OPTION),
+    formatOption(str, QString::fromLatin1(OptionsParser::TEST_OPTION)
-                 QLatin1String("plugin|all"), QLatin1String("Run plugin's tests"),
+                 + QLatin1String(" <plugin> [testfunction[:testdata]]..."), QString(),
+                 QLatin1String("Run plugin's tests"), optionIndentation, descriptionIndentation);
+    formatOption(str, QString::fromLatin1(OptionsParser::TEST_OPTION) + QLatin1String(" all"),
+                 QString(), QLatin1String("Run tests from all plugins"),
                  optionIndentation, descriptionIndentation);
 #endif
 }
@@ -662,13 +665,15 @@ void PluginManager::formatPluginVersions(QTextStream &str)
 void PluginManager::startTests()
 {
 #ifdef WITH_TESTS
-    foreach (PluginSpec *pluginSpec, d->testSpecs) {
+    foreach (const PluginManagerPrivate::TestSpec &testSpec, d->testSpecs) {
+        const PluginSpec * const pluginSpec = testSpec.pluginSpec;
         if (!pluginSpec->plugin())
             continue;
+
+        // Collect all test functions/methods of the plugin.
+        QStringList allTestFunctions;
         const QMetaObject *mo = pluginSpec->plugin()->metaObject();
-        QStringList methods;
+
-        methods.append(QLatin1String("arg0"));
-        // We only want slots starting with "test"
         for (int i = mo->methodOffset(); i < mo->methodCount(); ++i) {
 #if QT_VERSION >= 0x050000
             const QByteArray signature = mo->method(i).methodSignature();
@@ -677,13 +682,49 @@ void PluginManager::startTests()
 #endif
             if (signature.startsWith("test") && !signature.endsWith("_data()")) {
                 const QString method = QString::fromLatin1(signature);
-                methods.append(method.left(method.size()-2));
+                allTestFunctions.append(method.left(method.size()-2));
             }
         }
+
+        QStringList testFunctionsToExecute;
+
+        // User did not specify any test functions, so add every test function.
+        if (testSpec.testFunctions.isEmpty()) {
+            testFunctionsToExecute = allTestFunctions;
+
+        // User specified test functions. Add them if they are valid.
+        } else {
+            foreach (const QString &userTestFunction, testSpec.testFunctions) {
+                // There might be a test data suffix like in "testfunction:testdata1".
+                QString testFunctionName = userTestFunction;
+                const int index = testFunctionName.indexOf(QLatin1Char(':'));
+                if (index != -1)
+                    testFunctionName = testFunctionName.left(index);
+
+                if (allTestFunctions.contains(testFunctionName)) {
+                    // If the specified test data is invalid, the QTest framework will
+                    // print a reasonable error message for us.
+                    testFunctionsToExecute.append(userTestFunction);
+                } else {
+                    QTextStream out(stdout);
+                    out << "Unknown test function \"" << testFunctionName
+                        << "\" for plugin \"" << pluginSpec->name() << "\"." << endl
+                        << "  Available test functions for plugin \"" << pluginSpec->name()
+                        << "\" are:" << endl;
+                    foreach (const QString &testFunction, allTestFunctions)
+                        out << "    " << testFunction << endl;
+                }
+            }
+        }
+
+        // QTest::qExec() expects basically QCoreApplication::arguments(),
+        // so prepend a fake argument for the application name.
+        testFunctionsToExecute.prepend(QLatin1String("arg0"));
+
         // Don't run QTest::qExec with only one argument, that'd run
         // *all* slots as tests.
-        if (methods.size() > 1)
+        if (testFunctionsToExecute.size() > 1)
-            QTest::qExec(pluginSpec->plugin(), methods);
+            QTest::qExec(pluginSpec->plugin(), testFunctionsToExecute);
     }
     if (!d->testSpecs.isEmpty())
         QTimer::singleShot(1, QCoreApplication::instance(), SLOT(quit()));

src/libs/extensionsystem/pluginmanager_p.h

@@ -79,9 +79,26 @@ public:
     void writeSettings();
     void disablePluginIndirectly(PluginSpec *spec);
 
+    class TestSpec {
+    public:
+        TestSpec(PluginSpec *pluginSpec, const QStringList &testFunctions = QStringList())
+            : pluginSpec(pluginSpec), testFunctions(testFunctions) {}
+        PluginSpec *pluginSpec;
+        QStringList testFunctions;
+    };
+
+    bool containsTestSpec(PluginSpec *pluginSpec) const
+    {
+        foreach (const TestSpec &testSpec, testSpecs) {
+            if (testSpec.pluginSpec == pluginSpec)
+                return true;
+        }
+        return false;
+    }
+
     QHash<QString, PluginCollection *> pluginCategories;
     QList<PluginSpec *> pluginSpecs;
-    QList<PluginSpec *> testSpecs;
+    QList<TestSpec> testSpecs;
     QStringList pluginPaths;
     QString extension;
     QList<QObject *> allObjects; // ### make this a QList<QPointer<QObject> > > ?