AutoTest: Allow arguments for test runs

Arguments specified for run configurations were
ignored so far, but sometimes it might help to process
them. Add the possibility and a respective setting
to be able to pass arguments to the test run.

Task-number: QTCREATORBUG-17630
Change-Id: Ie64b784e8477efa02f50ce6b4cf3e55864952880
Reviewed-by: Leena Miettinen <riitta-leena.miettinen@qt.io>
Reviewed-by: Oliver Wolff <oliver.wolff@qt.io>

src/plugins/autotest/gtest/gtestconfiguration.cpp

@@ -27,7 +27,11 @@
 #include "gtestconstants.h"
 #include "gtestoutputreader.h"
 #include "gtestsettings.h"
+#include "../autotestplugin.h"
 #include "../testframeworkmanager.h"
+#include "../testsettings.h"
+
+#include <utils/algorithm.h>
 
 namespace Autotest {
 namespace Internal {
@@ -38,12 +42,46 @@ TestOutputReader *GTestConfiguration::outputReader(const QFutureInterface<TestRe
     return new GTestOutputReader(fi, app, buildDirectory(), projectFile());
 }
 
-QStringList GTestConfiguration::argumentsForTestRunner() const
+QStringList filterInterfering(const QStringList &provided, QStringList *omitted)
+{
+    static const QSet<QString> knownInterferingOptions { "--gtest_list_tests",
+                                                         "--gtest_filter=",
+                                                         "--gtest_also_run_disabled_tests",
+                                                         "--gtest_repeat=",
+                                                         "--gtest_shuffle",
+                                                         "--gtest_random_seed=",
+                                                         "--gtest_output=",
+                                                         "--gtest_stream_result_to=",
+                                                         "--gtest_break_on_failure",
+                                                         "--gtest_throw_on_failure",
+                                                         "--gtest_color="
+                                                         };
+
+    QSet<QString> allowed = Utils::filtered(provided.toSet(), [] (const QString &arg) {
+        return Utils::allOf(knownInterferingOptions, [&arg] (const QString &interfering) {
+            return !arg.startsWith(interfering);
+        });
+    });
+
+    if (omitted) {
+        QSet<QString> providedSet = provided.toSet();
+        providedSet.subtract(allowed);
+        omitted->append(providedSet.toList());
+    }
+    return allowed.toList();
+}
+
+QStringList GTestConfiguration::argumentsForTestRunner(QStringList *omitted) const
 {
     static const Core::Id id
             = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(GTest::Constants::FRAMEWORK_NAME);
 
     QStringList arguments;
+    if (AutotestPlugin::instance()->settings()->processArgs) {
+        arguments << filterInterfering(runnable().commandLineArguments.split(
+                                           ' ', QString::SkipEmptyParts), omitted);
+    }
+
     const QStringList &testSets = testCases();
     if (testSets.size())
         arguments << "--gtest_filter=" + testSets.join(':');

src/plugins/autotest/gtest/gtestconfiguration.h

@@ -36,7 +36,7 @@ public:
     explicit GTestConfiguration() {}
     TestOutputReader *outputReader(const QFutureInterface<TestResultPtr> &fi,
                                    QProcess *app) const override;
-    QStringList argumentsForTestRunner() const override;
+    QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
 };
 
 } // namespace Internal

src/plugins/autotest/qtest/qttest_utils.cpp

@@ -27,9 +27,11 @@
 #include "qttesttreeitem.h"
 #include "../testframeworkmanager.h"
 
+#include <utils/algorithm.h>
 #include <utils/qtcassert.h>
 
 #include <QByteArrayList>
+#include <QSet>
 
 namespace Autotest {
 namespace Internal {
@@ -83,6 +85,62 @@ QMultiHash<QString, QString> alternativeFiles(const Core::Id &id, const QStringL
     return result;
 }
 
+QStringList filterInterfering(const QStringList &provided, QStringList *omitted, bool isQuickTest)
+{
+    static const QSet<QString> knownInterferingSingleOptions {
+        "-txt", "-xml", "-csv", "-xunitxml", "-lightxml", "-silent", "-v1", "-v2", "-vs", "-vb",
+        "-functions", "-datatags", "-nocrashhandler", "-callgrind", "-perf", "-perfcounterlist",
+        "-tickcounter", "-eventcounter", "-help"
+    };
+    static const QSet<QString> knownInterferingOptionWithParameter = { "-o" };
+    static const QSet<QString> knownAllowedOptionsWithParameter {
+        "-eventdelay", "-keydelay", "-mousedelay", "-maxwarnings", "-perfcounter",
+        "-minimumvalue", "-minimumtotal", "-iterations", "-median"
+    };
+
+    // handle Quick options as well
+    static const QSet<QString> knownInterferingQuickOption = { "-qtquick1" };
+    static const QSet<QString> knownAllowedQuickOptionsWithParameter {
+        "-import", "-plugins", "-input"
+    };
+
+    QStringList allowed;
+    auto it = provided.cbegin();
+    auto end = provided.cend();
+    for ( ; it != end; ++it) {
+        QString currentOpt = *it;
+        if (knownAllowedOptionsWithParameter.contains(currentOpt)) {
+            allowed.append(currentOpt);
+            ++it;
+            QTC_ASSERT(it != end, return QStringList());
+            allowed.append(*it);
+        } else if (knownInterferingOptionWithParameter.contains(currentOpt)) {
+            if (omitted) {
+                omitted->append(currentOpt);
+                ++it;
+                QTC_ASSERT(it != end, return QStringList());
+                omitted->append(*it);
+            }
+        } else if (knownInterferingSingleOptions.contains(currentOpt)) {
+            if (omitted)
+                omitted->append(currentOpt);
+        } else if (isQuickTest) {
+            if (knownAllowedQuickOptionsWithParameter.contains(currentOpt)) {
+                allowed.append(currentOpt);
+                ++it;
+                QTC_ASSERT(it != end, return QStringList());
+                allowed.append(*it);
+            } else if (knownInterferingQuickOption.contains(currentOpt)) {
+                if (omitted)
+                    omitted->append(currentOpt);
+            }
+        } else { // might be bad, but we cannot know anything
+            allowed.append(currentOpt);
+        }
+    }
+    return allowed;
+}
+
 } // namespace QTestUtils
 } // namespace Internal
 } // namespace Autotest

src/plugins/autotest/qtest/qttest_utils.h

@@ -36,6 +36,7 @@ namespace QTestUtils {
 bool isQTestMacro(const QByteArray &macro);
 QHash<QString, QString> testCaseNamesForFiles(const Core::Id &id, const QStringList &files);
 QMultiHash<QString, QString> alternativeFiles(const Core::Id &id, const QStringList &files);
+QStringList filterInterfering(const QStringList &provided, QStringList *omitted, bool isQuickTest);
 
 } // namespace QTestUtils
 } // namespace Internal

src/plugins/autotest/qtest/qttestconfiguration.cpp

@@ -27,7 +27,10 @@
 #include "qttestconstants.h"
 #include "qttestoutputreader.h"
 #include "qttestsettings.h"
+#include "qttest_utils.h"
+#include "../autotestplugin.h"
 #include "../testframeworkmanager.h"
+#include "../testsettings.h"
 
 namespace Autotest {
 namespace Internal {
@@ -48,12 +51,17 @@ TestOutputReader *QtTestConfiguration::outputReader(const QFutureInterface<TestR
         return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
 }
 
-QStringList QtTestConfiguration::argumentsForTestRunner() const
+QStringList QtTestConfiguration::argumentsForTestRunner(QStringList *omitted) const
 {
     static const Core::Id id
             = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
 
     QStringList arguments;
+    if (AutotestPlugin::instance()->settings()->processArgs) {
+        arguments.append(QTestUtils::filterInterfering(
+                             runnable().commandLineArguments.split(' ', QString::SkipEmptyParts),
+                             omitted, false));
+    }
     TestFrameworkManager *manager = TestFrameworkManager::instance();
     auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
     if (qtSettings.isNull())

src/plugins/autotest/qtest/qttestconfiguration.h

@@ -36,7 +36,7 @@ public:
     explicit QtTestConfiguration() {}
     TestOutputReader *outputReader(const QFutureInterface<TestResultPtr> &fi,
                                    QProcess *app) const override;
-    QStringList argumentsForTestRunner() const override;
+    QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
 };
 
 } // namespace Internal

src/plugins/autotest/quick/quicktestconfiguration.cpp

@@ -27,7 +27,10 @@
 #include "../qtest/qttestconstants.h"
 #include "../qtest/qttestoutputreader.h"
 #include "../qtest/qttestsettings.h"
+#include "../qtest/qttest_utils.h"
+#include "../autotestplugin.h"
 #include "../testframeworkmanager.h"
+#include "../testsettings.h"
 
 namespace Autotest {
 namespace Internal {
@@ -47,12 +50,18 @@ TestOutputReader *QuickTestConfiguration::outputReader(const QFutureInterface<Te
         return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
 }
 
-QStringList QuickTestConfiguration::argumentsForTestRunner() const
+QStringList QuickTestConfiguration::argumentsForTestRunner(QStringList *omitted) const
 {
     static const Core::Id id
             = Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
 
     QStringList arguments;
+    if (AutotestPlugin::instance()->settings()->processArgs) {
+        arguments.append(QTestUtils::filterInterfering
+                         (runnable().commandLineArguments.split(' ', QString::SkipEmptyParts),
+                          omitted, true));
+    }
+
     TestFrameworkManager *manager = TestFrameworkManager::instance();
     auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
     if (qtSettings.isNull())

src/plugins/autotest/quick/quicktestconfiguration.h

@@ -36,7 +36,7 @@ public:
     explicit QuickTestConfiguration() {}
     TestOutputReader *outputReader(const QFutureInterface<TestResultPtr> &fi,
                                    QProcess *app) const override;
-    QStringList argumentsForTestRunner() const override;
+    QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
 
     void setUnnamedOnly(bool unnamedOnly);
     bool unnamedOnly() const { return m_unnamedOnly; }

src/plugins/autotest/testconfiguration.h

@@ -82,9 +82,10 @@ public:
     QString runConfigDisplayName() const { return m_guessedConfiguration ? m_guessedFrom
                                                                          : m_displayName; }
 
+    ProjectExplorer::StandardRunnable runnable() const { return m_runnable; }
     virtual TestOutputReader *outputReader(const QFutureInterface<TestResultPtr> &fi,
                                            QProcess *app) const = 0;
-    virtual QStringList argumentsForTestRunner() const = 0;
+    virtual QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const = 0;
 
 private:
     QStringList m_testCases;

src/plugins/autotest/testrunner.cpp

@@ -118,6 +118,15 @@ static QString rcInfo(const TestConfiguration * const config)
     return info + " \"" + config->runConfigDisplayName() + '"';
 }
 
+static QString constructOmittedDetailsString(const QStringList &omitted)
+{
+    QString details = TestRunner::tr("Omitted the following arguments specified on the run "
+                                     "configuration page for \"%1\":");
+    for (const QString &arg : omitted)
+        details += "\n" + arg;
+    return details;
+}
+
 static void performTestRun(QFutureInterface<TestResultPtr> &futureInterface,
                            const QList<TestConfiguration *> selectedTests,
                            const TestSettings &settings)
@@ -173,7 +182,13 @@ static void performTestRun(QFutureInterface<TestResultPtr> &futureInterface,
             continue;
         }
 
-        testProcess.setArguments(testConfiguration->argumentsForTestRunner());
+        QStringList omitted;
+        testProcess.setArguments(testConfiguration->argumentsForTestRunner(&omitted));
+        if (!omitted.isEmpty()) {
+            const QString &details = constructOmittedDetailsString(omitted);
+            futureInterface.reportResult(TestResultPtr(new FaultyTestResult(Result::MessageWarn,
+                details.arg(testConfiguration->displayName()))));
+        }
         testProcess.setWorkingDirectory(testConfiguration->workingDirectory());
         if (Utils::HostOsInfo::isWindowsHost())
             environment.insert("QT_LOGGING_TO_CONSOLE", "1");
@@ -339,12 +354,15 @@ void TestRunner::debugTests()
         return;
     }
 
-    ProjectExplorer::StandardRunnable inferior;
+    QStringList omitted;
+    ProjectExplorer::StandardRunnable inferior = config->runnable();
     inferior.executable = commandFilePath;
-    inferior.commandLineArguments = config->argumentsForTestRunner().join(' ');
+    inferior.commandLineArguments = config->argumentsForTestRunner(&omitted).join(' ');
-    inferior.environment = config->environment();
+    if (!omitted.isEmpty()) {
-    inferior.workingDirectory = config->workingDirectory();
+        const QString &details = constructOmittedDetailsString(omitted);
-
+        emit testResultReady(TestResultPtr(new FaultyTestResult(Result::MessageWarn,
+            details.arg(config->displayName()))));
+    }
     auto debugger = new Debugger::DebuggerRunTool(runControl);
     debugger->setInferior(inferior);
     debugger->setRunControlName(config->displayName());

src/plugins/autotest/testsettings.cpp

@@ -41,6 +41,7 @@ static const char limitResultOutputKey[]    = "LimitResultOutput";
 static const char autoScrollKey[]           = "AutoScrollResults";
 static const char filterScanKey[]           = "FilterScan";
 static const char filtersKey[]              = "WhiteListFilters";
+static const char processArgsKey[]          = "ProcessArgs";
 
 static const int defaultTimeout = 60000;
 
@@ -57,6 +58,7 @@ void TestSettings::toSettings(QSettings *s) const
     s->setValue(omitRunConfigWarnKey, omitRunConfigWarn);
     s->setValue(limitResultOutputKey, limitResultOutput);
     s->setValue(autoScrollKey, autoScroll);
+    s->setValue(processArgsKey, processArgs);
     s->setValue(filterScanKey, filterScan);
     s->setValue(filtersKey, whiteListFilters);
     // store frameworks and their current active state
@@ -73,6 +75,7 @@ void TestSettings::fromSettings(QSettings *s)
     omitRunConfigWarn = s->value(omitRunConfigWarnKey, false).toBool();
     limitResultOutput = s->value(limitResultOutputKey, true).toBool();
     autoScroll = s->value(autoScrollKey, true).toBool();
+    processArgs = s->value(processArgsKey, false).toBool();
     filterScan = s->value(filterScanKey, false).toBool();
     whiteListFilters = s->value(filtersKey, QStringList()).toStringList();
     // try to get settings for registered frameworks

src/plugins/autotest/testsettings.h

@@ -48,6 +48,7 @@ struct TestSettings
     bool limitResultOutput = true;
     bool autoScroll = true;
     bool filterScan = false;
+    bool processArgs = false;
     QHash<Core::Id, bool> frameworks;
     QStringList whiteListFilters;
 };

src/plugins/autotest/testsettingspage.cpp

@@ -148,6 +148,7 @@ void TestSettingsWidget::setSettings(const TestSettings &settings)
     m_ui.omitRunConfigWarnCB->setChecked(settings.omitRunConfigWarn);
     m_ui.limitResultOutputCB->setChecked(settings.limitResultOutput);
     m_ui.autoScrollCB->setChecked(settings.autoScroll);
+    m_ui.processArgsCB->setChecked(settings.processArgs);
     m_ui.filterGroupBox->setChecked(settings.filterScan);
     populateFrameworksListWidget(settings.frameworks);
     populateFiltersWidget(settings.whiteListFilters);
@@ -161,6 +162,7 @@ TestSettings TestSettingsWidget::settings() const
     result.omitRunConfigWarn = m_ui.omitRunConfigWarnCB->isChecked();
     result.limitResultOutput = m_ui.limitResultOutputCB->isChecked();
     result.autoScroll = m_ui.autoScrollCB->isChecked();
+    result.processArgs = m_ui.processArgsCB->isChecked();
     result.filterScan = m_ui.filterGroupBox->isChecked();
     result.frameworks = frameworks();
     result.whiteListFilters = filters();

src/plugins/autotest/testsettingspage.ui

@@ -73,6 +73,17 @@
             </property>
            </widget>
           </item>
+          <item>
+           <widget class="QCheckBox" name="processArgsCB">
+            <property name="toolTip">
+             <string>Allow passing arguments specified on the respective run configuration.
+Warning: this is an experimental feature and might lead to failing to execute the test executable.</string>
+            </property>
+            <property name="text">
+             <string>Process arguments</string>
+            </property>
+           </widget>
+          </item>
           <item>
            <layout class="QHBoxLayout" name="horizontalLayout_2" stretch="0,0,0">
             <property name="spacing">