feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

AutoTest: Allow arguments for test runs

Browse Source 
Arguments specified for run configurations were
ignored so far, but sometimes it might help to process
them. Add the possibility and a respective setting
to be able to pass arguments to the test run.

Task-number: QTCREATORBUG-17630
Change-Id: Ie64b784e8477efa02f50ce6b4cf3e55864952880
Reviewed-by: Leena Miettinen <riitta-leena.miettinen@qt.io>
Reviewed-by: Oliver Wolff <oliver.wolff@qt.io>
This commit is contained in:
Christian Stenger
2017-09-01 14:59:39 +02:00
parent b1373c4853
commit 4068ec44b6
14 changed files with 163 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
src/plugins/autotest/gtest/gtestconfiguration.cpp
View File

@@ -27,7 +27,11 @@
#include "gtestconstants.h"
#include "gtestoutputreader.h"
#include "gtestsettings.h"
#include "../autotestplugin.h"
#include "../testframeworkmanager.h"
#include "../testsettings.h"
#include <utils/algorithm.h>
namespace Autotest {
namespace Internal {
@@ -38,12 +42,46 @@ TestOutputReader *GTestConfiguration::outputReader(const QFutureInterface<TestRe
return new GTestOutputReader(fi, app, buildDirectory(), projectFile());
}
QStringList GTestConfiguration::argumentsForTestRunner() const
QStringList filterInterfering(const QStringList &provided, QStringList *omitted)
{
static const QSet<QString> knownInterferingOptions { "--gtest_list_tests",
"--gtest_filter=",
"--gtest_also_run_disabled_tests",
"--gtest_repeat=",
"--gtest_shuffle",
"--gtest_random_seed=",
"--gtest_output=",
"--gtest_stream_result_to=",
"--gtest_break_on_failure",
"--gtest_throw_on_failure",
"--gtest_color="
};
QSet<QString> allowed = Utils::filtered(provided.toSet(), [] (const QString &arg) {
return Utils::allOf(knownInterferingOptions, [&arg] (const QString &interfering) {
return !arg.startsWith(interfering);
});
});
if (omitted) {
QSet<QString> providedSet = provided.toSet();
providedSet.subtract(allowed);
omitted->append(providedSet.toList());
}
return allowed.toList();
}
QStringList GTestConfiguration::argumentsForTestRunner(QStringList *omitted) const
{
static const Core::Id id
= Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(GTest::Constants::FRAMEWORK_NAME);
QStringList arguments;
if (AutotestPlugin::instance()->settings()->processArgs) {
arguments << filterInterfering(runnable().commandLineArguments.split(
' ', QString::SkipEmptyParts), omitted);
}
const QStringList &testSets = testCases();
if (testSets.size())
arguments << "--gtest_filter=" + testSets.join(':');

2
src/plugins/autotest/gtest/gtestconfiguration.h
View File

@@ -36,7 +36,7 @@ public:
explicit GTestConfiguration() {}
TestOutputReader *outputReader(const QFutureInterface<TestResultPtr> &fi,
QProcess *app) const override;
QStringList argumentsForTestRunner() const override;
QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
};
} // namespace Internal

58
src/plugins/autotest/qtest/qttest_utils.cpp
View File

@@ -27,9 +27,11 @@
#include "qttesttreeitem.h"
#include "../testframeworkmanager.h"
#include <utils/algorithm.h>
#include <utils/qtcassert.h>
#include <QByteArrayList>
#include <QSet>
namespace Autotest {
namespace Internal {
@@ -83,6 +85,62 @@ QMultiHash<QString, QString> alternativeFiles(const Core::Id &id, const QStringL
return result;
}
QStringList filterInterfering(const QStringList &provided, QStringList *omitted, bool isQuickTest)
{
static const QSet<QString> knownInterferingSingleOptions {
"-txt", "-xml", "-csv", "-xunitxml", "-lightxml", "-silent", "-v1", "-v2", "-vs", "-vb",
"-functions", "-datatags", "-nocrashhandler", "-callgrind", "-perf", "-perfcounterlist",
"-tickcounter", "-eventcounter", "-help"
};
static const QSet<QString> knownInterferingOptionWithParameter = { "-o" };
static const QSet<QString> knownAllowedOptionsWithParameter {
"-eventdelay", "-keydelay", "-mousedelay", "-maxwarnings", "-perfcounter",
"-minimumvalue", "-minimumtotal", "-iterations", "-median"
};
// handle Quick options as well
static const QSet<QString> knownInterferingQuickOption = { "-qtquick1" };
static const QSet<QString> knownAllowedQuickOptionsWithParameter {
"-import", "-plugins", "-input"
};
QStringList allowed;
auto it = provided.cbegin();
auto end = provided.cend();
for ( ; it != end; ++it) {
QString currentOpt = *it;
if (knownAllowedOptionsWithParameter.contains(currentOpt)) {
allowed.append(currentOpt);
++it;
QTC_ASSERT(it != end, return QStringList());
allowed.append(*it);
} else if (knownInterferingOptionWithParameter.contains(currentOpt)) {
if (omitted) {
omitted->append(currentOpt);
++it;
QTC_ASSERT(it != end, return QStringList());
omitted->append(*it);
}
} else if (knownInterferingSingleOptions.contains(currentOpt)) {
if (omitted)
omitted->append(currentOpt);
} else if (isQuickTest) {
if (knownAllowedQuickOptionsWithParameter.contains(currentOpt)) {
allowed.append(currentOpt);
++it;
QTC_ASSERT(it != end, return QStringList());
allowed.append(*it);
} else if (knownInterferingQuickOption.contains(currentOpt)) {
if (omitted)
omitted->append(currentOpt);
}
} else { // might be bad, but we cannot know anything
allowed.append(currentOpt);
}
}
return allowed;
}
} // namespace QTestUtils
} // namespace Internal
} // namespace Autotest

1
src/plugins/autotest/qtest/qttest_utils.h
View File

@@ -36,6 +36,7 @@ namespace QTestUtils {
bool isQTestMacro(const QByteArray &macro);
QHash<QString, QString> testCaseNamesForFiles(const Core::Id &id, const QStringList &files);
QMultiHash<QString, QString> alternativeFiles(const Core::Id &id, const QStringList &files);
QStringList filterInterfering(const QStringList &provided, QStringList *omitted, bool isQuickTest);
} // namespace QTestUtils
} // namespace Internal

10
src/plugins/autotest/qtest/qttestconfiguration.cpp
View File

@@ -27,7 +27,10 @@
#include "qttestconstants.h"
#include "qttestoutputreader.h"
#include "qttestsettings.h"
#include "qttest_utils.h"
#include "../autotestplugin.h"
#include "../testframeworkmanager.h"
#include "../testsettings.h"
namespace Autotest {
namespace Internal {
@@ -48,12 +51,17 @@ TestOutputReader *QtTestConfiguration::outputReader(const QFutureInterface<TestR
return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
}
QStringList QtTestConfiguration::argumentsForTestRunner() const
QStringList QtTestConfiguration::argumentsForTestRunner(QStringList *omitted) const
{
static const Core::Id id
= Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
QStringList arguments;
if (AutotestPlugin::instance()->settings()->processArgs) {
arguments.append(QTestUtils::filterInterfering(
runnable().commandLineArguments.split(' ', QString::SkipEmptyParts),
omitted, false));
}
TestFrameworkManager *manager = TestFrameworkManager::instance();
auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
if (qtSettings.isNull())

2
src/plugins/autotest/qtest/qttestconfiguration.h
View File

@@ -36,7 +36,7 @@ public:
explicit QtTestConfiguration() {}
TestOutputReader *outputReader(const QFutureInterface<TestResultPtr> &fi,
QProcess *app) const override;
QStringList argumentsForTestRunner() const override;
QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
};
} // namespace Internal

11
src/plugins/autotest/quick/quicktestconfiguration.cpp
View File

@@ -27,7 +27,10 @@
#include "../qtest/qttestconstants.h"
#include "../qtest/qttestoutputreader.h"
#include "../qtest/qttestsettings.h"
#include "../qtest/qttest_utils.h"
#include "../autotestplugin.h"
#include "../testframeworkmanager.h"
#include "../testsettings.h"
namespace Autotest {
namespace Internal {
@@ -47,12 +50,18 @@ TestOutputReader *QuickTestConfiguration::outputReader(const QFutureInterface<Te
return new QtTestOutputReader(fi, app, buildDirectory(), QtTestOutputReader::PlainText);
}
QStringList QuickTestConfiguration::argumentsForTestRunner() const
QStringList QuickTestConfiguration::argumentsForTestRunner(QStringList *omitted) const
{
static const Core::Id id
= Core::Id(Constants::FRAMEWORK_PREFIX).withSuffix(QtTest::Constants::FRAMEWORK_NAME);
QStringList arguments;
if (AutotestPlugin::instance()->settings()->processArgs) {
arguments.append(QTestUtils::filterInterfering
(runnable().commandLineArguments.split(' ', QString::SkipEmptyParts),
omitted, true));
}
TestFrameworkManager *manager = TestFrameworkManager::instance();
auto qtSettings = qSharedPointerCast<QtTestSettings>(manager->settingsForTestFramework(id));
if (qtSettings.isNull())

2
src/plugins/autotest/quick/quicktestconfiguration.h
View File

@@ -36,7 +36,7 @@ public:
explicit QuickTestConfiguration() {}
TestOutputReader *outputReader(const QFutureInterface<TestResultPtr> &fi,
QProcess *app) const override;
QStringList argumentsForTestRunner() const override;
QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const override;
void setUnnamedOnly(bool unnamedOnly);
bool unnamedOnly() const { return m_unnamedOnly; }

3
src/plugins/autotest/testconfiguration.h
View File

@@ -82,9 +82,10 @@ public:
QString runConfigDisplayName() const { return m_guessedConfiguration ? m_guessedFrom
: m_displayName; }
ProjectExplorer::StandardRunnable runnable() const { return m_runnable; }
virtual TestOutputReader *outputReader(const QFutureInterface<TestResultPtr> &fi,
QProcess *app) const = 0;
virtual QStringList argumentsForTestRunner() const = 0;
virtual QStringList argumentsForTestRunner(QStringList *omitted = nullptr) const = 0;
private:
QStringList m_testCases;

30
src/plugins/autotest/testrunner.cpp
View File

@@ -118,6 +118,15 @@ static QString rcInfo(const TestConfiguration * const config)
return info + " \"" + config->runConfigDisplayName() + '"';
}
static QString constructOmittedDetailsString(const QStringList &omitted)
{
QString details = TestRunner::tr("Omitted the following arguments specified on the run "
"configuration page for \"%1\":");
for (const QString &arg : omitted)
details += "\n" + arg;
return details;
}
static void performTestRun(QFutureInterface<TestResultPtr> &futureInterface,
const QList<TestConfiguration *> selectedTests,
const TestSettings &settings)
@@ -173,7 +182,13 @@ static void performTestRun(QFutureInterface<TestResultPtr> &futureInterface,
continue;
}
testProcess.setArguments(testConfiguration->argumentsForTestRunner());
QStringList omitted;
testProcess.setArguments(testConfiguration->argumentsForTestRunner(&omitted));
if (!omitted.isEmpty()) {
const QString &details = constructOmittedDetailsString(omitted);
futureInterface.reportResult(TestResultPtr(new FaultyTestResult(Result::MessageWarn,
details.arg(testConfiguration->displayName()))));
}
testProcess.setWorkingDirectory(testConfiguration->workingDirectory());
if (Utils::HostOsInfo::isWindowsHost())
environment.insert("QT_LOGGING_TO_CONSOLE", "1");
@@ -339,12 +354,15 @@ void TestRunner::debugTests()
return;
}
ProjectExplorer::StandardRunnable inferior;
QStringList omitted;
ProjectExplorer::StandardRunnable inferior = config->runnable();
inferior.executable = commandFilePath;
inferior.commandLineArguments = config->argumentsForTestRunner().join(' ');
inferior.environment = config->environment();
inferior.workingDirectory = config->workingDirectory();
inferior.commandLineArguments = config->argumentsForTestRunner(&omitted).join(' ');
if (!omitted.isEmpty()) {
const QString &details = constructOmittedDetailsString(omitted);
emit testResultReady(TestResultPtr(new FaultyTestResult(Result::MessageWarn,
details.arg(config->displayName()))));
}
auto debugger = new Debugger::DebuggerRunTool(runControl);
debugger->setInferior(inferior);
debugger->setRunControlName(config->displayName());

3
src/plugins/autotest/testsettings.cpp
View File

@@ -41,6 +41,7 @@ static const char limitResultOutputKey[] = "LimitResultOutput";
static const char autoScrollKey[] = "AutoScrollResults";
static const char filterScanKey[] = "FilterScan";
static const char filtersKey[] = "WhiteListFilters";
static const char processArgsKey[] = "ProcessArgs";
static const int defaultTimeout = 60000;
@@ -57,6 +58,7 @@ void TestSettings::toSettings(QSettings *s) const
s->setValue(omitRunConfigWarnKey, omitRunConfigWarn);
s->setValue(limitResultOutputKey, limitResultOutput);
s->setValue(autoScrollKey, autoScroll);
s->setValue(processArgsKey, processArgs);
s->setValue(filterScanKey, filterScan);
s->setValue(filtersKey, whiteListFilters);
// store frameworks and their current active state
@@ -73,6 +75,7 @@ void TestSettings::fromSettings(QSettings *s)
omitRunConfigWarn = s->value(omitRunConfigWarnKey, false).toBool();
limitResultOutput = s->value(limitResultOutputKey, true).toBool();
autoScroll = s->value(autoScrollKey, true).toBool();
processArgs = s->value(processArgsKey, false).toBool();
filterScan = s->value(filterScanKey, false).toBool();
whiteListFilters = s->value(filtersKey, QStringList()).toStringList();
// try to get settings for registered frameworks

1
src/plugins/autotest/testsettings.h
View File

@@ -48,6 +48,7 @@ struct TestSettings
bool limitResultOutput = true;
bool autoScroll = true;
bool filterScan = false;
bool processArgs = false;
QHash<Core::Id, bool> frameworks;
QStringList whiteListFilters;
};

2
src/plugins/autotest/testsettingspage.cpp
View File

@@ -148,6 +148,7 @@ void TestSettingsWidget::setSettings(const TestSettings &settings)
m_ui.omitRunConfigWarnCB->setChecked(settings.omitRunConfigWarn);
m_ui.limitResultOutputCB->setChecked(settings.limitResultOutput);
m_ui.autoScrollCB->setChecked(settings.autoScroll);
m_ui.processArgsCB->setChecked(settings.processArgs);
m_ui.filterGroupBox->setChecked(settings.filterScan);
populateFrameworksListWidget(settings.frameworks);
populateFiltersWidget(settings.whiteListFilters);
@@ -161,6 +162,7 @@ TestSettings TestSettingsWidget::settings() const
result.omitRunConfigWarn = m_ui.omitRunConfigWarnCB->isChecked();
result.limitResultOutput = m_ui.limitResultOutputCB->isChecked();
result.autoScroll = m_ui.autoScrollCB->isChecked();
result.processArgs = m_ui.processArgsCB->isChecked();
result.filterScan = m_ui.filterGroupBox->isChecked();
result.frameworks = frameworks();
result.whiteListFilters = filters();

11
src/plugins/autotest/testsettingspage.ui
View File

@@ -73,6 +73,17 @@
</property>
</widget>
</item>
<item>
<widget class="QCheckBox" name="processArgsCB">
<property name="toolTip">
<string>Allow passing arguments specified on the respective run configuration.
Warning: this is an experimental feature and might lead to failing to execute the test executable.</string>
</property>
<property name="text">
<string>Process arguments</string>
</property>
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_2" stretch="0,0,0">
<property name="spacing">