feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

AutoTest: Support multiple test cases for Qt

Browse Source 
Multiple testcases inside a single executable are not
supported officially, but widely used.
Detect them and handle them as appropriate as possible.
Single test functions or data tags are not selectable
as they cannot get addressed correctly and rely
strongly on the implementation of the test main.

Fixes: QTCREATORBUG-18347
Change-Id: I0f0f42579709d8896e034a6df356cb560291d2ba
Reviewed-by: Christian Stenger <christian.stenger@qt.io>
Reviewed-by: David Schulz <david.schulz@qt.io>
This commit is contained in:
Christian Stenger
2021-04-23 13:40:46 +02:00
parent 0147fdfc8c
commit 615b37193b
8 changed files with 146 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/plugins/autotest/qtest/qttest_utils.cpp
View File

@@ -47,19 +47,24 @@ bool isQTestMacro(const QByteArray &macro)
return valid.contains(macro);
}
QHash<Utils::FilePath, QString> testCaseNamesForFiles(ITestFramework *framework,
const Utils::FilePaths &files)
QHash<Utils::FilePath, TestCases> testCaseNamesForFiles(ITestFramework *framework,
const Utils::FilePaths &files)
{
QHash<Utils::FilePath, QString> result;
QHash<Utils::FilePath, TestCases> result;
TestTreeItem *rootNode = framework->rootNode();
QTC_ASSERT(rootNode, return result);
rootNode->forFirstLevelChildren([&result, &files](ITestTreeItem *child) {
auto toTestCase = [](QtTestTreeItem *item){
return TestCase{item->name(), item->runsMultipleTestcases()};
};
rootNode->forFirstLevelChildren([&](ITestTreeItem *child) {
QtTestTreeItem *qtItem = static_cast<QtTestTreeItem *>(child);
if (files.contains(child->filePath()))
result.insert(child->filePath(), child->name());
child->forFirstLevelChildren([&result, &files, child](ITestTreeItem *grandChild) {
result[child->filePath()].append(toTestCase(qtItem));
child->forFirstLevelChildren([&](ITestTreeItem *grandChild) {
if (files.contains(grandChild->filePath()))
result.insert(grandChild->filePath(), child->name());
result[grandChild->filePath()].append(toTestCase(qtItem));
});
});
return result;

12
src/plugins/autotest/qtest/qttest_utils.h
View File

@@ -36,11 +36,19 @@ namespace Autotest {
class ITestFramework;
namespace Internal {
struct TestCase
{
QString name;
bool multipleTestCases;
};
using TestCases = QList<TestCase>;
namespace QTestUtils {
bool isQTestMacro(const QByteArray &macro);
QHash<Utils::FilePath, QString> testCaseNamesForFiles(ITestFramework *framework,
const Utils::FilePaths &files);
QHash<Utils::FilePath, TestCases> testCaseNamesForFiles(ITestFramework *framework,
const Utils::FilePaths &files);
QMultiHash<Utils::FilePath, Utils::FilePath> alternativeFiles(ITestFramework *framework,
const Utils::FilePaths &files);
QStringList filterInterfering(const QStringList &provided, QStringList *omitted, bool isQuickTest);

65
src/plugins/autotest/qtest/qttestparser.cpp
View File

@@ -26,7 +26,6 @@
#include "qttestparser.h"
#include "qttestframework.h"
#include "qttestvisitors.h"
#include "qttest_utils.h"
#include <cpptools/cppmodelmanager.h>
#include <cpptools/projectpart.h>
@@ -46,6 +45,7 @@ TestTreeItem *QtTestParseResult::createTestTreeItem() const
item->setLine(line);
item->setColumn(column);
item->setInherited(m_inherited);
item->setRunsMultipleTestcases(m_multiTest);
for (const TestParseResult *funcParseResult : children)
item->appendChild(funcParseResult->createTestTreeItem());
@@ -93,13 +93,13 @@ static bool qtTestLibDefined(const Utils::FilePath &fileName)
return false;
}
QString QtTestParser::testClass(const CppTools::CppModelManager *modelManager,
const Utils::FilePath &fileName) const
TestCases QtTestParser::testCases(const CppTools::CppModelManager *modelManager,
const Utils::FilePath &fileName) const
{
const QByteArray &fileContent = getFileContent(fileName);
CPlusPlus::Document::Ptr document = modelManager->document(fileName.toString());
if (document.isNull())
return QString();
return {};
const QList<CPlusPlus::Document::MacroUse> macros = document->macroUses();
@@ -109,8 +109,9 @@ QString QtTestParser::testClass(const CppTools::CppModelManager *modelManager,
const QByteArray name = macro.macro().name();
if (QTestUtils::isQTestMacro(name) && !macro.arguments().isEmpty()) {
const CPlusPlus::Document::Block arg = macro.arguments().at(0);
return QLatin1String(fileContent.mid(int(arg.bytesBegin()),
int(arg.bytesEnd() - arg.bytesBegin())));
const QString name = QLatin1String(fileContent.mid(int(arg.bytesBegin()),
int(arg.bytesEnd() - arg.bytesBegin())));
return { {name, false} };
}
}
// check if one has used a self-defined macro or QTest::qExec() directly
@@ -119,7 +120,7 @@ QString QtTestParser::testClass(const CppTools::CppModelManager *modelManager,
CPlusPlus::AST *ast = document->translationUnit()->ast();
TestAstVisitor astVisitor(document, m_cppSnapshot);
astVisitor.accept(ast);
return astVisitor.className();
return astVisitor.testCases();
}
static CPlusPlus::Document::Ptr declaringDocument(CPlusPlus::Document::Ptr doc,
@@ -284,31 +285,37 @@ bool QtTestParser::processDocument(QFutureInterface<TestParseResultPtr> futureIn
CPlusPlus::Document::Ptr doc = document(fileName);
if (doc.isNull())
return false;
const QString &oldTestCaseName = m_testCaseNames.value(fileName);
if ((!includesQtTest(doc, m_cppSnapshot) || !qtTestLibDefined(fileName)) && oldTestCaseName.isEmpty())
const TestCases &oldTestCases = m_testCases.value(fileName);
if ((!includesQtTest(doc, m_cppSnapshot) || !qtTestLibDefined(fileName))
&& oldTestCases.isEmpty()) {
return false;
}
const CppTools::CppModelManager *modelManager = CppTools::CppModelManager::instance();
QString testCaseName(testClass(modelManager, fileName));
TestCases testCases(testCases(modelManager, fileName));
bool reported = false;
// we might be in a reparse without the original entry point with the QTest::qExec()
if (testCaseName.isEmpty())
testCaseName = oldTestCaseName;
if (!testCaseName.isEmpty()) {
TestCaseData data;
Utils::optional<bool> earlyReturn = fillTestCaseData(testCaseName, doc, data);
if (earlyReturn.has_value())
return earlyReturn.value();
if (testCases.isEmpty() && !oldTestCases.empty())
testCases.append(oldTestCases);
for (const TestCase &testCase : testCases) {
if (!testCase.name.isEmpty()) {
TestCaseData data;
Utils::optional<bool> earlyReturn = fillTestCaseData(testCase.name, doc, data);
if (earlyReturn.has_value() || !data.valid)
continue;
QList<CppTools::ProjectPart::Ptr> projectParts = modelManager->projectPart(fileName);
if (projectParts.isEmpty()) // happens if shutting down while parsing
return false;
QList<CppTools::ProjectPart::Ptr> projectParts = modelManager->projectPart(fileName);
if (projectParts.isEmpty()) // happens if shutting down while parsing
return false;
QtTestParseResult *parseResult
= createParseResult(testCaseName, data, projectParts.first()->projectFile);
futureInterface.reportResult(TestParseResultPtr(parseResult));
return true;
data.multipleTestCases = testCase.multipleTestCases;
QtTestParseResult *parseResult
= createParseResult(testCase.name, data, projectParts.first()->projectFile);
futureInterface.reportResult(TestParseResultPtr(parseResult));
reported = true;
}
}
return false;
return reported;
}
Utils::optional<bool> QtTestParser::fillTestCaseData(
@@ -359,8 +366,10 @@ QtTestParseResult *QtTestParser::createParseResult(
parseResult->line = data.line;
parseResult->column = data.column;
parseResult->proFile = Utils::FilePath::fromString(projectFile);
parseResult->setRunsMultipleTestcases(data.multipleTestCases);
QMap<QString, QtTestCodeLocationAndType>::ConstIterator it = data.testFunctions.begin();
const QMap<QString, QtTestCodeLocationAndType>::ConstIterator end = data.testFunctions.end();
for ( ; it != end; ++it) {
const QtTestCodeLocationAndType &location = it.value();
QString functionName = it.key();
@@ -373,6 +382,7 @@ QtTestParseResult *QtTestParser::createParseResult(
func->line = location.m_line;
func->column = location.m_column;
func->setInherited(location.m_inherited);
func->setRunsMultipleTestcases(data.multipleTestCases);
const QtTestCodeLocationList &tagLocations = tagLocationsFor(func, data.dataTags);
for (const QtTestCodeLocationAndType &tag : tagLocations) {
@@ -385,6 +395,7 @@ QtTestParseResult *QtTestParser::createParseResult(
dataTag->line = tag.m_line;
dataTag->column = tag.m_column;
dataTag->setInherited(tag.m_inherited);
dataTag->setRunsMultipleTestcases(data.multipleTestCases);
func->children.append(dataTag);
}
@@ -396,7 +407,7 @@ QtTestParseResult *QtTestParser::createParseResult(
void QtTestParser::init(const Utils::FilePaths &filesToParse, bool fullParse)
{
if (!fullParse) { // in a full parse cached information might lead to wrong results
m_testCaseNames = QTestUtils::testCaseNamesForFiles(framework(), filesToParse);
m_testCases = QTestUtils::testCaseNamesForFiles(framework(), filesToParse);
m_alternativeFiles = QTestUtils::alternativeFiles(framework(), filesToParse);
}
CppParser::init(filesToParse, fullParse);
@@ -404,7 +415,7 @@ void QtTestParser::init(const Utils::FilePaths &filesToParse, bool fullParse)
void QtTestParser::release()
{
m_testCaseNames.clear();
m_testCases.clear();
m_alternativeFiles.clear();
CppParser::release();
}

11
src/plugins/autotest/qtest/qttestparser.h
View File

@@ -27,6 +27,7 @@
#include "../itestparser.h"
#include "qttest_utils.h"
#include "qttesttreeitem.h"
#include <utils/optional.h>
@@ -42,9 +43,12 @@ public:
explicit QtTestParseResult(ITestFramework *framework) : TestParseResult(framework) {}
void setInherited(bool inherited) { m_inherited = inherited; }
bool inherited() const { return m_inherited; }
void setRunsMultipleTestcases(bool multi) { m_multiTest = multi; }
bool runsMultipleTestcases() const { return m_multiTest; }
TestTreeItem *createTestTreeItem() const override;
private:
bool m_inherited = false;
bool m_multiTest = false;
};
class QtTestParser : public CppParser
@@ -58,8 +62,8 @@ public:
const Utils::FilePath &fileName) override;
private:
QString testClass(const CppTools::CppModelManager *modelManager,
const Utils::FilePath &fileName) const;
TestCases testCases(const CppTools::CppModelManager *modelManager,
const Utils::FilePath &fileName) const;
QHash<QString, QtTestCodeLocationList> checkForDataTags(const QString &fileName) const;
struct TestCaseData {
Utils::FilePath fileName;
@@ -67,6 +71,7 @@ private:
int column = 0;
QMap<QString, QtTestCodeLocationAndType> testFunctions;
QHash<QString, QtTestCodeLocationList> dataTags;
bool multipleTestCases = false;
bool valid = false;
};
@@ -75,7 +80,7 @@ private:
TestCaseData &data) const;
QtTestParseResult *createParseResult(const QString &testCaseName, const TestCaseData &data,
const QString &projectFile) const;
QHash<Utils::FilePath, QString> m_testCaseNames;
QHash<Utils::FilePath, TestCases> m_testCases;
QMultiHash<Utils::FilePath, Utils::FilePath> m_alternativeFiles;
};

73
src/plugins/autotest/qtest/qttesttreeitem.cpp
View File

@@ -48,6 +48,7 @@ TestTreeItem *QtTestTreeItem::copyWithoutChildren()
QtTestTreeItem *copied = new QtTestTreeItem(framework());
copied->copyBasicDataFrom(this);
copied->m_inherited = m_inherited;
copied->m_multiTest = m_multiTest;
return copied;
}
@@ -58,13 +59,24 @@ QVariant QtTestTreeItem::data(int column, int role) const
if (type() == Root)
break;
return QVariant(name() + nameSuffix());
case Qt::ToolTipRole: {
QString toolTip = TestTreeItem::data(column, role).toString();
if (m_multiTest && type() == TestCase) {
toolTip.append(QCoreApplication::translate("QtTestTreeItem",
"<p>Multiple testcases inside a single executable are not officially "
"supported. Depending on the implementation they might get executed "
"or not, but never will be explicitly selectable.</p>"));
}
return toolTip;
break;
}
case Qt::CheckStateRole:
switch (type()) {
case TestDataFunction:
case TestSpecialFunction:
return QVariant();
default:
return checked();
return m_multiTest ? QVariant() : checked();
}
case ItalicRole:
switch (type()) {
@@ -72,7 +84,7 @@ QVariant QtTestTreeItem::data(int column, int role) const
case TestSpecialFunction:
return true;
default:
return false;
return m_multiTest;
}
}
return TestTreeItem::data(column, role);
@@ -87,17 +99,23 @@ Qt::ItemFlags QtTestTreeItem::flags(int column) const
case TestFunction:
return defaultFlags | Qt::ItemIsAutoTristate | Qt::ItemIsUserCheckable;
default:
return TestTreeItem::flags(column);
return m_multiTest ? Qt::ItemIsEnabled | Qt::ItemIsSelectable
: TestTreeItem::flags(column);
}
}
Qt::CheckState QtTestTreeItem::checked() const
{
return m_multiTest ? Qt::Unchecked : TestTreeItem::checked();
}
bool QtTestTreeItem::canProvideTestConfiguration() const
{
switch (type()) {
case TestCase:
case TestFunction:
case TestDataTag:
return true;
return !m_multiTest;
default:
return false;
}
@@ -327,12 +345,14 @@ TestTreeItem *QtTestTreeItem::find(const TestParseResult *result)
}
return nullptr;
}
return findChildByFile(result->fileName);
return findChildByNameAndFile(result->name, result->fileName);
case GroupNode:
return findChildByFile(result->fileName);
return findChildByNameAndFile(result->name, result->fileName);
case TestCase: {
const QtTestParseResult *qtResult = static_cast<const QtTestParseResult *>(result);
return findChildByNameAndInheritance(qtResult->displayName, qtResult->inherited());
return findChildByNameAndInheritanceAndMultiTest(qtResult->displayName,
qtResult->inherited(),
qtResult->runsMultipleTestcases());
}
case TestFunction:
case TestDataFunction:
@@ -349,14 +369,15 @@ TestTreeItem *QtTestTreeItem::findChild(const TestTreeItem *other)
const Type otherType = other->type();
switch (type()) {
case Root:
return findChildByFileAndType(other->filePath(), otherType);
return findChildByFileNameAndType(other->filePath(), other->name(), otherType);
case GroupNode:
return otherType == TestCase ? findChildByFile(other->filePath()) : nullptr;
return otherType == TestCase ? findChildByNameAndFile(other->name(), other->filePath()) : nullptr;
case TestCase: {
if (otherType != TestFunction && otherType != TestDataFunction && otherType != TestSpecialFunction)
return nullptr;
auto qtOther = static_cast<const QtTestTreeItem *>(other);
return findChildByNameAndInheritance(other->name(), qtOther->inherited());
return findChildByNameAndInheritanceAndMultiTest(other->name(), qtOther->inherited(),
qtOther->runsMultipleTestcases());
}
case TestFunction:
case TestDataFunction:
@@ -396,20 +417,38 @@ bool QtTestTreeItem::isGroupable() const
return type() == TestCase;
}
TestTreeItem *QtTestTreeItem::findChildByNameAndInheritance(const QString &name, bool inherited) const
TestTreeItem *QtTestTreeItem::findChildByFileNameAndType(const Utils::FilePath &file,
const QString &name, Type type) const
{
return findFirstLevelChildItem([name, inherited](const TestTreeItem *other) {
return findFirstLevelChildItem([file, name, type](const TestTreeItem *other) {
return other->type() == type && other->filePath() == file && other->name() == name;
});
}
TestTreeItem *QtTestTreeItem::findChildByNameAndInheritanceAndMultiTest(const QString &name,
bool inherited,
bool multiTest) const
{
return findFirstLevelChildItem([name, inherited, multiTest](const TestTreeItem *other) {
const QtTestTreeItem *qtOther = static_cast<const QtTestTreeItem *>(other);
return qtOther->inherited() == inherited && qtOther->name() == name;
return qtOther->inherited() == inherited && qtOther->runsMultipleTestcases() == multiTest
&& qtOther->name() == name;
});
}
QString QtTestTreeItem::nameSuffix() const
{
static QString inheritedSuffix = QString(" [")
+ QCoreApplication::translate("QtTestTreeItem", "inherited")
+ QString("]");
return m_inherited ? inheritedSuffix : QString();
static const QString inherited{QCoreApplication::translate("QtTestTreeItem", "inherited")};
static const QString multi{QCoreApplication::translate("QtTestTreeItem", "multiple testcases")};
QString suffix;
if (m_inherited)
suffix.append(inherited);
if (m_multiTest && type() == TestCase) {
if (m_inherited)
suffix.append(", ");
suffix.append(multi);
}
return suffix.isEmpty() ? suffix : QString{" [" + suffix + "]"};
}
} // namespace Internal

9
src/plugins/autotest/qtest/qttesttreeitem.h
View File

@@ -39,6 +39,7 @@ public:
TestTreeItem *copyWithoutChildren() override;
QVariant data(int column, int role) const override;
Qt::ItemFlags flags(int column) const override;
Qt::CheckState checked() const override;
bool canProvideTestConfiguration() const override;
bool canProvideDebugConfiguration() const override;
ITestConfiguration *testConfiguration() const override;
@@ -52,12 +53,18 @@ public:
bool modify(const TestParseResult *result) override;
void setInherited(bool inherited) { m_inherited = inherited; }
bool inherited() const { return m_inherited; }
void setRunsMultipleTestcases(bool multiTest) { m_multiTest = multiTest; }
bool runsMultipleTestcases() const { return m_multiTest; }
TestTreeItem *createParentGroupNode() const override;
bool isGroupable() const override;
private:
TestTreeItem *findChildByNameAndInheritance(const QString &name, bool inherited) const;
TestTreeItem *findChildByFileNameAndType(const Utils::FilePath &file, const QString &name,
Type type) const;
TestTreeItem *findChildByNameAndInheritanceAndMultiTest(const QString &name, bool inherited,
bool multiTest) const;
QString nameSuffix() const;
bool m_inherited = false;
bool m_multiTest = false;
};
class QtTestCodeLocationAndType : public TestCodeLocationAndType

13
src/plugins/autotest/qtest/qttestvisitors.cpp
View File

@@ -30,6 +30,7 @@
#include <cplusplus/Symbols.h>
#include <cplusplus/TypeOfExpression.h>
#include <cpptools/cppmodelmanager.h>
#include <utils/algorithm.h>
#include <utils/qtcassert.h>
using namespace CPlusPlus;
@@ -131,7 +132,7 @@ bool TestAstVisitor::visit(CallAST *ast)
if (!toeItems.isEmpty()) {
if (const auto pointerType = toeItems.first().type()->asPointerType())
m_className = o.prettyType(pointerType->elementType());
m_classNames.append(o.prettyType(pointerType->elementType()));
}
}
}
@@ -139,7 +140,7 @@ bool TestAstVisitor::visit(CallAST *ast)
}
}
}
return false;
return true;
}
bool TestAstVisitor::visit(CompoundStatementAST *ast)
@@ -152,6 +153,14 @@ bool TestAstVisitor::visit(CompoundStatementAST *ast)
return true;
}
TestCases TestAstVisitor::testCases() const
{
const bool multi = m_classNames.size() > 1;
return Utils::transform(m_classNames, [multi](const QString &className) {
return TestCase{className, multi};
});
}
/********************** Test Data Function AST Visitor ************************/
TestDataFunctionVisitor::TestDataFunctionVisitor(Document::Ptr doc)

5
src/plugins/autotest/qtest/qttestvisitors.h
View File

@@ -25,6 +25,7 @@
#pragma once
#include "qttest_utils.h"
#include "qttesttreeitem.h"
#include <cplusplus/ASTVisitor.h>
@@ -70,10 +71,10 @@ public:
bool visit(CPlusPlus::CallAST *ast) override;
bool visit(CPlusPlus::CompoundStatementAST *ast) override;
QString className() const { return m_className; }
TestCases testCases() const;
private:
QString m_className;
QStringList m_classNames;
CPlusPlus::Scope *m_currentScope = nullptr;
CPlusPlus::Document::Ptr m_currentDoc;
CPlusPlus::Snapshot m_snapshot;