AutoTest: Support missing Catch2 test case macros

Add support for macros defining parameterized test cases
and for test cases using a fixture.

Task-number: QTCREATORBUG-19740
Change-Id: I631009f309cb48d2657acb6e52911e052ff85c5b
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/autotest/catch/catchcodeparser.cpp

@@ -45,9 +45,9 @@ CatchCodeParser::CatchCodeParser(const QByteArray &source, const LanguageFeature
 {
 }
 
-static TestCodeLocationAndType locationAndTypeFromToken(const Token &tkn)
+static CatchTestCodeLocationAndType locationAndTypeFromToken(const Token &tkn)
 {
-    TestCodeLocationAndType locationAndType;
+    CatchTestCodeLocationAndType locationAndType;
     locationAndType.m_type = TestTreeItem::TestFunction;
     locationAndType.m_line = tkn.lineno;
     locationAndType.m_column = 0;
@@ -77,13 +77,12 @@ static QStringList parseTags(const QString &tagsString)
     return tagsList;
 }
 
-TestCodeLocationList CatchCodeParser::findTests()
+CatchTestCodeLocationList CatchCodeParser::findTests()
 {
     m_tokens = tokensForSource(m_source, m_features);
     m_currentIndex = 0;
     for ( ; m_currentIndex < m_tokens.size(); ++m_currentIndex) {
-        const Token &token = m_tokens.at(m_currentIndex);
+        if (m_tokens.at(m_currentIndex).kind() == T_IDENTIFIER)
-        if (token.kind() == T_IDENTIFIER)
             handleIdentifier();
     }
     return m_testCases;
@@ -98,6 +97,17 @@ void CatchCodeParser::handleIdentifier()
         handleTestCase(false);
     } else if (identifier == "SCENARIO") {
         handleTestCase(true);
+    } else if (identifier == "TEMPLATE_TEST_CASE" || identifier == "TEMPLATE_PRODUCT_TEST_CASE"
+               || identifier == "TEMPLATE_LIST_TEST_CASE" || identifier == "TEMPLATE_TEST_CASE_SIG"
+               || identifier == "TEMPLATE_PRODUCT_TEST_CASE_SIG") {
+        handleParameterizedTestCase(false);
+    } else if (identifier == "TEST_CASE_METHOD") {
+        handleFixtureTestCase();
+    } else if (identifier == "TEMPLATE_TEST_CASE_METHOD_SIG"
+               || identifier == "TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG"
+               || identifier == "TEMPLATE_TEST_CASE_METHOD"
+               || identifier == "TEMPLATE_LIST_TEST_CASE_METHOD") {
+        handleParameterizedTestCase(true);
     }
 }
 
@@ -106,8 +116,8 @@ void CatchCodeParser::handleTestCase(bool isScenario)
     if (!skipCommentsUntil(T_LPAREN))
         return;
 
-    Token token = m_tokens.at(m_currentIndex);
+    CatchTestCodeLocationAndType locationAndType
-    TestCodeLocationAndType locationAndType = locationAndTypeFromToken(token);
+            = locationAndTypeFromToken(m_tokens.at(m_currentIndex));
 
     Kind stoppedAt;
     ++m_currentIndex;
@@ -126,6 +136,72 @@ void CatchCodeParser::handleTestCase(bool isScenario)
         testCaseName.prepend("Scenario: "); // use a flag?
 
     locationAndType.m_name = testCaseName;
+    locationAndType.tags = parseTags(tagsString);
+    m_testCases.append(locationAndType);
+}
+
+void CatchCodeParser::handleParameterizedTestCase(bool isFixture)
+{
+    if (!skipCommentsUntil(T_LPAREN))
+        return;
+
+    if (isFixture && !skipFixtureParameter())
+        return;
+
+    CatchTestCodeLocationAndType locationAndType
+            = locationAndTypeFromToken(m_tokens.at(m_currentIndex));
+
+    Kind stoppedAt;
+    ++m_currentIndex;
+    QString testCaseName = getStringLiteral(stoppedAt);
+    QString tagsString;
+
+    if (stoppedAt != T_COMMA)
+        return;
+
+    ++m_currentIndex;
+    tagsString = getStringLiteral(stoppedAt);
+
+    if (stoppedAt == T_COMMA)
+        stoppedAt = skipUntilCorrespondingRParen();
+
+    if (stoppedAt != T_RPAREN)
+        return;
+    locationAndType.m_name = testCaseName;
+    locationAndType.tags = parseTags(tagsString);
+    locationAndType.states = CatchTreeItem::Parameterized;
+    if (isFixture)
+        locationAndType.states |= CatchTreeItem::Fixture;
+    m_testCases.append(locationAndType);
+}
+
+void CatchCodeParser::handleFixtureTestCase()
+{
+    if (!skipCommentsUntil(T_LPAREN))
+        return;
+
+    if (!skipFixtureParameter())
+        return;
+
+    CatchTestCodeLocationAndType locationAndType
+            = locationAndTypeFromToken(m_tokens.at(m_currentIndex));
+
+    Kind stoppedAt;
+    ++m_currentIndex;
+    QString testCaseName = getStringLiteral(stoppedAt);
+    QString tagsString;
+
+    if (stoppedAt == T_COMMA) {
+        ++m_currentIndex;
+        tagsString = getStringLiteral(stoppedAt);
+    }
+
+    if (stoppedAt != T_RPAREN)
+        return;
+
+    locationAndType.m_name = testCaseName;
+    locationAndType.tags = parseTags(tagsString);
+    locationAndType.states = CatchTreeItem::Fixture;
     m_testCases.append(locationAndType);
 }
 
@@ -166,5 +242,30 @@ bool CatchCodeParser::skipCommentsUntil(Kind nextExpectedKind)
     return false;
 }
 
+Kind CatchCodeParser::skipUntilCorrespondingRParen()
+{
+    int openParens = 1;  // we have already one open, looking for the corresponding closing
+    int end = m_tokens.size();
+    while (m_currentIndex < end) {
+        Kind kind = m_tokens.at(m_currentIndex).kind();
+        if (kind == T_LPAREN) {
+            ++openParens;
+        } else if (kind == T_RPAREN) {
+            --openParens;
+            if (openParens == 0)
+                return T_RPAREN;
+        }
+        ++m_currentIndex;
+    }
+    return T_ERROR;
+}
+
+bool CatchCodeParser::skipFixtureParameter()
+{
+    if (!skipCommentsUntil(T_IDENTIFIER))
+        return false;
+    return skipCommentsUntil(T_COMMA);
+}
+
 } // namespace Internal
 } // namespace Autotest

src/plugins/autotest/catch/catchcodeparser.h

@@ -41,13 +41,17 @@ public:
     CatchCodeParser(const QByteArray &source, const CPlusPlus::LanguageFeatures &features,
                     const CPlusPlus::Document::Ptr &doc, const CPlusPlus::Snapshot &snapshot);
     virtual ~CatchCodeParser() = default;
-    TestCodeLocationList findTests();
+    CatchTestCodeLocationList findTests();
 private:
     void handleIdentifier();
     void handleTestCase(bool isScenario);
+    void handleParameterizedTestCase(bool isFixture);
+    void handleFixtureTestCase();
 
     QString getStringLiteral(CPlusPlus::Kind &stoppedAtKind);
     bool skipCommentsUntil(CPlusPlus::Kind nextExpectedKind);   // moves currentIndex if succeeds
+    CPlusPlus::Kind skipUntilCorrespondingRParen();             // moves currentIndex
+    bool skipFixtureParameter();
 
     const QByteArray &m_source;
     const CPlusPlus::LanguageFeatures &m_features;
@@ -55,7 +59,7 @@ private:
     const CPlusPlus::Snapshot &m_snapshot;
     CPlusPlus::Tokens m_tokens;
     int m_currentIndex = 0;
-    TestCodeLocationList m_testCases;
+    CatchTestCodeLocationList m_testCases;
     int m_lineNo = 0;
 };
 

src/plugins/autotest/catch/catchtestparser.cpp

@@ -39,7 +39,16 @@ namespace Internal {
 static bool isCatchTestCaseMacro(const QString &macroName)
 {
     const QStringList validTestCaseMacros = {
-        QStringLiteral("TEST_CASE"), QStringLiteral("SCENARIO")
+        QStringLiteral("TEST_CASE"), QStringLiteral("SCENARIO"),
+        QStringLiteral("TEMPLATE_TEST_CASE"), QStringLiteral("TEMPLATE_PRODUCT_TEST_CASE"),
+        QStringLiteral("TEMPLATE_LIST_TEST_CASE"),
+        QStringLiteral("TEMPLATE_TEST_CASE_SIG"), QStringLiteral("TEMPLATE_PRODUCT_TEST_CASE_SIG"),
+        QStringLiteral("TEST_CASE_METHOD"), QStringLiteral("TEMPLATE_TEST_CASE_METHOD"),
+        QStringLiteral("TEMPLATE_PRODUCT_TEST_CASE_METHOD"),
+        QStringLiteral("TEST_CASE_METHOD"), QStringLiteral("TEMPLATE_TEST_CASE_METHOD_SIG"),
+        QStringLiteral("TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG"),
+        QStringLiteral("TEMPLATE_TEST_CASE_METHOD"),
+        QStringLiteral("TEMPLATE_LIST_TEST_CASE_METHOD")
     };
     return validTestCaseMacros.contains(macroName);
 }
@@ -102,7 +111,7 @@ static bool handleCatchDocument(QFutureInterface<TestParseResultPtr> futureInter
     proFile = projectPart->projectFile;
 
     CatchCodeParser codeParser(fileContent, projectPart->languageFeatures, doc, snapshot);
-    const TestCodeLocationList foundTests = codeParser.findTests();
+    const CatchTestCodeLocationList foundTests = codeParser.findTests();
 
     CatchParseResult *parseResult = new CatchParseResult(framework);
     parseResult->itemType = TestTreeItem::TestCase;
@@ -111,7 +120,7 @@ static bool handleCatchDocument(QFutureInterface<TestParseResultPtr> futureInter
     parseResult->displayName = filePath;
     parseResult->proFile = projectParts.first()->projectFile;
 
-    for (const TestCodeLocationAndType & testLocation : foundTests) {
+    for (const CatchTestCodeLocationAndType & testLocation : foundTests) {
         CatchParseResult *testCase = new CatchParseResult(framework);
         testCase->fileName = filePath;
         testCase->name = testLocation.m_name;
@@ -119,6 +128,7 @@ static bool handleCatchDocument(QFutureInterface<TestParseResultPtr> futureInter
         testCase->itemType = testLocation.m_type;
         testCase->line = testLocation.m_line;
         testCase->column = testLocation.m_column;
+        testCase->states = testLocation.states;
 
         parseResult->children.append(testCase);
     }
@@ -146,6 +156,7 @@ TestTreeItem *CatchParseResult::createTestTreeItem() const
     item->setProFile(proFile);
     item->setLine(line);
     item->setColumn(column);
+    item->setStates(states);
 
     for (const TestParseResult *testSet : children)
         item->appendChild(testSet->createTestTreeItem());

src/plugins/autotest/catch/catchtestparser.h

@@ -24,6 +24,7 @@
 
 #pragma once
 
+#include "catchtreeitem.h"
 #include "../itestparser.h"
 
 namespace Autotest {
@@ -35,6 +36,7 @@ public:
     explicit CatchParseResult(ITestFramework *framework)
         : TestParseResult(framework) {}
     TestTreeItem *createTestTreeItem() const override;
+    CatchTreeItem::TestStates states;
 };
 
 class CatchTestParser : public CppParser

src/plugins/autotest/catch/catchtreeitem.cpp

@@ -33,6 +33,11 @@
 namespace Autotest {
 namespace Internal {
 
+QString CatchTreeItem::testCasesString() const
+{
+    return m_state & CatchTreeItem::Parameterized ? QString(name() + " -*") : name();
+}
+
 QVariant CatchTreeItem::data(int column, int role) const
 {
 
@@ -40,7 +45,7 @@ QVariant CatchTreeItem::data(int column, int role) const
     case Qt::DisplayRole:
         if (type() == Root)
             break;
-        return name();
+        return QString(name() + stateSuffix());
     case Qt::CheckStateRole:
         switch (type()) {
         case Root:
@@ -148,7 +153,7 @@ TestConfiguration *CatchTreeItem::testConfiguration() const
     config->setTestCaseCount(childCount());
     config->setProjectFile(proFile());
     config->setProject(project);
-    config->setTestCases(QStringList(name()));
+    config->setTestCases(QStringList(testCasesString()));
     config->setInternalTargets(internalTargets());
     return config;
 }
@@ -185,14 +190,16 @@ static void collectTestInfo(const TestTreeItem *item,
     if (ignoreCheckState || item->checked() == Qt::Checked) {
         const QString &projectFile = item->childAt(0)->proFile();
         item->forAllChildren([&testCasesForProfile, &projectFile](TestTreeItem *it) {
-            testCasesForProfile[projectFile].names.append(it->name());
+            CatchTreeItem *current = static_cast<CatchTreeItem *>(it);
+            testCasesForProfile[projectFile].names.append(current->testCasesString());
         });
         testCasesForProfile[projectFile].internalTargets.unite(item->internalTargets());
     } else if (item->checked() == Qt::PartiallyChecked) {
         item->forFirstLevelChildren([&testCasesForProfile](TestTreeItem *child) {
             QTC_ASSERT(child->type() == TestTreeItem::TestFunction, return);
             if (child->checked() == Qt::Checked) {
-                testCasesForProfile[child->proFile()].names.append(child->name());
+                CatchTreeItem *current = static_cast<CatchTreeItem *>(child);
+                testCasesForProfile[child->proFile()].names.append(current->testCasesString());
                 testCasesForProfile[child->proFile()].internalTargets.unite(
                             child->internalTargets());
             }
@@ -230,7 +237,8 @@ QList<TestConfiguration *> CatchTreeItem::getTestConfigurationsForFile(const Uti
         QStringList testCases;
 
         item->forFirstLevelChildren([&testCases](TestTreeItem *child) {
-            testCases << child->name();
+            CatchTreeItem *current = static_cast<CatchTreeItem *>(child);
+            testCases << current->testCasesString();
         });
 
         testConfig = new CatchConfiguration(framework());
@@ -244,6 +252,16 @@ QList<TestConfiguration *> CatchTreeItem::getTestConfigurationsForFile(const Uti
     return result;
 }
 
+QString CatchTreeItem::stateSuffix() const
+{
+    QStringList types;
+    if (m_state & CatchTreeItem::Parameterized)
+        types.append(QCoreApplication::translate("CatchTreeItem", "parameterized"));
+    if (m_state & CatchTreeItem::Fixture)
+        types.append(QCoreApplication::translate("CatchTreeItem", "fixture"));
+    return types.isEmpty() ? QString() : QString(" [" + types.join(", ") + ']');
+}
+
 QList<TestConfiguration *> CatchTreeItem::getTestConfigurations(bool ignoreCheckState) const
 {
     QList<TestConfiguration *> result;

src/plugins/autotest/catch/catchtreeitem.h

@@ -32,10 +32,22 @@ namespace Internal {
 class CatchTreeItem : public TestTreeItem
 {
 public:
+    enum TestState
+    {
+        Normal        = 0x0,
+        Parameterized = 0x1,
+        Fixture       = 0x2
+    };
+    Q_FLAGS(TestState)
+    Q_DECLARE_FLAGS(TestStates, TestState)
+
     explicit CatchTreeItem(ITestFramework *framework, const QString &name = QString(),
                            const QString &filePath = QString(), Type type = Root)
         : TestTreeItem(framework, name, filePath, type) {}
 
+    void setStates(CatchTreeItem::TestStates state) { m_state = state; }
+    QString testCasesString() const;
+
     QVariant data(int column, int role) const override;
 
     TestTreeItem *copyWithoutChildren() override;
@@ -53,8 +65,19 @@ public:
     QList<TestConfiguration *> getTestConfigurationsForFile(const Utils::FilePath &fileName) const override;
 
 private:
+    QString stateSuffix() const;
     QList<TestConfiguration *> getTestConfigurations(bool ignoreCheckState) const;
+    TestStates m_state = Normal;
 };
 
+class CatchTestCodeLocationAndType : public TestCodeLocationAndType
+{
+public:
+    CatchTreeItem::TestStates states = CatchTreeItem::Normal;
+    QStringList tags; // TODO: use them for the item
+};
+
+typedef QVector<CatchTestCodeLocationAndType> CatchTestCodeLocationList;
+
 } // namespace Internal
 } // namespace Autotest