AutoTest: Add some (debugging-related) options

Task-number: QTCREATORBUG-16694
Change-Id: Ie2c32c7e8fd73ba351d65a510af0d4d3574c9691
Reviewed-by: Leena Miettinen <riitta-leena.miettinen@qt.io>
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/autotest/gtest/gtestconfiguration.cpp

@@ -50,6 +50,13 @@ QStringList GTestConfiguration::argumentsForTestRunner(const TestSettings &setti
         arguments << QLatin1String("--gtest_shuffle")
                   << QString::fromLatin1("--gtest_random_seed=%1").arg(settings.gtestSeed);
     }
+    if (settings.gtestThrowOnFailure)
+        arguments << "--gtest_throw_on_failure";
+
+    if (runMode() == DebuggableTestConfiguration::Debug) {
+        if (settings.gtestBreakOnFailure)
+            arguments << "--gtest_break_on_failure";
+    }
     return arguments;
 }
 

src/plugins/autotest/qtest/qttestconfiguration.cpp

@@ -45,6 +45,12 @@ QStringList QtTestConfiguration::argumentsForTestRunner(const TestSettings &sett
         arguments << metricsOption;
     if (testCases().count())
         arguments << testCases();
+
+    if (runMode() == DebuggableTestConfiguration::Debug) {
+        if (settings.qtestNoCrashHandler)
+            arguments << "-nocrashhandler";
+    }
+
     return arguments;
 }
 

src/plugins/autotest/testsettings.cpp

@@ -41,11 +41,14 @@ static const char omitRunConfigWarnKey[] = "OmitRCWarnings";
 static const char limitResultOutputKey[] = "LimitResultOutput";
 static const char autoScrollKey[] = "AutoScrollResults";
 static const char alwaysParseKey[] = "AlwaysParse";
+static const char qtestNoCrashhandlerKey[] = "NoCrashhandlerOnDebug";
 static const char gtestRunDisabledKey[] = "RunDisabledGTests";
 static const char gtestRepeatKey[] = "RepeatGTests";
 static const char gtestShuffleKey[] = "ShuffleGTests";
 static const char gtestIterationsKey[] = "IterationsGTests";
 static const char gtestSeedKey[] = "SeedGTests";
+static const char gtestThrowOnFailureKey[] = "ThrowOnFailure";
+static const char gtestBreakOnFailureKey[] = "BreakOnFailure";
 
 static const int defaultTimeout = 60000;
 
@@ -65,11 +68,14 @@ void TestSettings::toSettings(QSettings *s) const
     s->setValue(QLatin1String(limitResultOutputKey), limitResultOutput);
     s->setValue(QLatin1String(autoScrollKey), autoScroll);
     s->setValue(QLatin1String(alwaysParseKey), alwaysParse);
+    s->setValue(QLatin1String(qtestNoCrashhandlerKey), qtestNoCrashHandler);
     s->setValue(QLatin1String(gtestRunDisabledKey), gtestRunDisabled);
     s->setValue(QLatin1String(gtestRepeatKey), gtestRepeat);
     s->setValue(QLatin1String(gtestShuffleKey), gtestShuffle);
     s->setValue(QLatin1String(gtestIterationsKey), gtestIterations);
     s->setValue(QLatin1String(gtestSeedKey), gtestSeed);
+    s->setValue(QLatin1String(gtestBreakOnFailureKey), gtestBreakOnFailure);
+    s->setValue(QLatin1String(gtestThrowOnFailureKey), gtestThrowOnFailure);
     // store frameworks and their current active state
     for (const Core::Id &id : frameworks.keys())
         s->setValue(QLatin1String(id.name()), frameworks.value(id));
@@ -104,11 +110,14 @@ void TestSettings::fromSettings(const QSettings *s)
     limitResultOutput = s->value(root + QLatin1String(limitResultOutputKey), true).toBool();
     autoScroll = s->value(root + QLatin1String(autoScrollKey), true).toBool();
     alwaysParse = s->value(root + QLatin1String(alwaysParseKey), true).toBool();
+    qtestNoCrashHandler = s->value(root + QLatin1String(qtestNoCrashhandlerKey), true).toBool();
     gtestRunDisabled = s->value(root + QLatin1String(gtestRunDisabledKey), false).toBool();
     gtestRepeat = s->value(root + QLatin1String(gtestRepeatKey), false).toBool();
     gtestShuffle = s->value(root + QLatin1String(gtestShuffleKey), false).toBool();
     gtestIterations = s->value(root + QLatin1String(gtestIterationsKey), 1).toInt();
     gtestSeed = s->value(root + QLatin1String(gtestSeedKey), 0).toInt();
+    gtestBreakOnFailure = s->value(root + QLatin1String(gtestBreakOnFailureKey), true).toBool();
+    gtestThrowOnFailure = s->value(root + QLatin1String(gtestThrowOnFailureKey), false).toBool();
     // try to get settings for registered frameworks
     TestFrameworkManager *frameworkManager = TestFrameworkManager::instance();
     const QList<Core::Id> &registered = frameworkManager->registeredFrameworkIds();
@@ -127,9 +136,12 @@ bool TestSettings::equals(const TestSettings &rhs) const
             && limitResultOutput == rhs.limitResultOutput
             && autoScroll == rhs.autoScroll
             && alwaysParse == rhs.alwaysParse
+            && qtestNoCrashHandler == rhs.qtestNoCrashHandler
             && gtestRunDisabled == rhs.gtestRunDisabled
             && gtestRepeat == rhs.gtestRepeat && gtestIterations == rhs.gtestIterations
             && gtestShuffle == rhs.gtestShuffle && gtestSeed == rhs.gtestSeed
+            && gtestBreakOnFailure == rhs.gtestBreakOnFailure
+            && gtestThrowOnFailure == rhs.gtestThrowOnFailure
             && frameworks == rhs.frameworks;
 }
 

src/plugins/autotest/testsettings.h

@@ -61,9 +61,12 @@ struct TestSettings
     bool limitResultOutput;
     bool autoScroll;
     bool alwaysParse;
+    bool qtestNoCrashHandler;
     bool gtestRunDisabled;
     bool gtestShuffle;
     bool gtestRepeat;
+    bool gtestThrowOnFailure;
+    bool gtestBreakOnFailure;
     QHash<Core::Id, bool> frameworks;
 };
 

src/plugins/autotest/testsettingspage.cpp

@@ -64,11 +64,14 @@ void TestSettingsWidget::setSettings(const TestSettings &settings)
     m_ui.limitResultOutputCB->setChecked(settings.limitResultOutput);
     m_ui.autoScrollCB->setChecked(settings.autoScroll);
     m_ui.alwaysParseCB->setChecked(settings.alwaysParse);
+    m_ui.disableCrashhandlerCB->setChecked(settings.qtestNoCrashHandler);
     m_ui.runDisabledGTestsCB->setChecked(settings.gtestRunDisabled);
     m_ui.repeatGTestsCB->setChecked(settings.gtestRepeat);
     m_ui.shuffleGTestsCB->setChecked(settings.gtestShuffle);
     m_ui.repetitionSpin->setValue(settings.gtestIterations);
     m_ui.seedSpin->setValue(settings.gtestSeed);
+    m_ui.breakOnFailureCB->setChecked(settings.gtestBreakOnFailure);
+    m_ui.throwOnFailureCB->setChecked(settings.gtestThrowOnFailure);
 
     switch (settings.metrics) {
     case MetricsType::Walltime:
@@ -101,11 +104,14 @@ TestSettings TestSettingsWidget::settings() const
     result.limitResultOutput = m_ui.limitResultOutputCB->isChecked();
     result.autoScroll = m_ui.autoScrollCB->isChecked();
     result.alwaysParse = m_ui.alwaysParseCB->isChecked();
+    result.qtestNoCrashHandler = m_ui.disableCrashhandlerCB->isChecked();
     result.gtestRunDisabled = m_ui.runDisabledGTestsCB->isChecked();
     result.gtestRepeat = m_ui.repeatGTestsCB->isChecked();
     result.gtestShuffle = m_ui.shuffleGTestsCB->isChecked();
     result.gtestIterations = m_ui.repetitionSpin->value();
     result.gtestSeed = m_ui.seedSpin->value();
+    result.gtestBreakOnFailure = m_ui.breakOnFailureCB->isChecked();
+    result.gtestThrowOnFailure = m_ui.throwOnFailureCB->isChecked();
 
     if (m_ui.walltimeRB->isChecked())
         result.metrics = MetricsType::Walltime;

src/plugins/autotest/testsettingspage.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>574</width>
+    <width>655</width>
-    <height>404</height>
+    <height>427</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -204,24 +204,18 @@
      <item>
       <layout class="QVBoxLayout" name="verticalLayout_4">
        <item>
-        <widget class="QGroupBox" name="groupBox">
+        <widget class="QGroupBox" name="groupBox_5">
          <property name="title">
-          <string>Benchmark Metrics</string>
+          <string>Qt Test</string>
          </property>
-         <layout class="QVBoxLayout" name="verticalLayout_3">
+         <layout class="QVBoxLayout" name="verticalLayout_8">
           <item>
-           <widget class="QRadioButton" name="walltimeRB">
+           <widget class="QCheckBox" name="disableCrashhandlerCB">
-            <property name="sizePolicy">
-             <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
-            </property>
             <property name="toolTip">
-             <string>Uses walltime metrics for executing benchmarks (default).</string>
+             <string>Enables interrupting tests on assertions.</string>
             </property>
             <property name="text">
-             <string>Walltime</string>
+             <string>Disable crash handler while debugging</string>
             </property>
             <property name="checked">
              <bool>true</bool>
@@ -229,88 +223,119 @@
            </widget>
           </item>
           <item>
-           <widget class="QRadioButton" name="tickcounterRB">
+           <widget class="QGroupBox" name="groupBox">
-            <property name="sizePolicy">
+            <property name="title">
-             <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
+             <string>Benchmark Metrics</string>
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
             </property>
-            <property name="toolTip">
+            <property name="flat">
-             <string>Uses tick counter when executing benchmarks.</string>
+             <bool>true</bool>
-            </property>
-            <property name="text">
-             <string>Tick counter</string>
             </property>
+            <layout class="QVBoxLayout" name="verticalLayout_3">
+             <item>
+              <widget class="QRadioButton" name="walltimeRB">
+               <property name="sizePolicy">
+                <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
+                 <horstretch>0</horstretch>
+                 <verstretch>0</verstretch>
+                </sizepolicy>
+               </property>
+               <property name="toolTip">
+                <string>Uses walltime metrics for executing benchmarks (default).</string>
+               </property>
+               <property name="text">
+                <string>Walltime</string>
+               </property>
+               <property name="checked">
+                <bool>true</bool>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QRadioButton" name="tickcounterRB">
+               <property name="sizePolicy">
+                <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
+                 <horstretch>0</horstretch>
+                 <verstretch>0</verstretch>
+                </sizepolicy>
+               </property>
+               <property name="toolTip">
+                <string>Uses tick counter when executing benchmarks.</string>
+               </property>
+               <property name="text">
+                <string>Tick counter</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QRadioButton" name="eventCounterRB">
+               <property name="sizePolicy">
+                <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
+                 <horstretch>0</horstretch>
+                 <verstretch>0</verstretch>
+                </sizepolicy>
+               </property>
+               <property name="toolTip">
+                <string>Uses event counter when executing benchmarks.</string>
+               </property>
+               <property name="text">
+                <string>Event counter</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QRadioButton" name="callgrindRB">
+               <property name="enabled">
+                <bool>false</bool>
+               </property>
+               <property name="sizePolicy">
+                <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
+                 <horstretch>0</horstretch>
+                 <verstretch>0</verstretch>
+                </sizepolicy>
+               </property>
+               <property name="toolTip">
+                <string>Uses Valgrind Callgrind when executing benchmarks (it must be installed).</string>
+               </property>
+               <property name="text">
+                <string>Callgrind</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <widget class="QRadioButton" name="perfRB">
+               <property name="enabled">
+                <bool>false</bool>
+               </property>
+               <property name="sizePolicy">
+                <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
+                 <horstretch>0</horstretch>
+                 <verstretch>0</verstretch>
+                </sizepolicy>
+               </property>
+               <property name="toolTip">
+                <string>Uses Perf when executing benchmarks (it must be installed).</string>
+               </property>
+               <property name="text">
+                <string>Perf</string>
+               </property>
+              </widget>
+             </item>
+             <item>
+              <spacer name="horizontalSpacer">
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
+               </property>
+               <property name="sizeHint" stdset="0">
+                <size>
+                 <width>40</width>
+                 <height>0</height>
+                </size>
+               </property>
+              </spacer>
+             </item>
+            </layout>
            </widget>
           </item>
-          <item>
-           <widget class="QRadioButton" name="eventCounterRB">
-            <property name="sizePolicy">
-             <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
-            </property>
-            <property name="toolTip">
-             <string>Uses event counter when executing benchmarks.</string>
-            </property>
-            <property name="text">
-             <string>Event counter</string>
-            </property>
-           </widget>
-          </item>
-          <item>
-           <widget class="QRadioButton" name="callgrindRB">
-            <property name="enabled">
-             <bool>false</bool>
-            </property>
-            <property name="sizePolicy">
-             <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
-            </property>
-            <property name="toolTip">
-             <string>Uses Valgrind Callgrind when executing benchmarks (it must be installed).</string>
-            </property>
-            <property name="text">
-             <string>Callgrind</string>
-            </property>
-           </widget>
-          </item>
-          <item>
-           <widget class="QRadioButton" name="perfRB">
-            <property name="enabled">
-             <bool>false</bool>
-            </property>
-            <property name="sizePolicy">
-             <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
-            </property>
-            <property name="toolTip">
-             <string>Uses Perf when executing benchmarks (it must be installed).</string>
-            </property>
-            <property name="text">
-             <string>Perf</string>
-            </property>
-           </widget>
-          </item>
-          <item>
-           <spacer name="horizontalSpacer">
-            <property name="orientation">
-             <enum>Qt::Horizontal</enum>
-            </property>
-            <property name="sizeHint" stdset="0">
-             <size>
-              <width>40</width>
-              <height>0</height>
-             </size>
-            </property>
-           </spacer>
-          </item>
          </layout>
         </widget>
        </item>
@@ -439,6 +464,29 @@
             </item>
            </layout>
           </item>
+          <item>
+           <widget class="QCheckBox" name="throwOnFailureCB">
+            <property name="toolTip">
+             <string>Turn assertion failures into C++ exceptions.</string>
+            </property>
+            <property name="text">
+             <string>Throw on failure</string>
+            </property>
+           </widget>
+          </item>
+          <item>
+           <widget class="QCheckBox" name="breakOnFailureCB">
+            <property name="toolTip">
+             <string>Turn failures into debugger breakpoints.</string>
+            </property>
+            <property name="text">
+             <string>Break on failure while debugging</string>
+            </property>
+            <property name="checked">
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
          </layout>
         </widget>
        </item>