AutoTest: Fix handling of test runs when project changes

If the startup project or the current active target changes
while tests are running strange things can happen as we rely
on these both being unchanged from triggering the run.
Explicitly check for these being unchanged and cancel the
test run if necessary.

Change-Id: I506c7b1c0ca4b6ea31559556f6141fe9276d0ad0
Reviewed-by: Leena Miettinen <riitta-leena.miettinen@qt.io>
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/autotest/testconfiguration.cpp

@@ -75,12 +75,14 @@ void TestConfiguration::completeTestInformation(ProjectExplorer::RunConfiguratio
                                                 TestRunMode runMode)
 {
     QTC_ASSERT(rc, return);
+    QTC_ASSERT(m_project, return);
+
     if (hasExecutable()) {
         qCDebug(LOG) << "Executable has been set already - not completing configuration again.";
         return;
     }
     Project *project = SessionManager::startupProject();
-    if (!project)
+    if (!project || project != m_project)
         return;
 
     Target *target = project->activeTarget();
@@ -94,7 +96,6 @@ void TestConfiguration::completeTestInformation(ProjectExplorer::RunConfiguratio
 
     m_runnable = rc->runnable();
     m_displayName = rc->displayName();
-    m_project = rc->project();
 
     const QString buildKey = rc->buildKey();
     BuildTargetInfo targetInfo = target->applicationTargets().buildTargetInfo(buildKey);
@@ -116,6 +117,7 @@ void TestConfiguration::completeTestInformation(TestRunMode runMode)
 {
     QTC_ASSERT(!m_projectFile.isEmpty(), return);
     QTC_ASSERT(!m_buildTargets.isEmpty(), return);
+    QTC_ASSERT(m_project, return);
 
     if (m_origRunConfig) {
         qCDebug(LOG) << "Using run configuration specified by user or found by first call";
@@ -129,8 +131,10 @@ void TestConfiguration::completeTestInformation(TestRunMode runMode)
         qCDebug(LOG) << "Failed to complete - using 'normal' way.";
     }
     Project *project = SessionManager::startupProject();
-    if (!project)
+    if (!project || project != m_project) {
+        m_project = nullptr;
         return;
+    }
 
     Target *target = project->activeTarget();
     if (!target)
@@ -204,7 +208,6 @@ void TestConfiguration::completeTestInformation(TestRunMode runMode)
             m_runnable = runnable;
             m_runnable.executable = currentExecutable;
             m_displayName = runConfig->displayName();
-            m_project = project;
             if (runMode == TestRunMode::Debug || runMode == TestRunMode::DebugWithoutDeploy)
                 m_runConfig = new TestRunConfiguration(runConfig->target(), this);
             break;
@@ -224,7 +227,6 @@ void TestConfiguration::completeTestInformation(TestRunMode runMode)
             if (isLocal(rc)) { // FIXME for now only Desktop support
                 const Runnable runnable = rc->runnable();
                 m_runnable.environment = runnable.environment;
-                m_project = project;
                 m_guessedConfiguration = true;
                 m_guessedFrom = rc->displayName();
                 if (runMode == TestRunMode::Debug)

src/plugins/autotest/testrunner.cpp

@@ -154,6 +154,7 @@ void TestRunner::scheduleNext()
     QTC_ASSERT(!m_selectedTests.isEmpty(), onFinished(); return);
     QTC_ASSERT(!m_currentConfig && !m_currentProcess, resetInternalPointers());
     QTC_ASSERT(m_fakeFutureInterface, onFinished(); return);
+    QTC_ASSERT(!m_canceled, onFinished(); return);
 
     m_currentConfig = m_selectedTests.dequeue();
 
@@ -215,10 +216,16 @@ void TestRunner::scheduleNext()
 
 void TestRunner::cancelCurrent(TestRunner::CancelReason reason)
 {
+    m_canceled = true;
     if (reason == UserCanceled) {
         // when using the stop button we need to report, for progress bar this happens automatically
         if (m_fakeFutureInterface && !m_fakeFutureInterface->isCanceled())
             m_fakeFutureInterface->reportCanceled();
+    } else if (reason == KitChanged) {
+        if (m_fakeFutureInterface)
+            m_fakeFutureInterface->reportCanceled();
+        emit testResultReady(TestResultPtr(new FaultyTestResult(Result::MessageWarn,
+                tr("Current kit has changed. Canceling test run."))));
     }
     if (m_currentProcess && m_currentProcess->state() != QProcess::NotRunning) {
         m_currentProcess->kill();
@@ -279,6 +286,7 @@ void TestRunner::prepareToRunTests(TestRunMode mode)
     }
 
     m_executingTests = true;
+    m_canceled = false;
     emit testRunStarted();
 
     // clear old log and output pane
@@ -301,6 +309,9 @@ void TestRunner::prepareToRunTests(TestRunMode mode)
         return;
     }
 
+    m_targetConnect = connect(project, &ProjectExplorer::Project::activeTargetChanged,
+                              [this]() { cancelCurrent(KitChanged); });
+
     if (!projectExplorerSettings.buildBeforeDeploy || mode == TestRunMode::DebugWithoutDeploy
             || mode == TestRunMode::RunWithoutDeploy) {
         runOrDebugTests();
@@ -386,9 +397,13 @@ int TestRunner::precheckTestConfigurations()
 void TestRunner::runTests()
 {
     QList<TestConfiguration *> toBeRemoved;
+    bool projectChanged = false;
     for (TestConfiguration *config : m_selectedTests) {
         config->completeTestInformation(TestRunMode::Run);
-        if (!config->hasExecutable()) {
+        if (!config->project()) {
+            projectChanged = true;
+            toBeRemoved.append(config);
+        } else if (!config->hasExecutable()) {
             if (auto rc = getRunConfiguration(firstTestCaseTarget(config)))
                 config->setOriginalRunConfiguration(rc);
             else
@@ -400,8 +415,10 @@ void TestRunner::runTests()
     qDeleteAll(toBeRemoved);
     toBeRemoved.clear();
     if (m_selectedTests.isEmpty()) {
-        emit testResultReady(TestResultPtr(new FaultyTestResult(Result::MessageWarn,
+        QString mssg = projectChanged ? tr("Startup project has changed. Canceling test run.")
-                tr("No test cases left for execution. Canceling test run."))));
+                                      : tr("No test cases left for execution. Canceling test run.");
+
+        emit testResultReady(TestResultPtr(new FaultyTestResult(Result::MessageWarn, mssg)));
         onFinished();
         return;
     }
@@ -454,6 +471,12 @@ void TestRunner::debugTests()
 
     TestConfiguration *config = m_selectedTests.first();
     config->completeTestInformation(TestRunMode::Debug);
+    if (!config->project()) {
+        emit testResultReady(TestResultPtr(new FaultyTestResult(Result::MessageWarn,
+            TestRunner::tr("Startup project has changed. Canceling test run."))));
+        onFinished();
+        return;
+    }
     if (!config->hasExecutable()) {
         if (auto *rc = getRunConfiguration(firstTestCaseTarget(config)))
             config->completeTestInformation(rc, TestRunMode::Debug);
@@ -573,7 +596,10 @@ void TestRunner::buildFinished(bool success)
                this, &TestRunner::buildFinished);
 
     if (success) {
+        if (!m_canceled)
             runOrDebugTests();
+        else if (m_executingTests)
+            onFinished();
     } else {
         emit testResultReady(TestResultPtr(new FaultyTestResult(Result::MessageFatal,
                                                   tr("Build failed. Canceling test run."))));
@@ -587,6 +613,7 @@ void TestRunner::onFinished()
     qDeleteAll(m_selectedTests);
     m_selectedTests.clear();
 
+    disconnect(m_targetConnect);
     m_fakeFutureInterface = nullptr;
     m_executingTests = false;
     emit testRunFinished();

src/plugins/autotest/testrunner.h

@@ -54,7 +54,7 @@ class AUTOTESTSHARED_EXPORT TestRunner : public QObject
     Q_OBJECT
 
 public:
-    enum CancelReason { UserCanceled, Timeout };
+    enum CancelReason { UserCanceled, Timeout, KitChanged };
 
     static TestRunner* instance();
     ~TestRunner();
@@ -91,6 +91,7 @@ private:
     QFutureInterface<TestResultPtr> *m_fakeFutureInterface = nullptr;
     QQueue<TestConfiguration *> m_selectedTests;
     bool m_executingTests = false;
+    bool m_canceled = false;
     TestConfiguration *m_currentConfig = nullptr;
     QProcess *m_currentProcess = nullptr;
     TestOutputReader *m_currentOutputReader = nullptr;
@@ -98,6 +99,8 @@ private:
 
     // temporarily used if building before running is necessary
     QMetaObject::Connection m_buildConnect;
+    // temporarily used for handling of switching the current target
+    QMetaObject::Connection m_targetConnect;
 };
 
 class RunConfigurationSelectionDialog : public QDialog