Reduce CPU load of test parsing

Use a thread pool with reduced max thread count.

Task-number: QTCREATORBUG-18185
Change-Id: I18bd3de82365edaf21d8dcf9c89035da1ac74756
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

src/libs/utils/mapreduce.h

@@ -57,17 +57,20 @@ protected:
 public:
     MapReduceBase(QFutureInterface<ReduceResult> futureInterface, ForwardIterator begin, ForwardIterator end,
                   MapFunction &&map, State &state, ReduceFunction &&reduce,
-                  MapReduceOption option, int size)
+                  MapReduceOption option, QThreadPool *pool, int size)
         : m_futureInterface(futureInterface),
           m_iterator(begin),
           m_end(end),
           m_map(std::forward<MapFunction>(map)),
           m_state(state),
           m_reduce(std::forward<ReduceFunction>(reduce)),
+          m_threadPool(pool),
           m_handleProgress(size >= 0),
           m_size(size),
           m_option(option)
     {
+        if (!m_threadPool)
+            m_threadPool = new QThreadPool(this);
         if (m_handleProgress) // progress is handled by us
             m_futureInterface.setProgressRange(0, MAX_PROGRESS);
         connect(&m_selfWatcher, &QFutureWatcher<void>::canceled,
@@ -88,7 +91,8 @@ protected:
     bool schedule()
     {
         bool didSchedule = false;
-        while (m_iterator != m_end && m_mapWatcher.size() < QThread::idealThreadCount()) {
+        while (m_iterator != m_end
+               && m_mapWatcher.size() < std::max(m_threadPool->maxThreadCount(), 1)) {
             didSchedule = true;
             auto watcher = new QFutureWatcher<MapResult>();
             connect(watcher, &QFutureWatcher<MapResult>::finished, this, [this, watcher]() {
@@ -103,7 +107,7 @@ protected:
             m_mapWatcher.append(watcher);
             m_watcherIndex.append(m_currentIndex);
             ++m_currentIndex;
-            watcher->setFuture(runAsync(&m_threadPool, std::cref(m_map),
+            watcher->setFuture(runAsync(m_threadPool, std::cref(m_map),
                                         ItemReferenceWrapper(*m_iterator)));
             ++m_iterator;
         }
@@ -165,7 +169,7 @@ protected:
     State &m_state;
     ReduceFunction m_reduce;
     QEventLoop m_loop;
-    QThreadPool m_threadPool; // for reusing threads
+    QThreadPool *m_threadPool; // for reusing threads
     QList<QFutureWatcher<MapResult> *> m_mapWatcher;
     QList<int> m_watcherIndex;
     int m_currentIndex = 0;
@@ -182,9 +186,10 @@ class MapReduce : public MapReduceBase<ForwardIterator, MapResult, MapFunction,
     using BaseType = MapReduceBase<ForwardIterator, MapResult, MapFunction, State, ReduceResult, ReduceFunction>;
 public:
     MapReduce(QFutureInterface<ReduceResult> futureInterface, ForwardIterator begin, ForwardIterator end,
-              MapFunction &&map, State &state, ReduceFunction &&reduce, MapReduceOption option, int size)
+              MapFunction &&map, State &state, ReduceFunction &&reduce, MapReduceOption option,
+              QThreadPool *pool, int size)
         : BaseType(futureInterface, begin, end, std::forward<MapFunction>(map), state,
-                   std::forward<ReduceFunction>(reduce), option, size)
+                   std::forward<ReduceFunction>(reduce), option, pool, size)
     {
     }
 
@@ -230,9 +235,10 @@ class MapReduce<ForwardIterator, void, MapFunction, State, ReduceResult, ReduceF
     using BaseType = MapReduceBase<ForwardIterator, void, MapFunction, State, ReduceResult, ReduceFunction>;
 public:
     MapReduce(QFutureInterface<ReduceResult> futureInterface, ForwardIterator begin, ForwardIterator end,
-              MapFunction &&map, State &state, ReduceFunction &&reduce, MapReduceOption option, int size)
+              MapFunction &&map, State &state, ReduceFunction &&reduce, MapReduceOption option,
+              QThreadPool *pool, int size)
         : BaseType(futureInterface, begin, end, std::forward<MapFunction>(map), state,
-                   std::forward<ReduceFunction>(reduce), option, size)
+                   std::forward<ReduceFunction>(reduce), option, pool, size)
     {
     }
 
@@ -274,13 +280,13 @@ template <typename ForwardIterator, typename InitFunction, typename MapFunction,
 void blockingIteratorMapReduce(QFutureInterface<ReduceResult> &futureInterface, ForwardIterator begin, ForwardIterator end,
                                InitFunction &&init, MapFunction &&map,
                                ReduceFunction &&reduce, CleanUpFunction &&cleanup,
-                               MapReduceOption option, int size)
+                               MapReduceOption option, QThreadPool *pool, int size)
 {
     auto state = callWithMaybeFutureInterface<ReduceResult, InitFunction>
             (futureInterface, std::forward<InitFunction>(init));
     MapReduce<ForwardIterator, typename Internal::resultType<MapFunction>::type, MapFunction, decltype(state), ReduceResult, ReduceFunction>
             mr(futureInterface, begin, end, std::forward<MapFunction>(map), state,
-               std::forward<ReduceFunction>(reduce), option, size);
+               std::forward<ReduceFunction>(reduce), option, pool, size);
     mr.exec();
     callWithMaybeFutureInterface<ReduceResult, CleanUpFunction, std::remove_reference_t<decltype(state)>&>
             (futureInterface, std::forward<CleanUpFunction>(cleanup), state);
@@ -291,13 +297,13 @@ template <typename Container, typename InitFunction, typename MapFunction, typen
 void blockingContainerMapReduce(QFutureInterface<ReduceResult> &futureInterface, Container &&container,
                                 InitFunction &&init, MapFunction &&map,
                                 ReduceFunction &&reduce, CleanUpFunction &&cleanup,
-                                MapReduceOption option)
+                                MapReduceOption option, QThreadPool *pool)
 {
     blockingIteratorMapReduce(futureInterface, std::begin(container), std::end(container),
                               std::forward<InitFunction>(init), std::forward<MapFunction>(map),
                               std::forward<ReduceFunction>(reduce),
                               std::forward<CleanUpFunction>(cleanup),
-                              option, static_cast<int>(container.size()));
+                              option, pool, static_cast<int>(container.size()));
 }
 
 template <typename Container, typename InitFunction, typename MapFunction, typename ReduceResult,
@@ -306,13 +312,13 @@ void blockingContainerRefMapReduce(QFutureInterface<ReduceResult> &futureInterfa
                                     std::reference_wrapper<Container> containerWrapper,
                                     InitFunction &&init, MapFunction &&map,
                                     ReduceFunction &&reduce, CleanUpFunction &&cleanup,
-                                    MapReduceOption option)
+                                    MapReduceOption option, QThreadPool *pool)
 {
     blockingContainerMapReduce(futureInterface, containerWrapper.get(),
                                std::forward<InitFunction>(init), std::forward<MapFunction>(map),
                                std::forward<ReduceFunction>(reduce),
                                std::forward<CleanUpFunction>(cleanup),
-                               option);
+                               option, pool);
 }
 
 template <typename ReduceResult>
@@ -371,7 +377,7 @@ QFuture<ReduceResult>
 mapReduce(ForwardIterator begin, ForwardIterator end, InitFunction &&init, MapFunction &&map,
           ReduceFunction &&reduce, CleanUpFunction &&cleanup,
           MapReduceOption option = MapReduceOption::Unordered,
-          QThread::Priority priority = QThread::InheritPriority,
+          QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority,
           int size = -1)
 {
     return runAsync(priority,
@@ -384,7 +390,7 @@ mapReduce(ForwardIterator begin, ForwardIterator end, InitFunction &&init, MapFu
                         std::decay_t<CleanUpFunction>>,
                     begin, end, std::forward<InitFunction>(init), std::forward<MapFunction>(map),
                     std::forward<ReduceFunction>(reduce), std::forward<CleanUpFunction>(cleanup),
-                    option, size);
+                    option, pool, size);
 }
 
 /*!
@@ -443,7 +449,7 @@ QFuture<ReduceResult>
 mapReduce(Container &&container, InitFunction &&init, MapFunction &&map,
           ReduceFunction &&reduce, CleanUpFunction &&cleanup,
           MapReduceOption option = MapReduceOption::Unordered,
-          QThread::Priority priority = QThread::InheritPriority)
+          QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority)
 {
     return runAsync(priority,
                     Internal::blockingContainerMapReduce<
@@ -456,7 +462,7 @@ mapReduce(Container &&container, InitFunction &&init, MapFunction &&map,
                     std::forward<Container>(container),
                     std::forward<InitFunction>(init), std::forward<MapFunction>(map),
                     std::forward<ReduceFunction>(reduce), std::forward<CleanUpFunction>(cleanup),
-                    option);
+                    option, pool);
 }
 
 template <typename Container, typename InitFunction, typename MapFunction,
@@ -466,7 +472,7 @@ QFuture<ReduceResult>
 mapReduce(std::reference_wrapper<Container> containerWrapper, InitFunction &&init, MapFunction &&map,
           ReduceFunction &&reduce, CleanUpFunction &&cleanup,
           MapReduceOption option = MapReduceOption::Unordered,
-          QThread::Priority priority = QThread::InheritPriority)
+          QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority)
 {
     return runAsync(priority,
                     Internal::blockingContainerRefMapReduce<
@@ -479,7 +485,7 @@ mapReduce(std::reference_wrapper<Container> containerWrapper, InitFunction &&ini
                     containerWrapper,
                     std::forward<InitFunction>(init), std::forward<MapFunction>(map),
                     std::forward<ReduceFunction>(reduce), std::forward<CleanUpFunction>(cleanup),
-                    option);
+                    option, pool);
 }
 
 template <typename ForwardIterator, typename MapFunction, typename State, typename ReduceFunction,
@@ -488,14 +494,15 @@ template <typename ForwardIterator, typename MapFunction, typename State, typena
 QFuture<StateResult>
 mapReduce(ForwardIterator begin, ForwardIterator end, MapFunction &&map, State &&initialState,
           ReduceFunction &&reduce, MapReduceOption option = MapReduceOption::Unordered,
-          QThread::Priority priority = QThread::InheritPriority, int size = -1)
+          QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority,
+          int size = -1)
 {
     return mapReduce(begin, end,
                      Internal::StateWrapper<State>(std::forward<State>(initialState)),
                      std::forward<MapFunction>(map),
                      Internal::ReduceWrapper<StateResult, MapResult, ReduceFunction>(std::forward<ReduceFunction>(reduce)),
                      &Internal::cleanupReportingState<StateResult>,
-                     option, priority, size);
+                     option, pool, priority, size);
 }
 
 template <typename Container, typename MapFunction, typename State, typename ReduceFunction,
@@ -504,14 +511,14 @@ template <typename Container, typename MapFunction, typename State, typename Red
 QFuture<StateResult>
 mapReduce(Container &&container, MapFunction &&map, State &&initialState, ReduceFunction &&reduce,
           MapReduceOption option = MapReduceOption::Unordered,
-          QThread::Priority priority = QThread::InheritPriority)
+          QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority)
 {
     return mapReduce(std::forward<Container>(container),
                      Internal::StateWrapper<State>(std::forward<State>(initialState)),
                      std::forward<MapFunction>(map),
                      Internal::ReduceWrapper<StateResult, MapResult, ReduceFunction>(std::forward<ReduceFunction>(reduce)),
                      &Internal::cleanupReportingState<StateResult>,
-                     option, priority);
+                     option, pool, priority);
 }
 
 template <typename ForwardIterator, typename MapFunction, typename State, typename ReduceFunction,
@@ -521,12 +528,13 @@ Q_REQUIRED_RESULT
 StateResult
 mappedReduced(ForwardIterator begin, ForwardIterator end, MapFunction &&map, State &&initialState,
               ReduceFunction &&reduce, MapReduceOption option = MapReduceOption::Unordered,
-              QThread::Priority priority = QThread::InheritPriority, int size = -1)
+              QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority,
+              int size = -1)
 {
     return mapReduce(begin, end,
                      std::forward<MapFunction>(map), std::forward<State>(initialState),
                      std::forward<ReduceFunction>(reduce),
-                     option, priority, size).result();
+                     option, pool, priority, size).result();
 }
 
 template <typename Container, typename MapFunction, typename State, typename ReduceFunction,
@@ -536,11 +544,11 @@ Q_REQUIRED_RESULT
 StateResult
 mappedReduced(Container &&container, MapFunction &&map, State &&initialState, ReduceFunction &&reduce,
               MapReduceOption option = MapReduceOption::Unordered,
-              QThread::Priority priority = QThread::InheritPriority)
+              QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority)
 {
     return mapReduce(std::forward<Container>(container), std::forward<MapFunction>(map),
                      std::forward<State>(initialState), std::forward<ReduceFunction>(reduce),
-                     option, priority).result();
+                     option, pool, priority).result();
 }
 
 template <typename ForwardIterator, typename MapFunction,
@@ -548,28 +556,29 @@ template <typename ForwardIterator, typename MapFunction,
 QFuture<MapResult>
 map(ForwardIterator begin, ForwardIterator end, MapFunction &&map,
     MapReduceOption option = MapReduceOption::Ordered,
-    QThread::Priority priority = QThread::InheritPriority, int size = -1)
+    QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority,
+    int size = -1)
 {
     return mapReduce(begin, end,
                      &Internal::dummyInit<MapResult>,
                      std::forward<MapFunction>(map),
                      Internal::DummyReduce<MapResult>(),
                      &Internal::dummyCleanup<MapResult>,
-                     option, priority, size);
+                     option, pool, priority, size);
 }
 
 template <typename Container, typename MapFunction,
           typename MapResult = typename Internal::resultType<MapFunction>::type>
 QFuture<MapResult>
 map(Container &&container, MapFunction &&map, MapReduceOption option = MapReduceOption::Ordered,
-    QThread::Priority priority = QThread::InheritPriority)
+    QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority)
 {
     return mapReduce(std::forward<Container>(container),
                      Internal::dummyInit<MapResult>,
                      std::forward<MapFunction>(map),
                      Internal::DummyReduce<MapResult>(),
                      Internal::dummyCleanup<MapResult>,
-                     option, priority);
+                     option, pool, priority);
 }
 
 template <template<typename> class ResultContainer, typename ForwardIterator, typename MapFunction,
@@ -578,11 +587,11 @@ Q_REQUIRED_RESULT
 ResultContainer<MapResult>
 mapped(ForwardIterator begin, ForwardIterator end, MapFunction &&mapFun,
     MapReduceOption option = MapReduceOption::Ordered,
-       QThread::Priority priority = QThread::InheritPriority, int size = -1)
+       QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority, int size = -1)
 {
     return Utils::transform<ResultContainer>(map(begin, end,
                                                  std::forward<MapFunction>(mapFun),
-                                                 option, priority, size).results(),
+                                                 option, pool, priority, size).results(),
                                              [](const MapResult &r) { return r; });
 }
 
@@ -592,11 +601,11 @@ Q_REQUIRED_RESULT
 ResultContainer<MapResult>
 mapped(Container &&container, MapFunction &&mapFun,
        MapReduceOption option = MapReduceOption::Ordered,
-       QThread::Priority priority = QThread::InheritPriority)
+       QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority)
 {
     return Utils::transform<ResultContainer>(map(container,
                                                  std::forward<MapFunction>(mapFun),
-                                                 option, priority).results(),
+                                                 option, pool, priority).results(),
                                              [](const MapResult &r) { return r; });
 }
 

src/plugins/autotest/testcodeparser.cpp

@@ -59,7 +59,8 @@ using namespace ProjectExplorer;
 
 TestCodeParser::TestCodeParser(TestTreeModel *parent)
     : QObject(parent),
-      m_model(parent)
+      m_model(parent),
+      m_threadPool(new QThreadPool(this))
 {
     // connect to ProgressManager to postpone test parsing when CppModelManager is parsing
     auto progressManager = qobject_cast<Core::ProgressManager *>(Core::ProgressManager::instance());
@@ -78,6 +79,7 @@ TestCodeParser::TestCodeParser(TestTreeModel *parent)
     connect(this, &TestCodeParser::parsingFinished, this, &TestCodeParser::releaseParserInternals);
     m_reparseTimer.setSingleShot(true);
     connect(&m_reparseTimer, &QTimer::timeout, this, &TestCodeParser::parsePostponedFiles);
+    m_threadPool->setMaxThreadCount(std::max(QThread::idealThreadCount()/4, 1));
 }
 
 TestCodeParser::~TestCodeParser()
@@ -404,6 +406,7 @@ void TestCodeParser::scanForTests(const QStringList &fileList, ITestParser *pars
             parseFileForTests(codeParsers, fi, file);
         },
         Utils::MapReduceOption::Unordered,
+        m_threadPool,
         QThread::LowestPriority);
     m_futureWatcher.setFuture(future);
     if (list.size() > 5) {

src/plugins/autotest/testcodeparser.h

@@ -38,6 +38,10 @@ namespace Core {
 class Id;
 }
 
+QT_BEGIN_NAMESPACE
+class QThreadPool;
+QT_END_NAMESPACE
+
 namespace Autotest {
 namespace Internal {
 
@@ -106,6 +110,7 @@ private:
     QVector<ITestParser *> m_testCodeParsers; // ptrs are still owned by TestFrameworkManager
     QTimer m_reparseTimer;
     ITestParser *m_updateParser = nullptr;
+    QThreadPool *m_threadPool = nullptr;
 };
 
 } // namespace Internal

tests/auto/mapreduce/tst_mapreduce.cpp

@@ -26,6 +26,7 @@
 #include <utils/algorithm.h>
 #include <utils/mapreduce.h>
 
+#include <QThreadPool>
 #include <QtTest>
 
 #if !defined(Q_CC_MSVC) || _MSC_VER >= 1900 // MSVC2015
@@ -58,6 +59,7 @@ static void returnxxThroughFutureInterface(QFutureInterface<int> &fi, int x)
 
 void tst_MapReduce::mapReduce()
 {
+    QThreadPool pool;
     const auto initWithFutureInterface = [](QFutureInterface<double> &fi) -> double {
         fi.reportResult(0.);
         return 0.;
@@ -102,6 +104,16 @@ void tst_MapReduce::mapReduce()
         Utils::sort(results); // mapping order is undefined
         QCOMPARE(results, QList<double>({0., 1., 4., 9., 16., 25., 27.5}));
     }
+    {
+        // reduce with threadpool
+        QList<double> results = Utils::mapReduce(QList<int>({1, 2, 3, 4, 5}),
+                                                 initWithFutureInterface, returnxx,
+                                                 reduceWithReturn, cleanupWithFutureInterface,
+                                                 Utils::MapReduceOption::Unordered, &pool)
+                .results();
+        Utils::sort(results); // mapping order is undefined
+        QCOMPARE(results, QList<double>({0., 1., 4., 9., 16., 25., 27.5}));
+    }
     {
         // lvalue ref container
         QList<int> container({1, 2, 3, 4, 5});
@@ -121,6 +133,15 @@ void tst_MapReduce::mapReduce()
                                   Utils::MapReduceOption::Ordered).results(),
                  QList<double>({0., 1., 4., 9., 16., 25., 27.5}));
     }
+    {
+        // std::cref with threadpool
+        QList<int> container({1, 2, 3, 4, 5});
+        QCOMPARE(Utils::mapReduce(std::cref(container),
+                                  initWithFutureInterface, returnxx,
+                                  reduceWithReturn, cleanupWithFutureInterface,
+                                  Utils::MapReduceOption::Ordered, &pool).results(),
+                 QList<double>({0., 1., 4., 9., 16., 25., 27.5}));
+    }
     {
         // std::ref
         QList<int> container({1, 2, 3, 4, 5});
@@ -151,6 +172,17 @@ void tst_MapReduce::mapReduce()
                                   Utils::MapReduceOption::Ordered).result(),
                  1.5);
     }
+    {
+        // simplified map reduce without init and cleanup with threadpool
+        QCOMPARE(Utils::mapReduce(QList<QString>({QLatin1String("blubb"), QLatin1String("foo"), QLatin1String("blah")}),
+                                  [](const QString &val) { return val.size(); },
+                                  90.,
+                                  [](double &state, int val) {
+                                      state /= double(val);
+                                  },
+                                  Utils::MapReduceOption::Ordered, &pool).result(),
+                 1.5);
+    }
     {
         // simplified map reduce
         // std::cref
@@ -161,7 +193,16 @@ void tst_MapReduce::mapReduce()
     }
     {
         // simplified map reduce
-        // std::cref
+        // std::cref with threadpool
+        QList<int> container({1, 2, 3});
+        QCOMPARE(Utils::mapReduce(std::cref(container), [](int val) { return 2*val; }, 10,
+                                  [](int &state, int val) { state += val; },
+                                  Utils::MapReduceOption::Unordered, &pool).result(),
+                 22);
+    }
+    {
+        // simplified map reduce
+        // std::ref
         QList<int> container({1, 2, 3});
         QCOMPARE(Utils::mapReduce(std::ref(container), [](int &val) { return 2*val; }, 10,
                                   [](int &state, int val) { state += val; }).result(),
@@ -173,6 +214,14 @@ void tst_MapReduce::mapReduce()
                                       [](int &state, int val) { state += val; }),
                  22);
     }
+    {
+        // blocking mapReduce = mappedReduced
+        // with threadpool
+        QCOMPARE(Utils::mappedReduced(QList<int>({1, 2, 3}), [](int &val) { return 2*val; }, 10,
+                                      [](int &state, int val) { state += val; },
+                                      Utils::MapReduceOption::Unordered, &pool),
+                 22);
+    }
 }
 
 void tst_MapReduce::mapReduceRvalueContainer()
@@ -214,7 +263,8 @@ void tst_MapReduce::map()
         QCOMPARE(container, QList<int>({4, 10, 2}));
 
         Utils::map(container.begin(), container.end(), [](int &x) { x *= 2; },
-            Utils::MapReduceOption::Unordered, QThread::InheritPriority, 3).waitForFinished();
+            Utils::MapReduceOption::Unordered,
+            nullptr, QThread::InheritPriority, 3).waitForFinished();
         QCOMPARE(container, QList<int>({8, 20, 4}));
     }