diff --git a/src/libs/utils/mapreduce.h b/src/libs/utils/mapreduce.h index 8c5452f79dc..81f8968d3fe 100644 --- a/src/libs/utils/mapreduce.h +++ b/src/libs/utils/mapreduce.h @@ -57,17 +57,20 @@ protected: public: MapReduceBase(QFutureInterface futureInterface, ForwardIterator begin, ForwardIterator end, MapFunction &&map, State &state, ReduceFunction &&reduce, - MapReduceOption option, int size) + MapReduceOption option, QThreadPool *pool, int size) : m_futureInterface(futureInterface), m_iterator(begin), m_end(end), m_map(std::forward(map)), m_state(state), m_reduce(std::forward(reduce)), + m_threadPool(pool), m_handleProgress(size >= 0), m_size(size), m_option(option) { + if (!m_threadPool) + m_threadPool = new QThreadPool(this); if (m_handleProgress) // progress is handled by us m_futureInterface.setProgressRange(0, MAX_PROGRESS); connect(&m_selfWatcher, &QFutureWatcher::canceled, @@ -88,7 +91,8 @@ protected: bool schedule() { bool didSchedule = false; - while (m_iterator != m_end && m_mapWatcher.size() < QThread::idealThreadCount()) { + while (m_iterator != m_end + && m_mapWatcher.size() < std::max(m_threadPool->maxThreadCount(), 1)) { didSchedule = true; auto watcher = new QFutureWatcher(); connect(watcher, &QFutureWatcher::finished, this, [this, watcher]() { @@ -103,7 +107,7 @@ protected: m_mapWatcher.append(watcher); m_watcherIndex.append(m_currentIndex); ++m_currentIndex; - watcher->setFuture(runAsync(&m_threadPool, std::cref(m_map), + watcher->setFuture(runAsync(m_threadPool, std::cref(m_map), ItemReferenceWrapper(*m_iterator))); ++m_iterator; } @@ -165,7 +169,7 @@ protected: State &m_state; ReduceFunction m_reduce; QEventLoop m_loop; - QThreadPool m_threadPool; // for reusing threads + QThreadPool *m_threadPool; // for reusing threads QList *> m_mapWatcher; QList m_watcherIndex; int m_currentIndex = 0; @@ -182,9 +186,10 @@ class MapReduce : public MapReduceBase; public: MapReduce(QFutureInterface futureInterface, ForwardIterator begin, ForwardIterator end, - MapFunction &&map, State &state, ReduceFunction &&reduce, MapReduceOption option, int size) + MapFunction &&map, State &state, ReduceFunction &&reduce, MapReduceOption option, + QThreadPool *pool, int size) : BaseType(futureInterface, begin, end, std::forward(map), state, - std::forward(reduce), option, size) + std::forward(reduce), option, pool, size) { } @@ -230,9 +235,10 @@ class MapReduce; public: MapReduce(QFutureInterface futureInterface, ForwardIterator begin, ForwardIterator end, - MapFunction &&map, State &state, ReduceFunction &&reduce, MapReduceOption option, int size) + MapFunction &&map, State &state, ReduceFunction &&reduce, MapReduceOption option, + QThreadPool *pool, int size) : BaseType(futureInterface, begin, end, std::forward(map), state, - std::forward(reduce), option, size) + std::forward(reduce), option, pool, size) { } @@ -274,13 +280,13 @@ template &futureInterface, ForwardIterator begin, ForwardIterator end, InitFunction &&init, MapFunction &&map, ReduceFunction &&reduce, CleanUpFunction &&cleanup, - MapReduceOption option, int size) + MapReduceOption option, QThreadPool *pool, int size) { auto state = callWithMaybeFutureInterface (futureInterface, std::forward(init)); MapReduce::type, MapFunction, decltype(state), ReduceResult, ReduceFunction> mr(futureInterface, begin, end, std::forward(map), state, - std::forward(reduce), option, size); + std::forward(reduce), option, pool, size); mr.exec(); callWithMaybeFutureInterface&> (futureInterface, std::forward(cleanup), state); @@ -291,13 +297,13 @@ template &futureInterface, Container &&container, InitFunction &&init, MapFunction &&map, ReduceFunction &&reduce, CleanUpFunction &&cleanup, - MapReduceOption option) + MapReduceOption option, QThreadPool *pool) { blockingIteratorMapReduce(futureInterface, std::begin(container), std::end(container), std::forward(init), std::forward(map), std::forward(reduce), std::forward(cleanup), - option, static_cast(container.size())); + option, pool, static_cast(container.size())); } template &futureInterfa std::reference_wrapper containerWrapper, InitFunction &&init, MapFunction &&map, ReduceFunction &&reduce, CleanUpFunction &&cleanup, - MapReduceOption option) + MapReduceOption option, QThreadPool *pool) { blockingContainerMapReduce(futureInterface, containerWrapper.get(), std::forward(init), std::forward(map), std::forward(reduce), std::forward(cleanup), - option); + option, pool); } template @@ -371,7 +377,7 @@ QFuture mapReduce(ForwardIterator begin, ForwardIterator end, InitFunction &&init, MapFunction &&map, ReduceFunction &&reduce, CleanUpFunction &&cleanup, MapReduceOption option = MapReduceOption::Unordered, - QThread::Priority priority = QThread::InheritPriority, + QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority, int size = -1) { return runAsync(priority, @@ -384,7 +390,7 @@ mapReduce(ForwardIterator begin, ForwardIterator end, InitFunction &&init, MapFu std::decay_t>, begin, end, std::forward(init), std::forward(map), std::forward(reduce), std::forward(cleanup), - option, size); + option, pool, size); } /*! @@ -443,7 +449,7 @@ QFuture mapReduce(Container &&container, InitFunction &&init, MapFunction &&map, ReduceFunction &&reduce, CleanUpFunction &&cleanup, MapReduceOption option = MapReduceOption::Unordered, - QThread::Priority priority = QThread::InheritPriority) + QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority) { return runAsync(priority, Internal::blockingContainerMapReduce< @@ -456,7 +462,7 @@ mapReduce(Container &&container, InitFunction &&init, MapFunction &&map, std::forward(container), std::forward(init), std::forward(map), std::forward(reduce), std::forward(cleanup), - option); + option, pool); } template mapReduce(std::reference_wrapper containerWrapper, InitFunction &&init, MapFunction &&map, ReduceFunction &&reduce, CleanUpFunction &&cleanup, MapReduceOption option = MapReduceOption::Unordered, - QThread::Priority priority = QThread::InheritPriority) + QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority) { return runAsync(priority, Internal::blockingContainerRefMapReduce< @@ -479,7 +485,7 @@ mapReduce(std::reference_wrapper containerWrapper, InitFunction &&ini containerWrapper, std::forward(init), std::forward(map), std::forward(reduce), std::forward(cleanup), - option); + option, pool); } template mapReduce(ForwardIterator begin, ForwardIterator end, MapFunction &&map, State &&initialState, ReduceFunction &&reduce, MapReduceOption option = MapReduceOption::Unordered, - QThread::Priority priority = QThread::InheritPriority, int size = -1) + QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority, + int size = -1) { return mapReduce(begin, end, Internal::StateWrapper(std::forward(initialState)), std::forward(map), Internal::ReduceWrapper(std::forward(reduce)), &Internal::cleanupReportingState, - option, priority, size); + option, pool, priority, size); } template mapReduce(Container &&container, MapFunction &&map, State &&initialState, ReduceFunction &&reduce, MapReduceOption option = MapReduceOption::Unordered, - QThread::Priority priority = QThread::InheritPriority) + QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority) { return mapReduce(std::forward(container), Internal::StateWrapper(std::forward(initialState)), std::forward(map), Internal::ReduceWrapper(std::forward(reduce)), &Internal::cleanupReportingState, - option, priority); + option, pool, priority); } template (map), std::forward(initialState), std::forward(reduce), - option, priority, size).result(); + option, pool, priority, size).result(); } template (container), std::forward(map), std::forward(initialState), std::forward(reduce), - option, priority).result(); + option, pool, priority).result(); } template map(ForwardIterator begin, ForwardIterator end, MapFunction &&map, MapReduceOption option = MapReduceOption::Ordered, - QThread::Priority priority = QThread::InheritPriority, int size = -1) + QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority, + int size = -1) { return mapReduce(begin, end, &Internal::dummyInit, std::forward(map), Internal::DummyReduce(), &Internal::dummyCleanup, - option, priority, size); + option, pool, priority, size); } template ::type> QFuture map(Container &&container, MapFunction &&map, MapReduceOption option = MapReduceOption::Ordered, - QThread::Priority priority = QThread::InheritPriority) + QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority) { return mapReduce(std::forward(container), Internal::dummyInit, std::forward(map), Internal::DummyReduce(), Internal::dummyCleanup, - option, priority); + option, pool, priority); } template class ResultContainer, typename ForwardIterator, typename MapFunction, @@ -578,11 +587,11 @@ Q_REQUIRED_RESULT ResultContainer mapped(ForwardIterator begin, ForwardIterator end, MapFunction &&mapFun, MapReduceOption option = MapReduceOption::Ordered, - QThread::Priority priority = QThread::InheritPriority, int size = -1) + QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority, int size = -1) { return Utils::transform(map(begin, end, std::forward(mapFun), - option, priority, size).results(), + option, pool, priority, size).results(), [](const MapResult &r) { return r; }); } @@ -592,11 +601,11 @@ Q_REQUIRED_RESULT ResultContainer mapped(Container &&container, MapFunction &&mapFun, MapReduceOption option = MapReduceOption::Ordered, - QThread::Priority priority = QThread::InheritPriority) + QThreadPool *pool = nullptr, QThread::Priority priority = QThread::InheritPriority) { return Utils::transform(map(container, std::forward(mapFun), - option, priority).results(), + option, pool, priority).results(), [](const MapResult &r) { return r; }); } diff --git a/src/plugins/autotest/testcodeparser.cpp b/src/plugins/autotest/testcodeparser.cpp index 001ea5d48b0..8cfcd0b133f 100644 --- a/src/plugins/autotest/testcodeparser.cpp +++ b/src/plugins/autotest/testcodeparser.cpp @@ -59,7 +59,8 @@ using namespace ProjectExplorer; TestCodeParser::TestCodeParser(TestTreeModel *parent) : QObject(parent), - m_model(parent) + m_model(parent), + m_threadPool(new QThreadPool(this)) { // connect to ProgressManager to postpone test parsing when CppModelManager is parsing auto progressManager = qobject_cast(Core::ProgressManager::instance()); @@ -78,6 +79,7 @@ TestCodeParser::TestCodeParser(TestTreeModel *parent) connect(this, &TestCodeParser::parsingFinished, this, &TestCodeParser::releaseParserInternals); m_reparseTimer.setSingleShot(true); connect(&m_reparseTimer, &QTimer::timeout, this, &TestCodeParser::parsePostponedFiles); + m_threadPool->setMaxThreadCount(std::max(QThread::idealThreadCount()/4, 1)); } TestCodeParser::~TestCodeParser() @@ -404,6 +406,7 @@ void TestCodeParser::scanForTests(const QStringList &fileList, ITestParser *pars parseFileForTests(codeParsers, fi, file); }, Utils::MapReduceOption::Unordered, + m_threadPool, QThread::LowestPriority); m_futureWatcher.setFuture(future); if (list.size() > 5) { diff --git a/src/plugins/autotest/testcodeparser.h b/src/plugins/autotest/testcodeparser.h index 6286600c335..e06563d18a6 100644 --- a/src/plugins/autotest/testcodeparser.h +++ b/src/plugins/autotest/testcodeparser.h @@ -38,6 +38,10 @@ namespace Core { class Id; } +QT_BEGIN_NAMESPACE +class QThreadPool; +QT_END_NAMESPACE + namespace Autotest { namespace Internal { @@ -106,6 +110,7 @@ private: QVector m_testCodeParsers; // ptrs are still owned by TestFrameworkManager QTimer m_reparseTimer; ITestParser *m_updateParser = nullptr; + QThreadPool *m_threadPool = nullptr; }; } // namespace Internal diff --git a/tests/auto/mapreduce/tst_mapreduce.cpp b/tests/auto/mapreduce/tst_mapreduce.cpp index 413d8753602..5fb8cb44b00 100644 --- a/tests/auto/mapreduce/tst_mapreduce.cpp +++ b/tests/auto/mapreduce/tst_mapreduce.cpp @@ -26,6 +26,7 @@ #include #include +#include #include #if !defined(Q_CC_MSVC) || _MSC_VER >= 1900 // MSVC2015 @@ -58,6 +59,7 @@ static void returnxxThroughFutureInterface(QFutureInterface &fi, int x) void tst_MapReduce::mapReduce() { + QThreadPool pool; const auto initWithFutureInterface = [](QFutureInterface &fi) -> double { fi.reportResult(0.); return 0.; @@ -102,6 +104,16 @@ void tst_MapReduce::mapReduce() Utils::sort(results); // mapping order is undefined QCOMPARE(results, QList({0., 1., 4., 9., 16., 25., 27.5})); } + { + // reduce with threadpool + QList results = Utils::mapReduce(QList({1, 2, 3, 4, 5}), + initWithFutureInterface, returnxx, + reduceWithReturn, cleanupWithFutureInterface, + Utils::MapReduceOption::Unordered, &pool) + .results(); + Utils::sort(results); // mapping order is undefined + QCOMPARE(results, QList({0., 1., 4., 9., 16., 25., 27.5})); + } { // lvalue ref container QList container({1, 2, 3, 4, 5}); @@ -121,6 +133,15 @@ void tst_MapReduce::mapReduce() Utils::MapReduceOption::Ordered).results(), QList({0., 1., 4., 9., 16., 25., 27.5})); } + { + // std::cref with threadpool + QList container({1, 2, 3, 4, 5}); + QCOMPARE(Utils::mapReduce(std::cref(container), + initWithFutureInterface, returnxx, + reduceWithReturn, cleanupWithFutureInterface, + Utils::MapReduceOption::Ordered, &pool).results(), + QList({0., 1., 4., 9., 16., 25., 27.5})); + } { // std::ref QList container({1, 2, 3, 4, 5}); @@ -151,6 +172,17 @@ void tst_MapReduce::mapReduce() Utils::MapReduceOption::Ordered).result(), 1.5); } + { + // simplified map reduce without init and cleanup with threadpool + QCOMPARE(Utils::mapReduce(QList({QLatin1String("blubb"), QLatin1String("foo"), QLatin1String("blah")}), + [](const QString &val) { return val.size(); }, + 90., + [](double &state, int val) { + state /= double(val); + }, + Utils::MapReduceOption::Ordered, &pool).result(), + 1.5); + } { // simplified map reduce // std::cref @@ -161,7 +193,16 @@ void tst_MapReduce::mapReduce() } { // simplified map reduce - // std::cref + // std::cref with threadpool + QList container({1, 2, 3}); + QCOMPARE(Utils::mapReduce(std::cref(container), [](int val) { return 2*val; }, 10, + [](int &state, int val) { state += val; }, + Utils::MapReduceOption::Unordered, &pool).result(), + 22); + } + { + // simplified map reduce + // std::ref QList container({1, 2, 3}); QCOMPARE(Utils::mapReduce(std::ref(container), [](int &val) { return 2*val; }, 10, [](int &state, int val) { state += val; }).result(), @@ -173,6 +214,14 @@ void tst_MapReduce::mapReduce() [](int &state, int val) { state += val; }), 22); } + { + // blocking mapReduce = mappedReduced + // with threadpool + QCOMPARE(Utils::mappedReduced(QList({1, 2, 3}), [](int &val) { return 2*val; }, 10, + [](int &state, int val) { state += val; }, + Utils::MapReduceOption::Unordered, &pool), + 22); + } } void tst_MapReduce::mapReduceRvalueContainer() @@ -214,7 +263,8 @@ void tst_MapReduce::map() QCOMPARE(container, QList({4, 10, 2})); Utils::map(container.begin(), container.end(), [](int &x) { x *= 2; }, - Utils::MapReduceOption::Unordered, QThread::InheritPriority, 3).waitForFinished(); + Utils::MapReduceOption::Unordered, + nullptr, QThread::InheritPriority, 3).waitForFinished(); QCOMPARE(container, QList({8, 20, 4})); }