ClangRefactoring: Add SymbolIndexerTaskScheduler

The scheduler is managing the asynchronous tasks by using the symbols
collector manager. Every symbols collector can be used by only one thread,
so the we have to pass the symbols collector around by the future interface
to make the available again after a task is finished.

Change-Id: Ic2eeaa986c2d93978d043216c46e8cb38cea769f
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>
This commit is contained in:
Marco Bubke
2018-08-22 21:13:05 +02:00
parent 6775347cd2
commit b2c3e683cf
13 changed files with 443 additions and 34 deletions

View File

@@ -24,7 +24,9 @@ HEADERS += \
$$PWD/indexdataconsumer.h \
$$PWD/projectpartqueue.h \
$$PWD/sourcesmanager.h \
$$PWD/symbolindexertaskqueue.h
$$PWD/symbolindexertaskqueue.h \
$$PWD/symbolindexertaskscheduler.h \
$$PWD/symbolscollectormanagerinterface.h
!isEmpty(LIBTOOLING_LIBS) {
SOURCES += \
@@ -71,4 +73,5 @@ SOURCES += \
$$PWD/projectpartartefact.cpp \
$$PWD/filestatuscache.cpp \
$$PWD/projectpartqueue.cpp \
$$PWD/symbolindexertaskqueue.cpp
$$PWD/symbolindexertaskqueue.cpp \
$$PWD/symbolindexertaskscheduler.cpp

View File

@@ -34,14 +34,18 @@
namespace ClangBackEnd {
class SymbolsCollectorInterface;
class SymbolStorageInterface;
class SymbolIndexerTask
{
public:
using CallableType = std::function<void()>;
using Callable = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage)>;
SymbolIndexerTask(FilePathId filePathId,
std::size_t projectPartId,
CallableType &&callable)
Callable &&callable)
: callable(std::move(callable)),
filePathId(filePathId),
projectPartId(projectPartId)
@@ -67,7 +71,7 @@ public:
}
public:
CallableType callable;
Callable callable;
FilePathId filePathId;
std::size_t projectPartId;
};

View File

@@ -0,0 +1,86 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "symbolindexertaskscheduler.h"
#include <symbolscollectormanagerinterface.h>
#include <symbolscollectorinterface.h>
#include <algorithm>
#include <thread>
namespace ClangBackEnd {
void SymbolIndexerTaskScheduler::addTasks(std::vector<Task> &&tasks)
{
for (auto &task : tasks) {
auto callWrapper = [task=std::move(task)] (
std::reference_wrapper<SymbolsCollectorInterface> symbolsCollector,
std::reference_wrapper<SymbolStorageInterface> symbolStorage)
-> SymbolsCollectorInterface& {
task(symbolsCollector.get(), symbolStorage.get());
return symbolsCollector;
};
m_futures.emplace_back(std::async(m_launchPolicy,
std::move(callWrapper),
std::ref(m_symbolsCollectorManager.unusedSymbolsCollector()),
std::ref(m_symbolStorage)));
}
}
const std::vector<SymbolIndexerTaskScheduler::Future> &SymbolIndexerTaskScheduler::futures() const
{
return m_futures;
}
int SymbolIndexerTaskScheduler::freeSlots()
{
removeFinishedFutures();
return std::max(m_hardware_concurrency - int(m_futures.size()), 0);
}
void SymbolIndexerTaskScheduler::syncTasks()
{
for (auto &future : m_futures)
future.wait();
}
void SymbolIndexerTaskScheduler::removeFinishedFutures()
{
auto notReady = [] (Future &future) {
return future.wait_for(std::chrono::duration<int>::zero()) != std::future_status::ready;
};
auto split = std::partition(m_futures.begin(), m_futures.end(), notReady);
std::for_each(split, m_futures.end(), [] (Future &future) {
future.get().setIsUsed(false);
});
m_futures.erase(split, m_futures.end());
}
} // namespace ClangBackEnd

View File

@@ -0,0 +1,75 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <functional>
#include <future>
#include <vector>
namespace ClangBackEnd {
class FilePathCachingInterface;
class SymbolsCollectorInterface;
class SymbolsCollectorManagerInterface;
class SymbolStorageInterface;
class SymbolIndexerTaskScheduler
{
public:
using Task = std::function<void(SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage)>;
using Future = std::future<SymbolsCollectorInterface&>;
SymbolIndexerTaskScheduler(SymbolsCollectorManagerInterface &symbolsCollectorManager,
SymbolStorageInterface &symbolStorage,
int hardware_concurrency,
std::launch launchPolicy = std::launch::async)
: m_symbolsCollectorManager(symbolsCollectorManager),
m_symbolStorage(symbolStorage),
m_hardware_concurrency(hardware_concurrency),
m_launchPolicy(launchPolicy)
{}
void addTasks(std::vector<Task> &&tasks);
const std::vector<Future> &futures() const;
int freeSlots();
void syncTasks();
private:
void removeFinishedFutures();
private:
std::vector<Future> m_futures;
SymbolsCollectorManagerInterface &m_symbolsCollectorManager;
SymbolStorageInterface &m_symbolStorage;
int m_hardware_concurrency;
std::launch m_launchPolicy;
};
} // namespace ClangBackEnd

View File

@@ -154,4 +154,14 @@ const SourceDependencies &SymbolsCollector::sourceDependencies() const
return m_collectMacrosSourceFileCallbacks.sourceDependencies();
}
bool SymbolsCollector::isUsed() const
{
return m_isUsed;
}
void SymbolsCollector::setIsUsed(bool isUsed)
{
m_isUsed = isUsed;
}
} // namespace ClangBackEnd

View File

@@ -35,7 +35,7 @@
namespace ClangBackEnd {
class SymbolsCollector : public SymbolsCollectorInterface
class SymbolsCollector final : public SymbolsCollectorInterface
{
public:
SymbolsCollector(FilePathCachingInterface &filePathCache);
@@ -56,6 +56,9 @@ public:
const FileStatuses &fileStatuses() const override;
const SourceDependencies &sourceDependencies() const override;
bool isUsed() const override;
void setIsUsed(bool isUsed) override;
private:
ClangTool m_clangTool;
SymbolEntries m_symbolEntries;
@@ -65,6 +68,7 @@ private:
CollectMacrosSourceFileCallbacks m_collectMacrosSourceFileCallbacks;
SourcesManager m_sourcesManager;
FilePathCachingInterface &m_filePathCache;
bool m_isUsed = false;
};
} // namespace ClangBackEnd

View File

@@ -63,6 +63,9 @@ public:
virtual const FileStatuses &fileStatuses() const = 0;
virtual const SourceDependencies &sourceDependencies() const = 0;
virtual bool isUsed() const = 0;
virtual void setIsUsed(bool isUsed) = 0;
protected:
~SymbolsCollectorInterface() = default;
};

View File

@@ -0,0 +1,43 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include <vector>
namespace ClangBackEnd {
class SymbolsCollectorInterface;
class SymbolsCollectorManagerInterface
{
public:
virtual SymbolsCollectorInterface &unusedSymbolsCollector() = 0;
protected:
~SymbolsCollectorManagerInterface() = default;
};
} // namespace ClangBackEnd

View File

@@ -62,4 +62,10 @@ public:
MOCK_CONST_METHOD0(sourceDependencies,
const ClangBackEnd::SourceDependencies &());
MOCK_CONST_METHOD0(isUsed,
bool());
MOCK_METHOD1(setIsUsed,
void(bool));
};

View File

@@ -0,0 +1,38 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#pragma once
#include "googletest.h"
#include <symbolscollectormanagerinterface.h>
class MockSymbolsCollectorManager : public ClangBackEnd::SymbolsCollectorManagerInterface
{
public:
MOCK_METHOD0(unusedSymbolsCollector,
ClangBackEnd::SymbolsCollectorInterface & ());
};

View File

@@ -29,12 +29,16 @@
namespace {
using ClangBackEnd::SymbolIndexerTask;
using ClangBackEnd::FilePathId;
using ClangBackEnd::SymbolsCollectorInterface;
using ClangBackEnd::SymbolIndexerTask;
using ClangBackEnd::SymbolStorageInterface;
using Callable = ClangBackEnd::SymbolIndexerTask::Callable;
MATCHER_P2(IsTask, filePathId, projectPartId,
std::string(negation ? "is't" : "is")
+ PrintToString(SymbolIndexerTask(filePathId, projectPartId, []{})))
+ PrintToString(SymbolIndexerTask(filePathId, projectPartId, Callable{})))
{
const SymbolIndexerTask &task = arg;
@@ -54,12 +58,12 @@ protected:
TEST_F(SymbolIndexerTaskQueue, AddTasks)
{
queue.addOrUpdateTasks({{{1, 2}, projectPartId("foo"), [] {}},
{{1, 4}, projectPartId("foo"), [] {}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("foo"), Callable{}},
{{1, 4}, projectPartId("foo"), Callable{}}});
queue.addOrUpdateTasks({{{1, 1}, projectPartId("foo"), [] {}},
{{1, 3}, projectPartId("foo"), [] {}},
{{1, 5}, projectPartId("foo"), [] {}}});
queue.addOrUpdateTasks({{{1, 1}, projectPartId("foo"), Callable{}},
{{1, 3}, projectPartId("foo"), Callable{}},
{{1, 5}, projectPartId("foo"), Callable{}}});
ASSERT_THAT(queue.tasks(),
ElementsAre(IsTask(FilePathId{1, 1}, projectPartId("foo")),
@@ -71,12 +75,12 @@ TEST_F(SymbolIndexerTaskQueue, AddTasks)
TEST_F(SymbolIndexerTaskQueue, ReplaceTask)
{
queue.addOrUpdateTasks({{{1, 1}, projectPartId("foo"), [] {}},
{{1, 3}, projectPartId("foo"), [] {}},
{{1, 5}, projectPartId("foo"), [] {}}});
queue.addOrUpdateTasks({{{1, 1}, projectPartId("foo"), Callable{}},
{{1, 3}, projectPartId("foo"), Callable{}},
{{1, 5}, projectPartId("foo"), Callable{}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("foo"), [] {}},
{{1, 3}, projectPartId("foo"), [] {}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("foo"), Callable{}},
{{1, 3}, projectPartId("foo"), Callable{}}});
ASSERT_THAT(queue.tasks(),
ElementsAre(IsTask(FilePathId{1, 1}, projectPartId("foo")),
@@ -87,12 +91,12 @@ TEST_F(SymbolIndexerTaskQueue, ReplaceTask)
TEST_F(SymbolIndexerTaskQueue, AddTaskWithDifferentProjectId)
{
queue.addOrUpdateTasks({{{1, 1}, projectPartId("foo"), [] {}},
{{1, 3}, projectPartId("foo"), [] {}},
{{1, 5}, projectPartId("foo"), [] {}}});
queue.addOrUpdateTasks({{{1, 1}, projectPartId("foo"), Callable{}},
{{1, 3}, projectPartId("foo"), Callable{}},
{{1, 5}, projectPartId("foo"), Callable{}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("bar"), [] {}},
{{1, 3}, projectPartId("bar"), [] {}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("bar"), Callable{}},
{{1, 3}, projectPartId("bar"), Callable{}}});
ASSERT_THAT(queue.tasks(),
ElementsAre(IsTask(FilePathId{1, 1}, projectPartId("foo")),
@@ -104,15 +108,15 @@ TEST_F(SymbolIndexerTaskQueue, AddTaskWithDifferentProjectId)
TEST_F(SymbolIndexerTaskQueue, RemoveTaskByProjectParts)
{
queue.addOrUpdateTasks({{{1, 1}, projectPartId("yi"), [] {}},
{{1, 3}, projectPartId("yi"), [] {}},
{{1, 5}, projectPartId("yi"), [] {}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("er"), [] {}},
{{1, 3}, projectPartId("er"), [] {}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("san"), [] {}},
{{1, 3}, projectPartId("san"), [] {}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("se"), [] {}},
{{1, 3}, projectPartId("se"), [] {}}});
queue.addOrUpdateTasks({{{1, 1}, projectPartId("yi"), Callable{}},
{{1, 3}, projectPartId("yi"), Callable{}},
{{1, 5}, projectPartId("yi"), Callable{}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("er"), Callable{}},
{{1, 3}, projectPartId("er"), Callable{}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("san"), Callable{}},
{{1, 3}, projectPartId("san"), Callable{}}});
queue.addOrUpdateTasks({{{1, 2}, projectPartId("se"), Callable{}},
{{1, 3}, projectPartId("se"), Callable{}}});
queue.removeTasks({"er", "san"});

View File

@@ -0,0 +1,131 @@
/****************************************************************************
**
** Copyright (C) 2018 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of Qt Creator.
**
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
****************************************************************************/
#include "googletest.h"
#include "mocksymbolscollectormanager.h"
#include "mocksymbolscollector.h"
#include "mocksymbolstorage.h"
#include <symbolindexertaskscheduler.h>
namespace {
using ClangBackEnd::SymbolsCollectorInterface;
using ClangBackEnd::SymbolStorageInterface;
class SymbolIndexerTaskScheduler : public testing::Test
{
protected:
void SetUp()
{
ON_CALL(mockSymbolsCollectorManager, unusedSymbolsCollector()).WillByDefault(ReturnRef(mockSymbolsCollector));
}
protected:
MockFunction<void()> mock;
ClangBackEnd::SymbolIndexerTaskScheduler::Task call{
[&] (SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage) {
mock.Call(); }};
ClangBackEnd::SymbolIndexerTaskScheduler::Task nocall{
[&] (SymbolsCollectorInterface &symbolsCollector,
SymbolStorageInterface &symbolStorage) {}};
NiceMock<MockSymbolsCollectorManager> mockSymbolsCollectorManager;
NiceMock<MockSymbolsCollector> mockSymbolsCollector;
MockSymbolStorage mockSymbolStorage;
ClangBackEnd::SymbolIndexerTaskScheduler scheduler{mockSymbolsCollectorManager, mockSymbolStorage, 4};
ClangBackEnd::SymbolIndexerTaskScheduler deferedScheduler{mockSymbolsCollectorManager, mockSymbolStorage, 4, std::launch::deferred};
};
TEST_F(SymbolIndexerTaskScheduler, AddTasks)
{
deferedScheduler.addTasks({nocall});
ASSERT_THAT(deferedScheduler.futures(), SizeIs(1));
}
TEST_F(SymbolIndexerTaskScheduler, AddTasksCallsFunction)
{
EXPECT_CALL(mock, Call()).Times(2);
scheduler.addTasks({call, call});
}
TEST_F(SymbolIndexerTaskScheduler, FreeSlots)
{
deferedScheduler.addTasks({nocall, nocall});
auto count = deferedScheduler.freeSlots();
ASSERT_THAT(count, 2);
}
TEST_F(SymbolIndexerTaskScheduler, ReturnZeroFreeSlotsIfMoreCallsThanCores)
{
deferedScheduler.addTasks({nocall, nocall, nocall, nocall, nocall, nocall});
auto count = deferedScheduler.freeSlots();
ASSERT_THAT(count, 0);
}
TEST_F(SymbolIndexerTaskScheduler, FreeSlotsAfterFinishing)
{
scheduler.addTasks({nocall, nocall});
scheduler.syncTasks();
auto count = scheduler.freeSlots();
ASSERT_THAT(count, 4);
}
TEST_F(SymbolIndexerTaskScheduler, NoFuturesAfterFreeSlots)
{
scheduler.addTasks({nocall, nocall});
scheduler.syncTasks();
scheduler.freeSlots();
ASSERT_THAT(scheduler.futures(), IsEmpty());
}
TEST_F(SymbolIndexerTaskScheduler, FreeSlotsCallSymbolsCollectorSetIsUnused)
{
scheduler.addTasks({nocall, nocall});
scheduler.syncTasks();
EXPECT_CALL(mockSymbolsCollector, setIsUsed(false)).Times(2);
scheduler.freeSlots();
}
TEST_F(SymbolIndexerTaskScheduler, AddTaskCallSymbolsCollectorManagerUnusedSymbolsCollector)
{
EXPECT_CALL(mockSymbolsCollectorManager, unusedSymbolsCollector()).Times(2);
scheduler.addTasks({nocall, nocall});
}
}

View File

@@ -96,7 +96,8 @@ SOURCES += \
projectpartqueue-test.cpp \
generatedfiles-test.cpp \
sourcesmanager-test.cpp \
symbolindexertaskqueue-test.cpp
symbolindexertaskqueue-test.cpp \
symbolindexertaskscheduler-test.cpp
!isEmpty(LIBCLANG_LIBS) {
SOURCES += \
@@ -232,7 +233,8 @@ HEADERS += \
mocksqlitetransactionbackend.h \
mockprojectpartprovider.h \
mockprecompiledheaderstorage.h \
mockeditormanager.h
mockeditormanager.h \
mocksymbolscollectormanager.h
!isEmpty(LIBCLANG_LIBS) {
HEADERS += \
chunksreportedmonitor.h \