ClangRefactoring: Add SymbolIndexerTaskQueue

A first step for concurrent index task.

Change-Id: I9a0dba9f4a67ee605281516785697045b34e2694
Reviewed-by: Ivan Donchevskii <ivan.donchevskii@qt.io>

src/tools/clangrefactoringbackend/source/clangrefactoringbackend-source.pri

@@ -23,7 +23,8 @@ HEADERS += \
     $$PWD/filestatuscache.h \
     $$PWD/indexdataconsumer.h \
     $$PWD/projectpartqueue.h \
-    $$PWD/sourcesmanager.h
+    $$PWD/sourcesmanager.h \
+    $$PWD/symbolindexertaskqueue.h
 
 !isEmpty(LIBTOOLING_LIBS) {
 SOURCES += \
@@ -69,4 +70,5 @@ SOURCES += \
     $$PWD/symbolindexer.cpp \
     $$PWD/projectpartartefact.cpp \
     $$PWD/filestatuscache.cpp \
-    $$PWD/projectpartqueue.cpp
+    $$PWD/projectpartqueue.cpp \
+    $$PWD/symbolindexertaskqueue.cpp

src/tools/clangrefactoringbackend/source/symbolindexertaskqueue.cpp

@@ -0,0 +1,134 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "symbolindexertaskqueue.h"
+
+namespace ClangBackEnd {
+
+namespace {
+
+template<class InputIt1,
+         class InputIt2,
+         class OutputIt,
+         class Merge>
+OutputIt set_union_merge(InputIt1 first1,
+                         InputIt1 last1,
+                         InputIt2 first2,
+                         InputIt2 last2,
+                         OutputIt d_first,
+                         Merge merge)
+{
+    for (; first1 != last1; ++d_first) {
+        if (first2 == last2)
+            return std::copy(first1, last1, d_first);
+        if (*first2 < *first1) {
+            *d_first = *first2++;
+        } else {
+            if (*first1 < *first2) {
+                *d_first = *first1;
+            } else {
+                *d_first = merge(*first1, *first2);
+                ++first2;
+            }
+            ++first1;
+        }
+    }
+    return std::copy(first2, last2, d_first);
+}
+
+}
+
+SymbolIndexerTaskQueue::SymbolIndexerTaskQueue()
+{
+
+}
+
+void SymbolIndexerTaskQueue::addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks)
+{
+    std::vector<SymbolIndexerTask> mergedTasks;
+    mergedTasks.reserve(m_tasks.size() + tasks.size());
+
+    auto merge = [] (SymbolIndexerTask &&first, SymbolIndexerTask &&second) {
+        first.callable = std::move(second.callable);
+
+        return std::move(first);
+    };
+
+    set_union_merge(std::make_move_iterator(tasks.begin()),
+                    std::make_move_iterator(tasks.end()),
+                    std::make_move_iterator(m_tasks.begin()),
+                    std::make_move_iterator(m_tasks.end()),
+                    std::back_inserter(mergedTasks),
+                    merge);
+
+    m_tasks = std::move(mergedTasks);
+}
+
+void SymbolIndexerTaskQueue::removeTasks(const Utils::SmallStringVector &projectPartIds)
+{
+    std::vector<std::size_t> ids = projectPartNumberIds(projectPartIds);
+
+    auto shouldBeRemoved = [&] (const SymbolIndexerTask& task) {
+        return std::binary_search(ids.begin(), ids.end(), task.projectPartId);
+    };
+
+    auto newEnd = std::remove_if(m_tasks.begin(), m_tasks.end(), shouldBeRemoved);
+
+    m_tasks.erase(newEnd, m_tasks.end());
+}
+
+const std::vector<SymbolIndexerTask> &SymbolIndexerTaskQueue::tasks() const
+{
+    return m_tasks;
+}
+
+std::size_t SymbolIndexerTaskQueue::projectPartNumberId(Utils::SmallStringView projectPartId)
+{
+    auto found = std::find(m_projectPartIds.begin(), m_projectPartIds.end(), projectPartId);
+
+    if (found != m_projectPartIds.end())
+        return std::size_t(std::distance(m_projectPartIds.begin(), found));
+
+    m_projectPartIds.emplace_back(projectPartId);
+
+    return m_projectPartIds.size() - 1;
+}
+
+std::vector<std::size_t> SymbolIndexerTaskQueue::projectPartNumberIds(const Utils::SmallStringVector &projectPartIds)
+{
+    std::vector<std::size_t> ids;
+    std::transform(projectPartIds.begin(),
+                   projectPartIds.end(),
+                   std::back_inserter(ids),
+                   [&] (Utils::SmallStringView projectPartId) {
+        return projectPartNumberId(projectPartId);
+    });
+
+    std::sort(ids.begin(), ids.end());
+
+    return ids;
+}
+
+} // namespace ClangBackEnd

src/tools/clangrefactoringbackend/source/symbolindexertaskqueue.h

@@ -0,0 +1,96 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#pragma once
+
+#include <filepathid.h>
+
+#include <utils/smallstringvector.h>
+
+#include <functional>
+#include <vector>
+
+namespace ClangBackEnd {
+
+class SymbolIndexerTask
+{
+public:
+    using CallableType = std::function<void()>;
+
+    SymbolIndexerTask(FilePathId filePathId,
+                      std::size_t projectPartId,
+                      CallableType &&callable)
+        : callable(std::move(callable)),
+          filePathId(filePathId),
+          projectPartId(projectPartId)
+    {
+    }
+
+    SymbolIndexerTask clone() const
+    {
+        return *this;
+    }
+
+    friend
+    bool operator==(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
+    {
+        return first.filePathId == second.filePathId && first.projectPartId == second.projectPartId;
+    }
+
+    friend
+    bool operator<(const SymbolIndexerTask &first, const SymbolIndexerTask &second)
+    {
+        return std::tie(first.filePathId, first.projectPartId)
+             < std::tie(second.filePathId, second.projectPartId);
+    }
+
+public:
+    CallableType callable;
+    FilePathId filePathId;
+    std::size_t projectPartId;
+};
+
+class SymbolIndexerTaskQueue
+{
+public:
+    SymbolIndexerTaskQueue();
+
+    void addOrUpdateTasks(std::vector<SymbolIndexerTask> &&tasks)
+    /* [[expects: std::is_sorted(tasks)]] */;
+    void removeTasks(const Utils::SmallStringVector &projectPartIds)
+    /* [[expects: std::is_sorted(projectPartIds)]] */;
+
+    const std::vector<SymbolIndexerTask> &tasks() const;
+
+    std::size_t projectPartNumberId(Utils::SmallStringView projectPartId);
+    std::vector<std::size_t> projectPartNumberIds(const Utils::SmallStringVector &projectPartIds)
+    /* [[ensures result: std::is_sorted(result)]] */;
+
+private:
+    std::vector<Utils::SmallString> m_projectPartIds;
+    std::vector<SymbolIndexerTask> m_tasks;
+};
+
+} // namespace ClangBackEnd

tests/unit/unittest/gtest-creator-printing.cpp

@@ -50,6 +50,7 @@
 #include <tokenprocessor.h>
 #include <filepathview.h>
 #include <symbolentry.h>
+#include <symbolindexertaskqueue.h>
 #include <symbol.h>
 #include <tooltipinfo.h>
 #include <projectpartentry.h>
@@ -1023,6 +1024,11 @@ std::ostream &operator<<(std::ostream &os, const ReferencesResult &value)
     return os;
 }
 
+std::ostream &operator<<(std::ostream &out, const SymbolIndexerTask &task)
+{
+    return out << "(" << task.filePathId << ", " << task.projectPartId << ")";
+}
+
 void PrintTo(const FilePath &filePath, ::std::ostream *os)
 {
     *os << filePath;

tests/unit/unittest/gtest-creator-printing.h

@@ -171,6 +171,7 @@ class UpdateGeneratedFilesMessage;
 class RemoveGeneratedFilesMessage;
 class SuspendResumeJobsEntry;
 class ReferencesResult;
+class SymbolIndexerTask;
 
 std::ostream &operator<<(std::ostream &out, const SourceLocationEntry &entry);
 std::ostream &operator<<(std::ostream &out, const IdPaths &idPaths);
@@ -252,6 +253,7 @@ std::ostream &operator<<(std::ostream &out, const UpdateGeneratedFilesMessage &m
 std::ostream &operator<<(std::ostream &out, const RemoveGeneratedFilesMessage &message);
 std::ostream &operator<<(std::ostream &os, const SuspendResumeJobsEntry &entry);
 std::ostream &operator<<(std::ostream &os, const ReferencesResult &value);
+std::ostream &operator<<(std::ostream &out, const SymbolIndexerTask &task);
 
 void PrintTo(const FilePath &filePath, ::std::ostream *os);
 void PrintTo(const FilePathView &filePathView, ::std::ostream *os);

tests/unit/unittest/symbolindexertaskqueue-test.cpp

@@ -0,0 +1,161 @@
+/****************************************************************************
+**
+** Copyright (C) 2018 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of Qt Creator.
+**
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+****************************************************************************/
+
+#include "googletest.h"
+
+#include <symbolindexertaskqueue.h>
+
+namespace {
+
+using ClangBackEnd::SymbolIndexerTask;
+using ClangBackEnd::FilePathId;
+
+MATCHER_P2(IsTask, filePathId, projectPartId,
+          std::string(negation ? "is't" : "is")
+          + PrintToString(SymbolIndexerTask(filePathId, projectPartId, []{})))
+{
+    const SymbolIndexerTask &task = arg;
+
+    return task.filePathId == filePathId && task.projectPartId == projectPartId;
+}
+
+class SymbolIndexerTaskQueue : public testing::Test
+{
+protected:
+    std::size_t projectPartId(const Utils::SmallString &projectPartId)
+    {
+        return queue.projectPartNumberId(projectPartId);
+    }
+protected:
+    ClangBackEnd::SymbolIndexerTaskQueue queue;
+};
+
+TEST_F(SymbolIndexerTaskQueue, AddTasks)
+{
+    queue.addOrUpdateTasks({{{1, 2}, projectPartId("foo"), [] {}},
+                            {{1, 4}, projectPartId("foo"), [] {}}});
+
+    queue.addOrUpdateTasks({{{1, 1}, projectPartId("foo"), [] {}},
+                            {{1, 3}, projectPartId("foo"), [] {}},
+                            {{1, 5}, projectPartId("foo"), [] {}}});
+
+    ASSERT_THAT(queue.tasks(),
+                ElementsAre(IsTask(FilePathId{1, 1}, projectPartId("foo")),
+                            IsTask(FilePathId{1, 2}, projectPartId("foo")),
+                            IsTask(FilePathId{1, 3}, projectPartId("foo")),
+                            IsTask(FilePathId{1, 4}, projectPartId("foo")),
+                            IsTask(FilePathId{1, 5}, projectPartId("foo"))));
+}
+
+TEST_F(SymbolIndexerTaskQueue, ReplaceTask)
+{
+    queue.addOrUpdateTasks({{{1, 1}, projectPartId("foo"), [] {}},
+                            {{1, 3}, projectPartId("foo"), [] {}},
+                            {{1, 5}, projectPartId("foo"), [] {}}});
+
+    queue.addOrUpdateTasks({{{1, 2}, projectPartId("foo"), [] {}},
+                            {{1, 3}, projectPartId("foo"), [] {}}});
+
+    ASSERT_THAT(queue.tasks(),
+                ElementsAre(IsTask(FilePathId{1, 1}, projectPartId("foo")),
+                            IsTask(FilePathId{1, 2}, projectPartId("foo")),
+                            IsTask(FilePathId{1, 3}, projectPartId("foo")),
+                            IsTask(FilePathId{1, 5}, projectPartId("foo"))));
+}
+
+TEST_F(SymbolIndexerTaskQueue, AddTaskWithDifferentProjectId)
+{
+    queue.addOrUpdateTasks({{{1, 1}, projectPartId("foo"), [] {}},
+                            {{1, 3}, projectPartId("foo"), [] {}},
+                            {{1, 5}, projectPartId("foo"), [] {}}});
+
+    queue.addOrUpdateTasks({{{1, 2}, projectPartId("bar"), [] {}},
+                            {{1, 3}, projectPartId("bar"), [] {}}});
+
+    ASSERT_THAT(queue.tasks(),
+                ElementsAre(IsTask(FilePathId{1, 1}, projectPartId("foo")),
+                            IsTask(FilePathId{1, 2}, projectPartId("bar")),
+                            IsTask(FilePathId{1, 3}, projectPartId("foo")),
+                            IsTask(FilePathId{1, 3}, projectPartId("bar")),
+                            IsTask(FilePathId{1, 5}, projectPartId("foo"))));
+}
+
+TEST_F(SymbolIndexerTaskQueue, RemoveTaskByProjectParts)
+{
+    queue.addOrUpdateTasks({{{1, 1}, projectPartId("yi"), [] {}},
+                            {{1, 3}, projectPartId("yi"), [] {}},
+                            {{1, 5}, projectPartId("yi"), [] {}}});
+    queue.addOrUpdateTasks({{{1, 2}, projectPartId("er"), [] {}},
+                            {{1, 3}, projectPartId("er"), [] {}}});
+    queue.addOrUpdateTasks({{{1, 2}, projectPartId("san"), [] {}},
+                            {{1, 3}, projectPartId("san"), [] {}}});
+    queue.addOrUpdateTasks({{{1, 2}, projectPartId("se"), [] {}},
+                            {{1, 3}, projectPartId("se"), [] {}}});
+
+    queue.removeTasks({"er", "san"});
+
+    ASSERT_THAT(queue.tasks(),
+                ElementsAre(IsTask(FilePathId{1, 1}, projectPartId("yi")),
+                            IsTask(FilePathId{1, 2}, projectPartId("se")),
+                            IsTask(FilePathId{1, 3}, projectPartId("yi")),
+                            IsTask(FilePathId{1, 3}, projectPartId("se")),
+                            IsTask(FilePathId{1, 5}, projectPartId("yi"))));
+}
+
+TEST_F(SymbolIndexerTaskQueue, GetProjectPartIdIfEmpty)
+{
+    auto id = queue.projectPartNumberId("foo");
+
+    ASSERT_THAT(id , 0);
+}
+
+TEST_F(SymbolIndexerTaskQueue, GetProjectPartIdIfNotExists)
+{
+    queue.projectPartNumberId("foo");
+
+    auto id = queue.projectPartNumberId("bar");
+
+    ASSERT_THAT(id , 1);
+}
+
+TEST_F(SymbolIndexerTaskQueue, GetProjectPartIdIfExists)
+{
+    queue.projectPartNumberId("foo");
+    queue.projectPartNumberId("bar");
+
+    auto id = queue.projectPartNumberId("foo");
+
+    ASSERT_THAT(id , 0);
+}
+
+TEST_F(SymbolIndexerTaskQueue, GetProjectPartIds)
+{
+    queue.projectPartNumberIds({"yi", "er", "san"});
+
+    auto ids = queue.projectPartNumberIds({"yi", "se", "san"});
+
+    ASSERT_THAT(ids , ElementsAre(0, 2, 3));
+}
+}

tests/unit/unittest/unittest.pro

@@ -95,7 +95,8 @@ SOURCES += \
     precompiledheaderstorage-test.cpp \
     projectpartqueue-test.cpp \
     generatedfiles-test.cpp \
-    sourcesmanager-test.cpp
+    sourcesmanager-test.cpp \
+    symbolindexertaskqueue-test.cpp
 
 !isEmpty(LIBCLANG_LIBS) {
 SOURCES += \