From 0fabde31abf8b6861291b7e54588a243a9e85396 Mon Sep 17 00:00:00 2001 From: Ivan Donchevskii Date: Fri, 2 Nov 2018 16:07:42 +0100 Subject: [PATCH] CompilationDatabase: Speed up parsing project file Avoid expensive indexOf and check for the exactly same flags before applying the full filter. Change-Id: I6936b2022a2b439aad7bf0a65280c3db16d00c34 Reviewed-by: Marco Bubke --- .../compilationdatabaseproject.cpp | 98 ++++++++++++------- .../compilationdatabaseutils.cpp | 29 +++--- .../compilationdatabaseutils.h | 2 + .../compilationdatabaseutils-test.cpp | 16 ++- 4 files changed, 90 insertions(+), 55 deletions(-) diff --git a/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseproject.cpp b/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseproject.cpp index 155ed5185aa..f8c19219886 100644 --- a/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseproject.cpp +++ b/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseproject.cpp @@ -75,6 +75,7 @@ QStringList jsonObjectFlags(const QJsonObject &object) if (arguments.isEmpty()) { flags = splitCommandLine(object["command"].toString()); } else { + flags.reserve(arguments.size()); for (const QJsonValue &arg : arguments) flags.append(arg.toString()); } @@ -284,27 +285,43 @@ FolderNode *createFoldersIfNeeded(FolderNode *root, const Utils::FileName &folde return parent; } -void createFolders(FolderNode *root, const Utils::FileName &rootPath) +FileType fileTypeForName(const QString &fileName) +{ + CppTools::ProjectFile::Kind fileKind = CppTools::ProjectFile::classify(fileName); + if (CppTools::ProjectFile::isHeader(fileKind)) + return FileType::Header; + return FileType::Source; +} + +void createTree(FolderNode *root, + const Utils::FileName &rootPath, + const CppTools::RawProjectParts &rpps) { root->setAbsoluteFilePathAndLine(rootPath, -1); - for (Node *child : root->nodes()) { - FileNode *fileNode = child->asFileNode(); - if (!fileNode) - continue; - - FolderNode *parentNode = createFoldersIfNeeded(root, - fileNode->filePath().parentDir()); - child->setParentFolderNode(nullptr); - std::unique_ptr childNode = root->takeNode(child); - if (!parentNode->fileNode(child->filePath())) - parentNode->addNode(std::move(childNode)); + for (const CppTools::RawProjectPart &rpp : rpps) { + for (const QString &filePath : rpp.files) { + Utils::FileName fileName = Utils::FileName::fromString(filePath); + FolderNode *parentNode = createFoldersIfNeeded(root, fileName.parentDir()); + if (!parentNode->fileNode(fileName)) { + parentNode->addNode(std::make_unique(fileName, + fileTypeForName(fileName.fileName()), + false)); + } + } } } -std::vector readJsonObjects(const QString &filePath) +struct Entry { - std::vector result; + QStringList flags; + Utils::FileName fileName; + QString workingDir; +}; + +std::vector readJsonObjects(const QString &filePath) +{ + std::vector result; QFile file(filePath); if (!file.open(QIODevice::ReadOnly)) return result; @@ -322,7 +339,12 @@ std::vector readJsonObjects(const QString &filePath) continue; } - result.push_back(document.object()); + const QJsonObject object = document.object(); + const Utils::FileName fileName = jsonObjectFilename(object); + const QStringList flags + = filterFromFileName(jsonObjectFlags(object), fileName.toFileInfo().baseName()); + result.push_back({flags, fileName, object["directory"].toString()}); + objectStart = contents.indexOf('{', objectEnd + 1); objectEnd = contents.indexOf('}', objectStart + 1); } @@ -334,7 +356,7 @@ std::vector readJsonObjects(const QString &filePath) void CompilationDatabaseProject::buildTreeAndProjectParts(const Utils::FileName &projectFile) { - std::vector array = readJsonObjects(projectFilePath().toString()); + std::vector array = readJsonObjects(projectFilePath().toString()); if (array.empty()) { emitParsingFinished(false); return; @@ -346,37 +368,37 @@ void CompilationDatabaseProject::buildTreeAndProjectParts(const Utils::FileName Utils::FileName commonPath; ToolChain *cToolchain = nullptr; ToolChain *cxxToolchain = nullptr; - for (const QJsonObject &object : array) { - Utils::FileName fileName = jsonObjectFilename(object); - const QStringList flags = jsonObjectFlags(object); + + std::sort(array.begin(), array.end(), [](const Entry &lhs, const Entry &rhs) { + return std::lexicographical_compare(lhs.flags.begin(), lhs.flags.end(), + rhs.flags.begin(), rhs.flags.end()); + }); + + const Entry *prevEntry = nullptr; + for (const Entry &entry : array) { + if (prevEntry && prevEntry->flags == entry.flags) { + rpps.back().files.append(entry.fileName.toString()); + continue; + } + + prevEntry = &entry; commonPath = rpps.empty() - ? fileName.parentDir() - : Utils::FileUtils::commonPath(commonPath, fileName); - - ProjectExplorer::FileType type = ProjectExplorer::FileType::Unknown; - root->addNode(std::make_unique(fileName, type, false)); + ? entry.fileName.parentDir() + : Utils::FileUtils::commonPath(commonPath, entry.fileName); CppTools::RawProjectPart rpp = makeRawProjectPart(projectFile, m_kit.get(), cToolchain, cxxToolchain, - object["directory"].toString(), - fileName, - flags); - int rppIndex = Utils::indexOf(rpps, [&rpp](const CppTools::RawProjectPart ¤tRpp) { - return rpp.buildSystemTarget == currentRpp.buildSystemTarget - && rpp.headerPaths == currentRpp.headerPaths - && rpp.projectMacros == currentRpp.projectMacros - && rpp.flagsForCxx.commandLineFlags == currentRpp.flagsForCxx.commandLineFlags; - }); - if (rppIndex == -1) - rpps.append(rpp); - else - rpps[rppIndex].files.append(rpp.files); + entry.workingDir, + entry.fileName, + entry.flags); + + rpps.append(rpp); } - createFolders(root.get(), commonPath); + createTree(root.get(), commonPath, rpps); root->addNode(std::make_unique( projectFile, diff --git a/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseutils.cpp b/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseutils.cpp index 2dd20dd9e5f..1884aa1c6d3 100644 --- a/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseutils.cpp +++ b/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseutils.cpp @@ -81,6 +81,19 @@ static CppTools::ProjectFile::Kind fileKindFromString(QString flag) return ProjectFile::Unclassified; } +QStringList filterFromFileName(const QStringList &flags, QString baseName) +{ + baseName.append('.'); // to match name.c, name.o, etc. + QStringList result; + result.reserve(flags.size()); + for (const QString &flag : flags) { + if (!flag.contains(baseName)) + result.push_back(flag); + } + + return result; +} + void filteredFlags(const QString &fileName, const QString &workingDir, QStringList &flags, @@ -88,13 +101,13 @@ void filteredFlags(const QString &fileName, Macros ¯os, CppTools::ProjectFile::Kind &fileKind) { - if (flags.isEmpty()) + if (flags.empty()) return; // Skip compiler call if present. bool skipNext = Utils::HostOsInfo::isWindowsHost() - ? (!flags.first().startsWith('/') && !flags.first().startsWith('-')) - : (!flags.first().startsWith('-')); + ? (!flags.front().startsWith('/') && !flags.front().startsWith('-')) + : (!flags.front().startsWith('-')); Utils::optional includePathType; Utils::optional macroType; bool fileKindIsNext = false; @@ -121,11 +134,6 @@ void filteredFlags(const QString &fileName, continue; } - if (flag == "-o") { - skipNext = true; - continue; - } - if (flag != "-x" && (fileKindIsNext || flag == "/TC" || flag == "/TP" || flag.startsWith("/Tc") || flag.startsWith("/Tp") || flag.startsWith("-x"))) { @@ -139,16 +147,13 @@ void filteredFlags(const QString &fileName, continue; } - if (flag == "-c" || flag == "-pedantic" + if (flag == "-o" || flag == "-MF" || flag == "-c" || flag == "-pedantic" || flag.startsWith("-O") || flag.startsWith("-W") || flag.startsWith("-w") || QString::compare(flag, "-fpic", Qt::CaseInsensitive) == 0 || QString::compare(flag, "-fpie", Qt::CaseInsensitive) == 0) { continue; } - if (flag.endsWith(fileName)) - continue; - if ((flag.startsWith("-I") || flag.startsWith("-isystem") || flag.startsWith("/I")) && flag != "-I" && flag != "-isystem" && flag != "/I") { bool userInclude = flag.startsWith("-I"); diff --git a/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseutils.h b/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseutils.h index 5e123a8c271..e465b6ba23c 100644 --- a/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseutils.h +++ b/src/plugins/compilationdatabaseprojectmanager/compilationdatabaseutils.h @@ -38,6 +38,8 @@ class Macro; namespace CompilationDatabaseProjectManager { +QStringList filterFromFileName(const QStringList &flags, QString baseName); + void filteredFlags(const QString &fileName, const QString &workingDir, QStringList &flags, diff --git a/tests/unit/unittest/compilationdatabaseutils-test.cpp b/tests/unit/unittest/compilationdatabaseutils-test.cpp index 2d528b9793a..252ab2ffd62 100644 --- a/tests/unit/unittest/compilationdatabaseutils-test.cpp +++ b/tests/unit/unittest/compilationdatabaseutils-test.cpp @@ -53,11 +53,18 @@ TEST_F(CompilationDatabaseUtils, FilterEmptyFlags) ASSERT_THAT(flags.isEmpty(), true); } +TEST_F(CompilationDatabaseUtils, FilterFromFilename) +{ + flags = filterFromFileName(QStringList{"-o", "foo.o"}, "foo"); + + ASSERT_THAT(flags, QStringList{"-o"}); +} + TEST_F(CompilationDatabaseUtils, FilterArguments) { fileName = "compileroptionsbuilder.cpp"; workingDir = "C:/build-qtcreator-MinGW_32bit-Debug"; - flags = QStringList { + flags = filterFromFileName(QStringList { "clang++", "-c", "-m32", @@ -77,7 +84,7 @@ TEST_F(CompilationDatabaseUtils, FilterArguments) "-x", "c++", "C:\\qt-creator\\src\\plugins\\cpptools\\compileroptionsbuilder.cpp" - }; + }, "compileroptionsbuilder"); filteredFlags(fileName, workingDir, flags, headerPaths, macros, fileKind); @@ -146,7 +153,7 @@ TEST_F(CompilationDatabaseUtils, FilterCommand) { fileName = "SemaCodeComplete.cpp"; workingDir = "C:/build-qt_llvm-msvc2017_64bit-Debug"; - flags = splitCommandLine(kCmakeCommand); + flags = filterFromFileName(splitCommandLine(kCmakeCommand), "SemaCodeComplete"); filteredFlags(fileName, workingDir, flags, headerPaths, macros, fileKind); @@ -188,8 +195,7 @@ TEST_F(CompilationDatabaseUtils, FileKindDifferentFromExtension2) TEST_F(CompilationDatabaseUtils, SkipOutputFiles) { - fileName = "foo.cpp"; - flags = QStringList{"-o", "foo.o"}; + flags = filterFromFileName(QStringList{"-o", "foo.o"}, "foo"); filteredFlags(fileName, workingDir, flags, headerPaths, macros, fileKind);