From 686a40d199864c1b2e29defc1ba67fd73f5aaaf0 Mon Sep 17 00:00:00 2001 From: Jarek Kobus Date: Fri, 12 May 2023 21:38:33 +0200 Subject: [PATCH] SubDirFileIterator: Add manual performance test Compare the total time spent on iterating a big file tree with 3 different time iterators: - Utils::SubDirFileIterator - manually written iterator using QDir::entryInfoList() - QDirIterator The iterator run through about one million files (including about ~100K directories). The more files above this number to be iterated over, the bigger the relative time difference of SubDirFileIterator compared to other iterators. The number of generated files depends on the running machine's core number. In my case: Number of cores: 24 Number of generated files: 898753 Number of generated directories: 112345 Time spent on generating file tree: ~2 seconds Time spent on iterating using SubDirIterator: ~80 seconds Time spent on iterating using manual iterator: ~8 seconds Time spent on iterating using QDirIterator: ~4 seconds Time spent on removing generated file tree: ~2 seconds Task-number: QTCREATORBUG-28892 Change-Id: I94d7cf0169a470820dc27f39c9cdb4150eea51c1 Reviewed-by: hjk Reviewed-by: Qt CI Bot Reviewed-by: --- tests/manual/CMakeLists.txt | 1 + tests/manual/manual.qbs | 1 + .../manual/subdirfileiterator/CMakeLists.txt | 10 + .../subdirfileiterator/subdirfileiterator.qbs | 22 ++ .../tst_subdirfileiterator.cpp | 257 ++++++++++++++++++ 5 files changed, 291 insertions(+) create mode 100644 tests/manual/subdirfileiterator/CMakeLists.txt create mode 100644 tests/manual/subdirfileiterator/subdirfileiterator.qbs create mode 100644 tests/manual/subdirfileiterator/tst_subdirfileiterator.cpp diff --git a/tests/manual/CMakeLists.txt b/tests/manual/CMakeLists.txt index c171a70deb6..fb0b611f4f7 100644 --- a/tests/manual/CMakeLists.txt +++ b/tests/manual/CMakeLists.txt @@ -16,5 +16,6 @@ add_subdirectory(proparser) # add_subdirectory(qt4projectmanager) # add_subdirectory(search) add_subdirectory(shootout) +add_subdirectory(subdirfileiterator) add_subdirectory(tasktree) add_subdirectory(widgets) diff --git a/tests/manual/manual.qbs b/tests/manual/manual.qbs index e3ebf0707a3..8e157a9216b 100644 --- a/tests/manual/manual.qbs +++ b/tests/manual/manual.qbs @@ -13,6 +13,7 @@ Project { "pluginview/pluginview.qbs", "proparser/testreader.qbs", "shootout/shootout.qbs", + "subdirfileiterator/subdirfileiterator.qbs", "tasktree/tasktree.qbs", "widgets/widgets.qbs", ] diff --git a/tests/manual/subdirfileiterator/CMakeLists.txt b/tests/manual/subdirfileiterator/CMakeLists.txt new file mode 100644 index 00000000000..9613e197b77 --- /dev/null +++ b/tests/manual/subdirfileiterator/CMakeLists.txt @@ -0,0 +1,10 @@ +file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}") + +add_qtc_test(tst_manual_subdirfileiterator + MANUALTEST + DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\"" + DEPENDS Utils app_version + SOURCES + tst_subdirfileiterator.cpp +) diff --git a/tests/manual/subdirfileiterator/subdirfileiterator.qbs b/tests/manual/subdirfileiterator/subdirfileiterator.qbs new file mode 100644 index 00000000000..852fb9b545b --- /dev/null +++ b/tests/manual/subdirfileiterator/subdirfileiterator.qbs @@ -0,0 +1,22 @@ +import qbs.FileInfo + +QtcManualtest { + name: "Manual SubDirFileIterator test" + type: ["application"] + + Depends { name: "Utils" } + Depends { name: "app_version_header" } + + files: [ + "tst_subdirfileiterator.cpp", + ] + + cpp.defines: { + var defines = base; + var absLibExecPath = FileInfo.joinPaths(qbs.installRoot, qbs.installPrefix, + qtc.ide_libexec_path); + var relLibExecPath = FileInfo.relativePath(destinationDirectory, absLibExecPath); + defines.push('TEST_RELATIVE_LIBEXEC_PATH="' + relLibExecPath + '"'); + return defines; + } +} diff --git a/tests/manual/subdirfileiterator/tst_subdirfileiterator.cpp b/tests/manual/subdirfileiterator/tst_subdirfileiterator.cpp new file mode 100644 index 00000000000..4093030a719 --- /dev/null +++ b/tests/manual/subdirfileiterator/tst_subdirfileiterator.cpp @@ -0,0 +1,257 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +using namespace Tasking; +using namespace Utils; + +static const int s_subDirsCount = 8; +static const int s_subFilesCount = 8; +static const int s_treeDepth = 4; + +static const QDir::Filters s_filters = QDir::Dirs | QDir::Files | QDir::Hidden + | QDir::NoDotAndDotDot; + +static const char s_dirPrefix[] = "dir_"; +static const char s_filePrefix[] = "file_"; + +static int expectedDirsCountHelper(int depth) +{ + if (depth == 1) + return s_subDirsCount + 1; // +1 -> dir itself + return expectedDirsCountHelper(depth - 1) * s_subDirsCount + 1; // +1 -> dir itself +} + +static int expectedDirsCount(int tasksCount) +{ + return expectedDirsCountHelper(s_treeDepth) * tasksCount + 1; // +1 -> root temp dir +} + +static int expectedFilesCountHelper(int depth) +{ + if (depth == 0) + return s_subFilesCount; + return expectedFilesCountHelper(depth - 1) * s_subDirsCount + s_subFilesCount; +} + +static int expectedFilesCount(int tasksCount) +{ + return expectedFilesCountHelper(s_treeDepth) * tasksCount + 1; // +1 -> fileTemplate.txt +} + +static void generate(QPromise &promise, const QString &parentPath, + const QString &templateFile, int depth) +{ + const QDir parentDir(parentPath); + for (int i = 0; i < s_subFilesCount; ++i) + QFile::copy(templateFile, parentDir.filePath(QString("%1%2").arg(s_filePrefix).arg(i))); + + if (depth == 0) + return; + + if (promise.isCanceled()) + return; + + const QString templateDir("dirTemplate"); + if (!parentDir.mkdir(templateDir)) { + promise.future().cancel(); + return; + } + + const QString templateDirPath = parentDir.filePath(templateDir); + generate(promise, templateDirPath, templateFile, depth - 1); + + if (promise.isCanceled()) + return; + + for (int i = 0; i < s_subDirsCount - 1; ++i) { + const QString dirName = QString("%1%2").arg(s_dirPrefix).arg(i); + const FilePath source = FilePath::fromString(templateDirPath); + const FilePath destination = FilePath::fromString(parentDir.filePath(dirName)); + if (!source.copyRecursively(destination)) { + promise.future().cancel(); + return; + } + } +} + +class tst_SubDirFileIterator : public QObject +{ + Q_OBJECT + +private slots: + void initTestCase() + { + TemporaryDirectory::setMasterTemporaryDirectory( + QDir::tempPath() + "/" + Core::Constants::IDE_CASED_ID + "-XXXXXX"); + + const QString libExecPath(qApp->applicationDirPath() + '/' + + QLatin1String(TEST_RELATIVE_LIBEXEC_PATH)); + LauncherInterface::setPathToLauncher(libExecPath); + + qDebug() << "This manual test compares the performance of the SubDirFileIterator with " + "a manually written iterator using QDir::entryInfoList()."; + QTC_SCOPED_TIMER("GENERATING TEMPORARY FILES TREE"); + const int tasksCount = QThread::idealThreadCount(); + m_filesCount = expectedFilesCount(tasksCount); + m_dirsCount = expectedDirsCount(tasksCount); + m_tempDir.reset(new QTemporaryDir); + qDebug() << "Generating on" << tasksCount << "cores..."; + qDebug() << "Generating" << m_filesCount << "files..."; + qDebug() << "Generating" << m_dirsCount << "dirs..."; + qDebug() << "Generating inside" << m_tempDir->path() << "dir..."; + const QString templateFile = m_tempDir->filePath("fileTemplate.txt"); + { + QFile file(templateFile); + QVERIFY(file.open(QIODevice::ReadWrite)); + file.write("X"); + } + + // Parallelize tree generation + const QDir parentDir(m_tempDir->path()); + const auto onSetup = [](const QString &parentPath, const QString &templateFile) { + return [parentPath, templateFile](Async &async) { + async.setConcurrentCallData(generate, parentPath, templateFile, s_treeDepth); + }; + }; + QList tasks {parallel}; + for (int i = 0; i < tasksCount; ++i) { + const QString dirName = QString("%1%2").arg(s_dirPrefix).arg(i); + QVERIFY(parentDir.mkdir(dirName)); + tasks.append(AsyncTask(onSetup(parentDir.filePath(dirName), templateFile))); + } + + TaskTree taskTree(tasks); + QVERIFY(taskTree.runBlocking()); + } + + void cleanupTestCase() + { + QTC_SCOPED_TIMER("CLEANING UP"); + + // Parallelize tree removal + const auto removeTree = [](const QString &parentPath) { + FilePath::fromString(parentPath).removeRecursively(); + }; + + const QDir parentDir(m_tempDir->path()); + const auto onSetup = [removeTree](const QString &parentPath) { + return [parentPath, removeTree](Async &async) { + async.setConcurrentCallData(removeTree, parentPath); + }; + }; + QList tasks {parallel}; + const int tasksCount = QThread::idealThreadCount(); + for (int i = 0; i < tasksCount; ++i) { + const QString dirName = QString("%1%2").arg(s_dirPrefix).arg(i); + tasks.append(AsyncTask(onSetup(parentDir.filePath(dirName)))); + } + + TaskTree taskTree(tasks); + QVERIFY(taskTree.runBlocking()); + + m_tempDir.reset(); + Singleton::deleteAll(); + } + + void testSubDirFileIterator() + { + QTC_SCOPED_TIMER("ITERATING with SubDirFileIterator"); + int filesCount = 0; + { + const FilePath root(FilePath::fromString(m_tempDir->path())); + SubDirFileIterator it({root}, {}, {}); + auto i = it.begin(); + const auto e = it.end(); + while (i != e) { + ++filesCount; + ++i; + if (filesCount % 100000 == 0) + qDebug() << filesCount << '/' << m_filesCount << "files visited so far..."; + } + } + qDebug() << "Visited" << filesCount << "files."; + } + + void testManualIterator() + { + QTC_SCOPED_TIMER("ITERATING with manual iterator"); + int filesCount = 0; + int dirsCount = 0; + { + const QDir root(m_tempDir->path()); + std::unordered_set visitedFiles; + std::unordered_set visitedDirs; + visitedDirs.emplace(root.absolutePath()); + QFileInfoList workingList = root.entryInfoList(s_filters); + ++dirsCount; // for root itself + while (!workingList.isEmpty()) { + const QFileInfo fi = workingList.takeLast(); + const QString absoluteFilePath = fi.absoluteFilePath(); + if (fi.isDir()) { + if (!visitedDirs.emplace(absoluteFilePath).second) + continue; + if (fi.isSymbolicLink() && !visitedDirs.emplace(fi.symLinkTarget()).second) + continue; + ++dirsCount; + workingList.append(QDir(absoluteFilePath).entryInfoList(s_filters)); + } else { + if (!visitedFiles.emplace(absoluteFilePath).second) + continue; + if (fi.isSymbolicLink() && !visitedFiles.emplace(fi.symLinkTarget()).second) + continue; + ++filesCount; + if (filesCount % 100000 == 0) + qDebug() << filesCount << '/' << m_filesCount << "files visited so far..."; + } + } + } + qDebug() << "Visited" << filesCount << "files and" << dirsCount << "directories."; + } + + void testQDirIterator() + { + QTC_SCOPED_TIMER("ITERATING with QDirIterator"); + int filesCount = 0; + int dirsCount = 0; + { + ++dirsCount; // for root itself + QDirIterator it(m_tempDir->path(), s_filters, QDirIterator::Subdirectories + | QDirIterator::FollowSymlinks); + while (it.hasNext()) { + const QFileInfo fi = it.nextFileInfo(); + if (fi.isDir()) { + ++dirsCount; + } else { + ++filesCount; + if (filesCount % 100000 == 0) + qDebug() << filesCount << '/' << m_filesCount << "files visited so far..."; + } + } + } + qDebug() << "Visited" << filesCount << "files and" << dirsCount << "directories."; + } + +private: + int m_dirsCount = 0; + int m_filesCount = 0; + std::unique_ptr m_tempDir; +}; + +QTEST_GUILESS_MAIN(tst_SubDirFileIterator) + +#include "tst_subdirfileiterator.moc"