feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

FileSearch: Remove FileIterator and subclasses

Browse Source 
Rename tst_SubDirFileIterator into tst_SubDirFileContainer.

Change-Id: I0907ff93f1d6537d200fdc9f5783cfd2a8eb0aa9
Reviewed-by: <github-actions-qt-creator@cristianadam.eu>
Reviewed-by: Eike Ziller <eike.ziller@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
This commit is contained in:
Jarek Kobus
2023-05-24 17:40:40 +02:00
parent 2f7ecdaf06
commit 80326d8743
8 changed files with 25 additions and 465 deletions
Download Patch File Download Diff File Expand all files Collapse all files

313
src/libs/utils/filesearch.cpp
View File

@@ -5,10 +5,7 @@
#include "algorithm.h"
#include "async.h"
#include "filepath.h"
#include "mapreduce.h"
#include "qtcassert.h"
#include "searchresultitem.h"
#include "stringutils.h"
#include "utilstr.h"
@@ -17,11 +14,9 @@
#include <QScopeGuard>
#include <QTextCodec>
#include <cctype>
Q_LOGGING_CATEGORY(searchLog, "qtc.utils.filesearch", QtWarningMsg)
using namespace Utils;
namespace Utils {
const int MAX_LINE_SIZE = 400;
@@ -32,7 +27,7 @@ static QString clippedText(const QString &text, int maxLength)
return text;
}
QTextDocument::FindFlags Utils::textDocumentFlagsForFindFlags(FindFlags flags)
QTextDocument::FindFlags textDocumentFlagsForFindFlags(FindFlags flags)
{
QTextDocument::FindFlags textDocFlags;
if (flags & FindBackward)
@@ -195,7 +190,7 @@ static SearchResultItems searchInContents(const QFuture<void> &future, const QSt
return searchWithoutRegExp(future, searchTerm, flags, filePath, contents);
}
void Utils::searchInContents(QPromise<SearchResultItems> &promise, const QString &searchTerm,
void searchInContents(QPromise<SearchResultItems> &promise, const QString &searchTerm,
FindFlags flags, const FilePath &filePath, const QString &contents)
{
const QFuture<void> future(promise.future());
@@ -217,12 +212,8 @@ static inline QString msgFound(const QString &searchTerm, int numMatches, int nu
nullptr, numMatches).arg(searchTerm).arg(numFilesSearched);
}
namespace {
static bool getFileContent(const FilePath &filePath,
QTextCodec *encoding,
QString *tempString,
const QMap<FilePath, QString> &fileToContentsMap)
static bool getFileContent(const FilePath &filePath, QTextCodec *encoding,
QString *tempString, const QMap<FilePath, QString> &fileToContentsMap)
{
if (fileToContentsMap.contains(filePath)) {
*tempString = fileToContentsMap.value(filePath);
@@ -236,103 +227,6 @@ static bool getFileContent(const FilePath &filePath,
return true;
}
class FileSearch
{
public:
void operator()(QFutureInterface<SearchResultItems> &futureInterface,
const FileIterator::Item &item) const;
const QString m_searchTerm;
const FindFlags m_flags;
const QMap<FilePath, QString> m_fileToContentsMap;
};
void FileSearch::operator()(QFutureInterface<SearchResultItems> &futureInterface,
const FileIterator::Item &item) const
{
if (futureInterface.isCanceled())
return;
qCDebug(searchLog) << "Searching in" << item.filePath;
futureInterface.setProgressRange(0, 1);
futureInterface.setProgressValue(0);
QString contents;
if (!getFileContent(item.filePath, item.encoding, &contents, m_fileToContentsMap)) {
qCDebug(searchLog) << "- failed to get content for" << item.filePath;
futureInterface.cancel(); // failure
return;
}
const QFuture<void> future(futureInterface.future());
const SearchResultItems results = searchInContents(future, m_searchTerm, m_flags, item.filePath,
contents);
if (!futureInterface.isCanceled()) {
futureInterface.reportResult(results);
futureInterface.setProgressValue(1);
}
qCDebug(searchLog) << "- finished searching in" << item.filePath;
}
struct SearchState
{
SearchState(const QString &term, FileIterator *iterator) : searchTerm(term), files(iterator) {}
QString searchTerm;
FileIterator *files = nullptr;
SearchResultItems cachedResults;
int numFilesSearched = 0;
int numMatches = 0;
};
SearchState initFileSearch(QFutureInterface<SearchResultItems> &futureInterface,
const QString &searchTerm, FileIterator *files)
{
futureInterface.setProgressRange(0, files->maxProgress());
futureInterface.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, 0, 0));
return SearchState(searchTerm, files);
}
void collectSearchResults(QFutureInterface<SearchResultItems> &futureInterface,
SearchState &state,
const SearchResultItems &results)
{
state.numMatches += results.size();
state.cachedResults << results;
state.numFilesSearched += 1;
if (futureInterface.isProgressUpdateNeeded()
|| futureInterface.progressValue() == 0 /*workaround for regression in Qt*/) {
if (!state.cachedResults.isEmpty()) {
futureInterface.reportResult(state.cachedResults);
state.cachedResults.clear();
}
futureInterface.setProgressRange(0, state.files->maxProgress());
futureInterface.setProgressValueAndText(state.files->currentProgress(),
msgFound(state.searchTerm,
state.numMatches,
state.numFilesSearched));
}
}
void cleanUpFileSearch(QFutureInterface<SearchResultItems> &futureInterface,
SearchState &state)
{
if (!state.cachedResults.isEmpty()) {
futureInterface.reportResult(state.cachedResults);
state.cachedResults.clear();
}
if (futureInterface.isCanceled()) {
futureInterface.setProgressValueAndText(state.files->currentProgress(),
msgCanceled(state.searchTerm,
state.numMatches,
state.numFilesSearched));
} else {
futureInterface.setProgressValueAndText(state.files->currentProgress(),
msgFound(state.searchTerm,
state.numMatches,
state.numFilesSearched));
}
delete state.files;
}
} // namespace
static void fileSearch(QPromise<SearchResultItems> &promise,
const FileContainerIterator::Item &item, const QString &searchTerm,
FindFlags flags, const QMap<FilePath, QString> &fileToContentsMap)
@@ -452,26 +346,14 @@ static void findInFilesImpl(QPromise<SearchResultItems> &promise, const QString
loop.exec(QEventLoop::ExcludeUserInputEvents);
}
QFuture<SearchResultItems> Utils::findInFiles(const QString &searchTerm,
const FileContainer &container, FindFlags flags,
QFuture<SearchResultItems> findInFiles(const QString &searchTerm, const FileContainer &container,
FindFlags flags,
const QMap<FilePath, QString> &fileToContentsMap)
{
return Utils::asyncRun(findInFilesImpl, searchTerm, container, flags, fileToContentsMap);
}
QFuture<SearchResultItems> Utils::findInFiles(const QString &searchTerm, FileIterator *files,
FindFlags flags, const QMap<FilePath, QString> &fileToContentsMap)
{
return mapReduce(files->begin(), files->end(),
[searchTerm, files](QFutureInterface<SearchResultItems> &futureInterface) {
return initFileSearch(futureInterface, searchTerm, files);
},
FileSearch{searchTerm, flags, fileToContentsMap},
&collectSearchResults,
&cleanUpFileSearch);
}
QString Utils::expandRegExpReplacement(const QString &replaceText, const QStringList &capturedTexts)
QString expandRegExpReplacement(const QString &replaceText, const QStringList &capturedTexts)
{
// handles \1 \\ \& \t \n $1 $$ $&
QString result;
@@ -520,9 +402,7 @@ QString Utils::expandRegExpReplacement(const QString &replaceText, const QString
return result;
}
namespace Utils {
namespace Internal {
QString matchCaseReplacement(const QString &originalText, const QString &replaceText)
static QString matchCaseReplacementHelper(const QString &originalText, const QString &replaceText)
{
if (originalText.isEmpty() || replaceText.isEmpty())
return replaceText;
@@ -557,7 +437,6 @@ QString matchCaseReplacement(const QString &originalText, const QString &replace
return replaceText; // mixed
}
}
} // namespace
static QList<QRegularExpression> filtersToRegExps(const QStringList &filters)
{
@@ -604,7 +483,6 @@ QStringList splitFilterUiText(const QString &text)
return Utils::filtered(trimmedPortableParts, [](const QString &s) { return !s.isEmpty(); });
}
QString msgFilePatternLabel()
{
return Tr::tr("Fi&le pattern:");
@@ -644,39 +522,11 @@ QString matchCaseReplacement(const QString &originalText, const QString &replace
//keep prefix and suffix, and do actual replacement on the 'middle' of the string
return originalText.left(prefixLen)
+ Internal::matchCaseReplacement(originalText.mid(prefixLen, originalTextLen - prefixLen - suffixLen),
+ matchCaseReplacementHelper(originalText.mid(prefixLen, originalTextLen - prefixLen - suffixLen),
replaceText.mid(prefixLen, replaceTextLen - prefixLen - suffixLen))
+ originalText.right(suffixLen);
}
// #pragma mark -- FileIterator
void FileIterator::advance(FileIterator::const_iterator *it) const
{
if (it->m_index < 0) // == end
return;
++it->m_index;
const_cast<FileIterator *>(this)->update(it->m_index);
if (it->m_index >= currentFileCount())
it->m_index = -1; // == end
}
FileIterator::const_iterator FileIterator::begin() const
{
const_cast<FileIterator *>(this)->update(0);
if (currentFileCount() == 0)
return end();
return FileIterator::const_iterator(this, 0/*index*/);
}
FileIterator::const_iterator FileIterator::end() const
{
return FileIterator::const_iterator(this, -1/*end*/);
}
// #pragma mark -- FileListIterator
void FileContainerIterator::operator++()
{
QTC_ASSERT(m_data.m_container, return);
@@ -867,145 +717,4 @@ SubDirFileContainer::SubDirFileContainer(const FilePaths &directories, const QSt
: FileContainer(subDirAdvancerProvider(directories, filters, exclusionFilters, encoding),
s_progressMaximum) {}
QList<FileIterator::Item> constructItems(const FilePaths &fileList,
const QList<QTextCodec *> &encodings)
{
QList<FileIterator::Item> items;
items.reserve(fileList.size());
QTextCodec *defaultEncoding = QTextCodec::codecForLocale();
for (int i = 0; i < fileList.size(); ++i)
items.append(FileIterator::Item(fileList.at(i), encodings.value(i, defaultEncoding)));
return items;
}
FileListIterator::FileListIterator(const FilePaths &fileList, const QList<QTextCodec *> &encodings)
: m_items(constructItems(fileList, encodings))
{
}
void FileListIterator::update(int requestedIndex)
{
if (requestedIndex > m_maxIndex)
m_maxIndex = requestedIndex;
}
int FileListIterator::currentFileCount() const
{
return m_items.size();
}
const FileIterator::Item &FileListIterator::itemAt(int index) const
{
return m_items.at(index);
}
int FileListIterator::maxProgress() const
{
return m_items.size();
}
int FileListIterator::currentProgress() const
{
return m_maxIndex + 1;
}
// #pragma mark -- SubDirFileIterator
SubDirFileIterator::SubDirFileIterator(const FilePaths &directories,
const QStringList &filters,
const QStringList &exclusionFilters,
QTextCodec *encoding)
: m_filterFiles(filterFilesFunction(filters, exclusionFilters))
, m_progress(0)
{
m_encoding = (encoding == nullptr ? QTextCodec::codecForLocale() : encoding);
qreal maxPer = qreal(s_progressMaximum) / directories.count();
for (const FilePath &directoryEntry : directories) {
if (!directoryEntry.isEmpty()) {
const FilePath canonicalPath = directoryEntry.canonicalPath();
if (!canonicalPath.isEmpty() && directoryEntry.exists()) {
m_dirs.push(directoryEntry);
m_knownDirs.insert(canonicalPath);
m_progressValues.push(maxPer);
m_processedValues.push(false);
}
}
}
}
SubDirFileIterator::~SubDirFileIterator()
{
qDeleteAll(m_items);
}
void SubDirFileIterator::update(int index)
{
if (index < m_items.size())
return;
// collect files from the directories until we have enough for the given index
while (!m_dirs.isEmpty() && index >= m_items.size()) {
FilePath dir = m_dirs.pop();
const qreal dirProgressMax = m_progressValues.pop();
const bool processed = m_processedValues.pop();
if (dir.exists()) {
using Dir = FilePath;
using CanonicalDir = FilePath;
std::vector<std::pair<Dir, CanonicalDir>> subDirs;
if (!processed) {
for (const FilePath &entry :
dir.dirEntries(QDir::Dirs | QDir::Hidden | QDir::NoDotAndDotDot)) {
const FilePath canonicalDir = entry.canonicalPath();
if (!m_knownDirs.contains(canonicalDir))
subDirs.emplace_back(entry, canonicalDir);
}
}
if (subDirs.empty()) {
const FilePaths allFilePaths = dir.dirEntries(QDir::Files | QDir::Hidden);
const FilePaths filePaths = m_filterFiles(allFilePaths);
m_items.reserve(m_items.size() + filePaths.size());
Utils::reverseForeach(filePaths, [this](const FilePath &file) {
m_items.append(new Item(file, m_encoding));
});
m_progress += dirProgressMax;
} else {
qreal subProgress = dirProgressMax/(subDirs.size()+1);
m_dirs.push(dir);
m_progressValues.push(subProgress);
m_processedValues.push(true);
Utils::reverseForeach(subDirs,
[this, subProgress](const std::pair<Dir, CanonicalDir> &dir) {
m_dirs.push(dir.first);
m_knownDirs.insert(dir.second);
m_progressValues.push(subProgress);
m_processedValues.push(false);
});
}
} else {
m_progress += dirProgressMax;
}
}
if (index >= m_items.size())
m_progress = s_progressMaximum;
}
int SubDirFileIterator::currentFileCount() const
{
return m_items.size();
}
const FileIterator::Item &SubDirFileIterator::itemAt(int index) const
{
return *m_items.at(index);
}
int SubDirFileIterator::maxProgress() const
{
return s_progressMaximum;
}
int SubDirFileIterator::currentProgress() const
{
return qMin(qRound(m_progress), s_progressMaximum);
}
}
} // namespace Utils

114
src/libs/utils/filesearch.h
View File

@@ -10,8 +10,6 @@
#include <QMap>
#include <QPromise>
#include <QSet>
#include <QStack>
#include <QTextDocument>
#include <functional>
@@ -158,121 +156,9 @@ public:
QTextCodec *encoding = nullptr);
};
class QTCREATOR_UTILS_EXPORT FileIterator
{
public:
class Item
{
public:
Item() = default;
Item(const FilePath &path, QTextCodec *codec)
: filePath(path)
, encoding(codec)
{}
FilePath filePath;
QTextCodec *encoding = nullptr;
};
using value_type = Item;
class const_iterator
{
public:
using iterator_category = std::forward_iterator_tag;
using value_type = Item;
using difference_type = std::ptrdiff_t;
using pointer = const value_type*;
using reference = const value_type&;
const_iterator() = default;
const_iterator(const FileIterator *parent, int id)
: m_parent(parent), m_index(id)
{}
const_iterator(const const_iterator &) = default;
const_iterator &operator=(const const_iterator &) = default;
reference operator*() const { return m_parent->itemAt(m_index); }
pointer operator->() const { return &m_parent->itemAt(m_index); }
void operator++() { m_parent->advance(this); }
bool operator==(const const_iterator &other) const
{
return m_parent == other.m_parent && m_index == other.m_index;
}
bool operator!=(const const_iterator &other) const { return !operator==(other); }
const FileIterator *m_parent = nullptr;
int m_index = -1; // -1 == end
};
virtual ~FileIterator() = default;
const_iterator begin() const;
const_iterator end() const;
virtual int maxProgress() const = 0;
virtual int currentProgress() const = 0;
void advance(const_iterator *it) const;
virtual const Item &itemAt(int index) const = 0;
protected:
virtual void update(int requestedIndex) = 0;
virtual int currentFileCount() const = 0;
};
class QTCREATOR_UTILS_EXPORT FileListIterator : public FileIterator
{
public:
explicit FileListIterator(const FilePaths &fileList = {},
const QList<QTextCodec *> &encodings = {});
int maxProgress() const override;
int currentProgress() const override;
protected:
void update(int requestedIndex) override;
int currentFileCount() const override;
const Item &itemAt(int index) const override;
private:
const QList<Item> m_items;
int m_maxIndex = -1;
};
class QTCREATOR_UTILS_EXPORT SubDirFileIterator : public FileIterator
{
public:
SubDirFileIterator(const FilePaths &directories,
const QStringList &filters,
const QStringList &exclusionFilters,
QTextCodec *encoding = nullptr);
~SubDirFileIterator() override;
int maxProgress() const override;
int currentProgress() const override;
protected:
void update(int requestedIndex) override;
int currentFileCount() const override;
const Item &itemAt(int index) const override;
private:
std::function<FilePaths(const FilePaths &)> m_filterFiles;
QTextCodec *m_encoding;
QStack<FilePath> m_dirs;
QSet<FilePath> m_knownDirs;
QStack<qreal> m_progressValues;
QStack<bool> m_processedValues;
qreal m_progress;
// Use heap allocated objects directly because we want references to stay valid even after resize
QList<Item *> m_items;
};
QTCREATOR_UTILS_EXPORT QFuture<SearchResultItems> findInFiles(const QString &searchTerm,
const FileContainer &container, FindFlags flags, const QMap<FilePath, QString> &fileToContentsMap);
QTCREATOR_UTILS_EXPORT QFuture<SearchResultItems> findInFiles(const QString &searchTerm,
FileIterator *files, FindFlags flags, const QMap<FilePath, QString> &fileToContentsMap);
QTCREATOR_UTILS_EXPORT QString expandRegExpReplacement(const QString &replaceText,
const QStringList &capturedTexts);
QTCREATOR_UTILS_EXPORT QString matchCaseReplacement(const QString &originalText,

16
tests/auto/filesearch/tst_filesearch.cpp
View File

@@ -37,21 +37,6 @@ SearchResultItem searchResult(const FilePath &fileName, const QString &matchingL
void test_helper(const FilePath &filePath, const SearchResultItems &expectedResults,
const QString &term, Utils::FindFlags flags = {})
{
{
FileIterator *it = new FileListIterator({filePath}, {QTextCodec::codecForLocale()});
QFutureWatcher<SearchResultItems> watcher;
QSignalSpy ready(&watcher, &QFutureWatcherBase::resultsReadyAt);
watcher.setFuture(Utils::findInFiles(term, it, flags, {}));
watcher.future().waitForFinished();
QTest::qWait(100); // process events
QCOMPARE(ready.count(), 1);
SearchResultItems results = watcher.resultAt(0);
QCOMPARE(results.count(), expectedResults.count());
for (int i = 0; i < expectedResults.size(); ++i)
QCOMPARE(results.at(i), expectedResults.at(i));
}
{
const FileListContainer container({filePath}, {QTextCodec::codecForLocale()});
QFutureWatcher<SearchResultItems> watcher;
QSignalSpy ready(&watcher, &QFutureWatcherBase::resultsReadyAt);
@@ -63,7 +48,6 @@ void test_helper(const FilePath &filePath, const SearchResultItems &expectedResu
QCOMPARE(results.count(), expectedResults.count());
for (int i = 0; i < expectedResults.size(); ++i)
QCOMPARE(results.at(i), expectedResults.at(i));
}
}
void tst_FileSearch::multipleResults()

2
tests/manual/CMakeLists.txt
View File

@@ -17,6 +17,6 @@ add_subdirectory(proparser)
# add_subdirectory(search)
add_subdirectory(shootout)
add_subdirectory(spinner)
add_subdirectory(subdirfileiterator)
add_subdirectory(subdirfilecontainer)
add_subdirectory(tasking)
add_subdirectory(widgets)

2
tests/manual/manual.qbs
View File

@@ -14,7 +14,7 @@ Project {
"proparser/testreader.qbs",
"shootout/shootout.qbs",
"spinner/spinner.qbs",
"subdirfileiterator/subdirfileiterator.qbs",
"subdirfilecontainer/subdirfilecontainer.qbs",
"tasking/demo/demo.qbs",
"tasking/imagescaling/imagescaling.qbs",
"widgets/widgets.qbs",

4
tests/manual/subdirfileiterator/CMakeLists.txt → tests/manual/subdirfilecontainer/CMakeLists.txt
View File

@@ -1,10 +1,10 @@
file(RELATIVE_PATH RELATIVE_TEST_PATH "${PROJECT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
file(RELATIVE_PATH TEST_RELATIVE_LIBEXEC_PATH "/${RELATIVE_TEST_PATH}" "/${IDE_LIBEXEC_PATH}")
add_qtc_test(tst_manual_subdirfileiterator
add_qtc_test(tst_manual_subdirfilecontainer
MANUALTEST
DEFINES "TEST_RELATIVE_LIBEXEC_PATH=\"${TEST_RELATIVE_LIBEXEC_PATH}\""
DEPENDS Utils app_version
SOURCES
tst_subdirfileiterator.cpp
tst_subdirfilecontainer.cpp
)

4
tests/manual/subdirfileiterator/subdirfileiterator.qbs → tests/manual/subdirfilecontainer/subdirfilecontainer.qbs
View File

@@ -1,14 +1,14 @@
import qbs.FileInfo
QtcManualtest {
name: "Manual SubDirFileIterator test"
name: "Manual SubDirFileContainer test"
type: ["application"]
Depends { name: "Utils" }
Depends { name: "app_version_header" }
files: [
"tst_subdirfileiterator.cpp",
"tst_subdirfilecontainer.cpp",
]
cpp.defines: {

29
tests/manual/subdirfileiterator/tst_subdirfileiterator.cpp → tests/manual/subdirfilecontainer/tst_subdirfilecontainer.cpp
View File

@@ -104,7 +104,7 @@ static void generateCopy(QPromise<void> &promise, const QString &sourcePath,
promise.future().cancel();
}
class tst_SubDirFileIterator : public QObject
class tst_SubDirFileContainer : public QObject
{
Q_OBJECT
@@ -118,8 +118,8 @@ private slots:
+ QLatin1String(TEST_RELATIVE_LIBEXEC_PATH));
LauncherInterface::setPathToLauncher(libExecPath);
qDebug() << "This manual test compares the performance of the SubDirFileIterator with "
"a manually written iterator using QDir::entryInfoList().";
qDebug() << "This manual test compares the performance of the SubDirFileContainer with a "
"manually written iterator using QDir::entryInfoList() and with QDirIterator.";
QTC_SCOPED_TIMER("GENERATING TEMPORARY FILES TREE");
m_threadsCount = threadsCount();
m_filesCount = expectedFilesCount();
@@ -183,25 +183,6 @@ private slots:
Singleton::deleteAll();
}
void testSubDirFileIterator()
{
QTC_SCOPED_TIMER("ITERATING with SubDirFileIterator");
int filesCount = 0;
{
const FilePath root(FilePath::fromString(m_tempDir->path()));
SubDirFileIterator it({root}, {}, {});
auto i = it.begin();
const auto e = it.end();
while (i != e) {
++filesCount;
++i;
if (filesCount % 100000 == 0)
qDebug() << filesCount << '/' << m_filesCount << "files visited so far...";
}
}
QCOMPARE(filesCount, m_filesCount);
}
void testSubDirFileContainer()
{
QTC_SCOPED_TIMER("ITERATING with FileContainer");
@@ -292,6 +273,6 @@ private:
std::unique_ptr<QTemporaryDir> m_tempDir;
};
QTEST_GUILESS_MAIN(tst_SubDirFileIterator)
QTEST_GUILESS_MAIN(tst_SubDirFileContainer)
#include "tst_subdirfileiterator.moc"
#include "tst_subdirfilecontainer.moc"