From e8c0c7895c8aee93bebd58e516b912dad3ffb4ab Mon Sep 17 00:00:00 2001 From: con Date: Thu, 24 Jun 2010 15:44:46 +0200 Subject: [PATCH] Use iterator instead of collecting all the files to search beforehand. Especially for the file system filter, it's better not to block while collecting all the file names by first iterating over the file system. Now, the filters return an iterator, and the search thread takes a file from it, searches it, and takes the next file. This also unifies the file iterator for the custom locator filters and the find in files on file system search filter. Task-number: QTCREATORBUG-1690 --- src/libs/utils/filesearch.cpp | 170 ++++++++++++++++-- src/libs/utils/filesearch.h | 43 ++++- src/plugins/locator/directoryfilter.cpp | 76 ++------ .../projectexplorer/allprojectsfind.cpp | 4 +- src/plugins/projectexplorer/allprojectsfind.h | 2 +- .../projectexplorer/currentprojectfind.cpp | 4 +- .../projectexplorer/currentprojectfind.h | 2 +- src/plugins/texteditor/basefilefind.h | 2 +- src/plugins/texteditor/findincurrentfile.cpp | 4 +- src/plugins/texteditor/findincurrentfile.h | 2 +- src/plugins/texteditor/findinfiles.cpp | 14 +- src/plugins/texteditor/findinfiles.h | 2 +- 12 files changed, 227 insertions(+), 98 deletions(-) diff --git a/src/libs/utils/filesearch.cpp b/src/libs/utils/filesearch.cpp index 677a678959d..39632154774 100644 --- a/src/libs/utils/filesearch.cpp +++ b/src/libs/utils/filesearch.cpp @@ -70,11 +70,11 @@ namespace { void runFileSearch(QFutureInterface &future, QString searchTerm, - QStringList files, + FileIterator *files, QTextDocument::FindFlags flags, QMap fileToContentsMap) { - future.setProgressRange(0, files.size()); + future.setProgressRange(0, files->maxProgress()); int numFilesSearched = 0; int numMatches = 0; @@ -95,11 +95,13 @@ void runFileSearch(QFutureInterface &future, QFile file; QBuffer buffer; - foreach (const QString &s, files) { + while (files->hasNext()) { + const QString &s = files->next(); + future.setProgressRange(0, files->maxProgress()); if (future.isPaused()) future.waitForResume(); if (future.isCanceled()) { - future.setProgressValueAndText(numFilesSearched, msgCanceled(searchTerm, numMatches, numFilesSearched)); + future.setProgressValueAndText(files->currentProgress(), msgCanceled(searchTerm, numMatches, numFilesSearched)); break; } QIODevice *device; @@ -180,20 +182,22 @@ void runFileSearch(QFutureInterface &future, firstChunk = false; } ++numFilesSearched; - future.setProgressValueAndText(numFilesSearched, msgFound(searchTerm, numMatches, numFilesSearched, files.size())); + if (future.isProgressUpdateNeeded()) + future.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, numMatches, numFilesSearched)); device->close(); } if (!future.isCanceled()) - future.setProgressValueAndText(numFilesSearched, msgFound(searchTerm, numMatches, numFilesSearched)); + future.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, numMatches, numFilesSearched)); + delete files; } void runFileSearchRegExp(QFutureInterface &future, QString searchTerm, - QStringList files, + FileIterator *files, QTextDocument::FindFlags flags, QMap fileToContentsMap) { - future.setProgressRange(0, files.size()); + future.setProgressRange(0, files->maxProgress()); int numFilesSearched = 0; int numMatches = 0; if (flags & QTextDocument::FindWholeWords) @@ -204,11 +208,13 @@ void runFileSearchRegExp(QFutureInterface &future, QFile file; QString str; QTextStream stream; - foreach (const QString &s, files) { + while (files->hasNext()) { + const QString &s = files->next(); + future.setProgressRange(0, files->maxProgress()); if (future.isPaused()) future.waitForResume(); if (future.isCanceled()) { - future.setProgressValueAndText(numFilesSearched, msgCanceled(searchTerm, numMatches, numFilesSearched)); + future.setProgressValueAndText(files->currentProgress(), msgCanceled(searchTerm, numMatches, numFilesSearched)); break; } @@ -237,28 +243,30 @@ void runFileSearchRegExp(QFutureInterface &future, ++lineNr; } ++numFilesSearched; - future.setProgressValueAndText(numFilesSearched, msgFound(searchTerm, numMatches, numFilesSearched, files.size())); + if (future.isProgressUpdateNeeded()) + future.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, numMatches, numFilesSearched)); if (needsToCloseFile) file.close(); } if (!future.isCanceled()) - future.setProgressValueAndText(numFilesSearched, msgFound(searchTerm, numMatches, numFilesSearched)); + future.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, numMatches, numFilesSearched)); + delete files; } } // namespace -QFuture Utils::findInFiles(const QString &searchTerm, const QStringList &files, +QFuture Utils::findInFiles(const QString &searchTerm, FileIterator *files, QTextDocument::FindFlags flags, QMap fileToContentsMap) { - return QtConcurrent::run > + return QtConcurrent::run > (runFileSearch, searchTerm, files, flags, fileToContentsMap); } -QFuture Utils::findInFilesRegExp(const QString &searchTerm, const QStringList &files, +QFuture Utils::findInFilesRegExp(const QString &searchTerm, FileIterator *files, QTextDocument::FindFlags flags, QMap fileToContentsMap) { - return QtConcurrent::run > + return QtConcurrent::run > (runFileSearchRegExp, searchTerm, files, flags, fileToContentsMap); } @@ -294,3 +302,133 @@ QString Utils::expandRegExpReplacement(const QString &replaceText, const QString } return result; } + +// #pragma mark -- FileIterator + +FileIterator::FileIterator() + : m_list(QStringList()), + m_iterator(0), + m_index(0) +{ +} + +FileIterator::FileIterator(const QStringList &fileList) + : m_list(fileList), + m_iterator(new QStringListIterator(m_list)), + m_index(0) +{ +} + +FileIterator::~FileIterator() +{ + if (m_iterator) + delete m_iterator; +} + +bool FileIterator::hasNext() const +{ + Q_ASSERT(m_iterator); + return m_iterator->hasNext(); +} + +QString FileIterator::next() +{ + Q_ASSERT(m_iterator); + ++m_index; + return m_iterator->next(); +} + +int FileIterator::maxProgress() const +{ + return m_list.size(); +} + +int FileIterator::currentProgress() const +{ + return m_index; +} + +// #pragma mark -- SubDirFileIterator + +namespace { + const int MAX_PROGRESS = 360; +} + +SubDirFileIterator::SubDirFileIterator(const QStringList &directories, const QStringList &filters) + : m_filters(filters), m_progress(0) +{ + int maxPer = MAX_PROGRESS/directories.count(); + foreach (const QString &directoryEntry, directories) { + if (!directoryEntry.isEmpty()) { + m_dirs.push(QDir(directoryEntry)); + m_progressValues.push(maxPer); + m_processedValues.push(false); + } + } +} + +bool SubDirFileIterator::hasNext() const +{ + if (!m_currentFiles.isEmpty()) + return true; + while(!m_dirs.isEmpty() && m_currentFiles.isEmpty()) { + QDir dir = m_dirs.pop(); + int dirProgressMax = m_progressValues.pop(); + bool processed = m_processedValues.pop(); + if (dir.exists()) { + QStringList subDirs; + if (!processed) { + subDirs = dir.entryList(QDir::Dirs|QDir::Hidden|QDir::NoDotAndDotDot); + } + if (subDirs.isEmpty()) { + QStringList fileEntries = dir.entryList(m_filters, + QDir::Files|QDir::Hidden); + QStringListIterator it(fileEntries); + it.toBack(); + while (it.hasPrevious()) { + const QString &file = it.previous(); + m_currentFiles.append(dir.path()+ QLatin1Char('/') +file); + } + m_progress += dirProgressMax; + } else { + int subProgress = dirProgressMax/(subDirs.size()+1); + int selfProgress = subProgress + dirProgressMax%(subDirs.size()+1); + m_dirs.push(dir); + m_progressValues.push(selfProgress); + m_processedValues.push(true); + QStringListIterator it(subDirs); + it.toBack(); + while (it.hasPrevious()) { + const QString &directory = it.previous(); + m_dirs.push(QDir(dir.path()+ QLatin1Char('/') + directory)); + m_progressValues.push(subProgress); + m_processedValues.push(false); + } + } + } else { + m_progress += dirProgressMax; + } + } + if (m_currentFiles.isEmpty()) { + m_progress = MAX_PROGRESS; + return false; + } + + return true; +} + +QString SubDirFileIterator::next() +{ + Q_ASSERT(!m_currentFiles.isEmpty()); + return m_currentFiles.takeFirst(); +} + +int SubDirFileIterator::maxProgress() const +{ + return MAX_PROGRESS; +} + +int SubDirFileIterator::currentProgress() const +{ + return m_progress; +} diff --git a/src/libs/utils/filesearch.h b/src/libs/utils/filesearch.h index f2860e34e01..0b3f8953870 100644 --- a/src/libs/utils/filesearch.h +++ b/src/libs/utils/filesearch.h @@ -35,10 +35,49 @@ #include #include #include +#include +#include #include namespace Utils { +class QTCREATOR_UTILS_EXPORT FileIterator +{ +public: + FileIterator(); + explicit FileIterator(const QStringList &fileList); + ~FileIterator(); + + virtual bool hasNext() const; + virtual QString next(); + virtual int maxProgress() const; + virtual int currentProgress() const; + +private: + QStringList m_list; + QStringListIterator *m_iterator; + int m_index; +}; + +class QTCREATOR_UTILS_EXPORT SubDirFileIterator : public FileIterator +{ +public: + SubDirFileIterator(const QStringList &directories, const QStringList &filters); + + bool hasNext() const; + QString next(); + int maxProgress() const; + int currentProgress() const; + +private: + QStringList m_filters; + mutable QStack m_dirs; + mutable QStack m_progressValues; + mutable QStack m_processedValues; + mutable int m_progress; + mutable QStringList m_currentFiles; +}; + class QTCREATOR_UTILS_EXPORT FileSearchResult { public: @@ -62,10 +101,10 @@ public: QStringList regexpCapturedTexts; }; -QTCREATOR_UTILS_EXPORT QFuture findInFiles(const QString &searchTerm, const QStringList &files, +QTCREATOR_UTILS_EXPORT QFuture findInFiles(const QString &searchTerm, FileIterator *files, QTextDocument::FindFlags flags, QMap fileToContentsMap = QMap()); -QTCREATOR_UTILS_EXPORT QFuture findInFilesRegExp(const QString &searchTerm, const QStringList &files, +QTCREATOR_UTILS_EXPORT QFuture findInFilesRegExp(const QString &searchTerm, FileIterator *files, QTextDocument::FindFlags flags, QMap fileToContentsMap = QMap()); QTCREATOR_UTILS_EXPORT QString expandRegExpReplacement(const QString &replaceText, const QStringList &capturedTexts); diff --git a/src/plugins/locator/directoryfilter.cpp b/src/plugins/locator/directoryfilter.cpp index 7049d50c066..b41cd6c5939 100644 --- a/src/plugins/locator/directoryfilter.cpp +++ b/src/plugins/locator/directoryfilter.cpp @@ -36,6 +36,7 @@ #include #include +#include using namespace Locator; using namespace Locator::Internal; @@ -181,76 +182,35 @@ void DirectoryFilter::updateOptionButtons() void DirectoryFilter::refresh(QFutureInterface &future) { - const int MAX = 360; - future.setProgressRange(0, MAX); - if (m_directories.count() < 1) { + QStringList directories; + { QMutexLocker locker(&m_lock); - files().clear(); - generateFileNames(); - future.setProgressValueAndText(MAX, tr("%1 filter update: 0 files").arg(m_name)); - return; + if (m_directories.count() < 1) { + files().clear(); + generateFileNames(); + future.setProgressRange(0, 1); + future.setProgressValueAndText(1, tr("%1 filter update: 0 files").arg(m_name)); + return; + } + directories = m_directories; } - int progress = 0; - int MAX_PER = MAX; + Utils::SubDirFileIterator it(directories, m_filters); + future.setProgressRange(0, it.maxProgress()); QStringList filesFound; - QStack dirs; - QStack progressValues; - QStack processedValues; - { // initialize - QMutexLocker locker(&m_lock); - MAX_PER = MAX/m_directories.count(); - foreach (const QString &directoryEntry, m_directories) { - if (!directoryEntry.isEmpty()) { - dirs.push(QDir(directoryEntry)); - progressValues.push(MAX_PER); - processedValues.push(false); - } - } - } - while (!dirs.isEmpty() && !future.isCanceled()) { + while (!future.isCanceled() && it.hasNext()) { + filesFound << it.next(); if (future.isProgressUpdateNeeded()) { - future.setProgressValueAndText(progress, + future.setProgressValueAndText(it.currentProgress(), tr("%1 filter update: %n files", 0, filesFound.size()).arg(m_name)); } - QDir dir = dirs.pop(); - int dirProgressMax = progressValues.pop(); - bool processed = processedValues.pop(); - if (dir.exists()) { - QStringList subDirs; - if (!processed) { - subDirs = dir.entryList(QDir::Dirs|QDir::Hidden|QDir::NoDotAndDotDot, - QDir::Name|QDir::IgnoreCase|QDir::LocaleAware); - } - if (subDirs.isEmpty()) { - QStringList fileEntries = dir.entryList(m_filters, - QDir::Files|QDir::Hidden, - QDir::Name|QDir::IgnoreCase|QDir::LocaleAware); - foreach (const QString &file, fileEntries) - filesFound.append(dir.path()+ QLatin1Char('/') +file); - progress += dirProgressMax; - } else { - int subProgress = dirProgressMax/(subDirs.size()+1); - int selfProgress = subProgress + dirProgressMax%(subDirs.size()+1); - dirs.push(dir); - progressValues.push(selfProgress); - processedValues.push(true); - foreach (const QString &directory, subDirs) { - dirs.push(QDir(dir.path()+ QLatin1Char('/') + directory)); - progressValues.push(subProgress); - processedValues.push(false); - } - } - } else { - progress += dirProgressMax; - } } if (!future.isCanceled()) { QMutexLocker locker(&m_lock); files() = filesFound; generateFileNames(); - future.setProgressValue(MAX); + future.setProgressValue(it.maxProgress()); } else { - future.setProgressValueAndText(progress, tr("%1 filter update: canceled").arg(m_name)); + future.setProgressValueAndText(it.currentProgress(), tr("%1 filter update: canceled").arg(m_name)); } } diff --git a/src/plugins/projectexplorer/allprojectsfind.cpp b/src/plugins/projectexplorer/allprojectsfind.cpp index d621662c87d..abb1b90b23a 100644 --- a/src/plugins/projectexplorer/allprojectsfind.cpp +++ b/src/plugins/projectexplorer/allprojectsfind.cpp @@ -77,7 +77,7 @@ QKeySequence AllProjectsFind::defaultShortcut() const return QKeySequence(); } -QStringList AllProjectsFind::files() +Utils::FileIterator *AllProjectsFind::files() { Q_ASSERT(m_plugin->session()); QList filterRegs; @@ -103,7 +103,7 @@ QStringList AllProjectsFind::files() } } files.removeDuplicates(); - return files; + return new Utils::FileIterator(files); } QWidget *AllProjectsFind::createConfigWidget() diff --git a/src/plugins/projectexplorer/allprojectsfind.h b/src/plugins/projectexplorer/allprojectsfind.h index 613c63abd36..2f78757c156 100644 --- a/src/plugins/projectexplorer/allprojectsfind.h +++ b/src/plugins/projectexplorer/allprojectsfind.h @@ -61,7 +61,7 @@ public: void readSettings(QSettings *settings); protected: - QStringList files(); + Utils::FileIterator *files(); private: ProjectExplorerPlugin *m_plugin; diff --git a/src/plugins/projectexplorer/currentprojectfind.cpp b/src/plugins/projectexplorer/currentprojectfind.cpp index cae569bbdf4..7256a8603b3 100644 --- a/src/plugins/projectexplorer/currentprojectfind.cpp +++ b/src/plugins/projectexplorer/currentprojectfind.cpp @@ -75,7 +75,7 @@ QKeySequence CurrentProjectFind::defaultShortcut() const return QKeySequence(); } -QStringList CurrentProjectFind::files() +Utils::FileIterator *CurrentProjectFind::files() { Project *project = m_plugin->currentProject(); Q_ASSERT(project); @@ -98,7 +98,7 @@ QStringList CurrentProjectFind::files() files += project->files(Project::AllFiles); } files.removeDuplicates(); - return files; + return new Utils::FileIterator(files); } QWidget *CurrentProjectFind::createConfigWidget() diff --git a/src/plugins/projectexplorer/currentprojectfind.h b/src/plugins/projectexplorer/currentprojectfind.h index 53d57ff2646..2e98e49e1b1 100644 --- a/src/plugins/projectexplorer/currentprojectfind.h +++ b/src/plugins/projectexplorer/currentprojectfind.h @@ -64,7 +64,7 @@ public: void readSettings(QSettings *settings); protected: - QStringList files(); + Utils::FileIterator *files(); private: ProjectExplorerPlugin *m_plugin; diff --git a/src/plugins/texteditor/basefilefind.h b/src/plugins/texteditor/basefilefind.h index c0fdaa82f61..b423eb8b531 100644 --- a/src/plugins/texteditor/basefilefind.h +++ b/src/plugins/texteditor/basefilefind.h @@ -71,7 +71,7 @@ public: const QList &items); protected: - virtual QStringList files() = 0; + virtual Utils::FileIterator *files() = 0; void writeCommonSettings(QSettings *settings); void readCommonSettings(QSettings *settings, const QString &defaultFilter); QWidget *createPatternWidget(); diff --git a/src/plugins/texteditor/findincurrentfile.cpp b/src/plugins/texteditor/findincurrentfile.cpp index 22170f35e5c..736bc78ae4d 100644 --- a/src/plugins/texteditor/findincurrentfile.cpp +++ b/src/plugins/texteditor/findincurrentfile.cpp @@ -67,12 +67,12 @@ QKeySequence FindInCurrentFile::defaultShortcut() const return QKeySequence(); } -QStringList FindInCurrentFile::files() +Utils::FileIterator *FindInCurrentFile::files() { QStringList fileList; if (isEnabled()) fileList << m_currentFile->fileName(); - return fileList; + return new Utils::FileIterator(fileList); } bool FindInCurrentFile::isEnabled() const diff --git a/src/plugins/texteditor/findincurrentfile.h b/src/plugins/texteditor/findincurrentfile.h index 1f5ec7ba5d7..7c6f28c93a6 100644 --- a/src/plugins/texteditor/findincurrentfile.h +++ b/src/plugins/texteditor/findincurrentfile.h @@ -62,7 +62,7 @@ public: void readSettings(QSettings *settings); protected: - QStringList files(); + Utils::FileIterator *files(); private slots: void handleFileChange(Core::IEditor *editor); diff --git a/src/plugins/texteditor/findinfiles.cpp b/src/plugins/texteditor/findinfiles.cpp index e2c28b417f1..963cb0dea13 100644 --- a/src/plugins/texteditor/findinfiles.cpp +++ b/src/plugins/texteditor/findinfiles.cpp @@ -68,18 +68,10 @@ void FindInFiles::findAll(const QString &txt, QTextDocument::FindFlags findFlags BaseFileFind::findAll(txt, findFlags); } -QStringList FindInFiles::files() +Utils::FileIterator *FindInFiles::files() { - QStringList fileList; - QDirIterator it(m_directory->currentText(), - fileNameFilters(), - QDir::Files|QDir::Readable, - QDirIterator::Subdirectories); - - while (it.hasNext()) - fileList << it.next(); - - return fileList; + return new Utils::SubDirFileIterator(QStringList() << m_directory->currentText(), + fileNameFilters()); } QWidget *FindInFiles::createConfigWidget() diff --git a/src/plugins/texteditor/findinfiles.h b/src/plugins/texteditor/findinfiles.h index 91e4c484746..baf899439d1 100644 --- a/src/plugins/texteditor/findinfiles.h +++ b/src/plugins/texteditor/findinfiles.h @@ -60,7 +60,7 @@ public: void readSettings(QSettings *settings); protected: - QStringList files(); + Utils::FileIterator *files(); private slots: void openFileBrowser();