Use the encoding settings when doing multi-file searches.

Task-number: QTCREATORBUG-65

src/libs/utils/filesearch.cpp

@@ -82,20 +82,20 @@ void runFileSearch(QFutureInterface<FileSearchResultList> &future,
     bool caseInsensitive = !(flags & QTextDocument::FindCaseSensitively);
     bool wholeWord = (flags & QTextDocument::FindWholeWords);
 
-    QByteArray sa = searchTerm.toUtf8();
+    const QString searchTermLower = searchTerm.toLower();
-    int scMaxIndex = sa.length()-1;
+    const QString searchTermUpper = searchTerm.toUpper();
-    const char *sc = sa.constData();
 
-    QByteArray sal = searchTerm.toLower().toUtf8();
+    int termLength = searchTerm.length();
-    const char *scl = sal.constData();
+    int termMaxIndex = termLength - 1;
+    const QChar *termData = searchTerm.constData();
+    const QChar *termDataLower = searchTermLower.constData();
+    const QChar *termDataUpper = searchTermUpper.constData();
 
-    QByteArray sau = searchTerm.toUpper().toUtf8();
+    int chunkSize = qMax(100000, 2 * termLength);
-    const char *scu = sau.constData();
-
-    int chunkSize = qMax(100000, sa.length());
 
     QFile file;
-    QBuffer buffer;
+    QString str;
+    QTextStream stream;
     FileSearchResultList results;
     while (files->hasNext()) {
         const QString &s = files->next();
@@ -105,83 +105,104 @@ void runFileSearch(QFutureInterface<FileSearchResultList> &future,
             future.setProgressValueAndText(files->currentProgress(), msgCanceled(searchTerm, numMatches, numFilesSearched));
             break;
         }
-        QIODevice *device;
+
+        bool needsToCloseFile = false;
         if (fileToContentsMap.contains(s)) {
-            buffer.setData(fileToContentsMap.value(s).toLocal8Bit());
+            str = fileToContentsMap.value(s);
-            device = &buffer;
+            stream.setString(&str);
         } else {
             file.setFileName(s);
-            device = &file;
+            if (!file.open(QIODevice::ReadOnly))
+                continue;
+            needsToCloseFile = true;
+            stream.setDevice(&file);
+            stream.setCodec(files->encoding());
         }
-        if (!device->open(QIODevice::ReadOnly))
+
-            continue;
         int lineNr = 1;
-        const char *startOfLastLine = NULL;
+        const QChar *startOfLastLine = NULL;
-
         bool firstChunk = true;
-        while (!device->atEnd()) {
+        while (!stream.atEnd()) {
-            if (!firstChunk)
+            int chunkProcessingStart = 0;
-                device->seek(device->pos()-sa.length()+1);
+            if (!firstChunk) {
+                // we need one additional char to the left and right
+                // for whole word searches
+                // so we jump back two additional chars, and start at index 1
+                stream.seek(stream.pos() - termLength - 1);
+                chunkProcessingStart = 1;
+            }
+            firstChunk = false;
+
+            const QString chunk = stream.read(chunkSize);
+            int chunkLength = chunk.length();
+            const QChar *chunkPtr = chunk.constData();
+            // we need one additional char to the right for whole word searches,
+            // except at the very end
+            const QChar *chunkProcessingEnd = (stream.atEnd() ? chunkPtr + chunkLength : chunkPtr + chunkLength - 1);
 
-            const QByteArray chunk = device->read(chunkSize);
-            const char *chunkPtr = chunk.constData();
             startOfLastLine = chunkPtr;
-            for (const char *regionPtr = chunkPtr; regionPtr < chunkPtr + chunk.length()-scMaxIndex; ++regionPtr) {
+            for (const QChar *regionPtr = chunkPtr + chunkProcessingStart;
-                const char *regionEnd = regionPtr + scMaxIndex;
+                    regionPtr + termMaxIndex < chunkProcessingEnd;
-
+                    ++regionPtr) {
-                if (*regionPtr == '\n') {
+                const QChar *regionEnd = regionPtr + termMaxIndex;
+                if (*regionPtr == QLatin1Char('\n')) {
                     startOfLastLine = regionPtr + 1;
                     ++lineNr;
-                }
+                } else if ( /* optimization check for start and end of region */
-                else if (
                         // case sensitive
-                        (!caseInsensitive && *regionPtr == sc[0] && *regionEnd == sc[scMaxIndex])
+                        (!caseInsensitive && *regionPtr == termData[0] && *regionEnd == termData[termMaxIndex])
                         ||
                         // case insensitive
-                        (caseInsensitive && (*regionPtr == scl[0] || *regionPtr == scu[0])
+                        (caseInsensitive && (*regionPtr == termDataLower[0] || *regionPtr == termDataUpper[0])
-                        && (*regionEnd == scl[scMaxIndex] || *regionEnd == scu[scMaxIndex]))
+                        && (*regionEnd == termDataLower[termMaxIndex] || *regionEnd == termDataUpper[termMaxIndex]))
                          ) {
-                    const char *afterRegion = regionEnd + 1;
-                    const char *beforeRegion = regionPtr - 1;
                     bool equal = true;
-                    if (wholeWord &&
+
-                            (  isalnum((unsigned char)*beforeRegion)
+                    // whole word check
-                            || (*beforeRegion == '_')
+                    const QChar *beforeRegion = regionPtr - 1;
-                            || isalnum((unsigned char)*afterRegion)
+                    const QChar *afterRegion = regionEnd + 1;
-                            || (*afterRegion == '_'))) {
+                    if (wholeWord && (
+                            ((beforeRegion >= chunkPtr) && (beforeRegion->isLetterOrNumber() || ((*beforeRegion) == QLatin1Char('_')))) ||
+                            ((afterRegion < chunkPtr + chunkLength) && (afterRegion->isLetterOrNumber() || ((*afterRegion) == QLatin1Char('_'))))
+                            )) {
                         equal = false;
                     }
 
-                    int regionIndex = 1;
+                    if (equal) {
-                    for (const char *regionCursor = regionPtr + 1; regionCursor < regionEnd; ++regionCursor, ++regionIndex) {
+                        // check all chars
-                        if (  // case sensitive
+                        int regionIndex = 1;
-                              (!caseInsensitive && equal && *regionCursor != sc[regionIndex])
+                        for (const QChar *regionCursor = regionPtr + 1; regionCursor < regionEnd; ++regionCursor, ++regionIndex) {
-                              ||
+                            if (  // case sensitive
-                              // case insensitive
+                                  (!caseInsensitive && *regionCursor != termData[regionIndex])
-                              (caseInsensitive && equal && *regionCursor != sc[regionIndex] && *regionCursor != scl[regionIndex] && *regionCursor != scu[regionIndex])
+                                  ||
-                               ) {
+                                  // case insensitive
-                         equal = false;
+                                  (caseInsensitive && *regionCursor != termData[regionIndex]
+                                   && *regionCursor != termDataLower[regionIndex] && *regionCursor != termDataUpper[regionIndex])
+                                   ) {
+                                equal = false;
+                            }
                         }
                     }
                     if (equal) {
-                        int textLength = chunk.length() - (startOfLastLine - chunkPtr);
+                        int textLength = chunkLength - (startOfLastLine - chunkPtr);
                         if (textLength > 0) {
-                            QByteArray res;
+                            QString res;
                             res.reserve(256);
                             int i = 0;
                             int n = 0;
-                            while (startOfLastLine[i] != '\n' && startOfLastLine[i] != '\r' && i < textLength && n++ < 256)
+                            while (startOfLastLine[i] != QLatin1Char('\n') && startOfLastLine[i] != QLatin1Char('\r') && i < textLength && n++ < 256)
                                 res.append(startOfLastLine[i++]);
-                            results << FileSearchResult(s, lineNr, QString::fromUtf8(res),
+                            res.squeeze();
-                                                          regionPtr - startOfLastLine, sa.length(),
+                            results << FileSearchResult(s, lineNr, res,
+                                                          regionPtr - startOfLastLine, termLength,
                                                           QStringList());
                             ++numMatches;
                         }
                     }
                 }
             }
-            firstChunk = false;
         }
+
         ++numFilesSearched;
         if (future.isProgressUpdateNeeded()) {
             if (!results.isEmpty()) {
@@ -191,7 +212,11 @@ void runFileSearch(QFutureInterface<FileSearchResultList> &future,
             future.setProgressRange(0, files->maxProgress());
             future.setProgressValueAndText(files->currentProgress(), msgFound(searchTerm, numMatches, numFilesSearched));
         }
-        device->close();
+
+        // clean up
+        if (needsToCloseFile)
+            file.close();
+
     }
     if (!results.isEmpty()) {
         future.reportResult(results);
@@ -240,6 +265,7 @@ void runFileSearchRegExp(QFutureInterface<FileSearchResultList> &future,
                 continue;
             needsToCloseFile = true;
             stream.setDevice(&file);
+            stream.setCodec(files->encoding());
         }
         int lineNr = 1;
         QString line;
@@ -334,14 +360,16 @@ QString Utils::expandRegExpReplacement(const QString &replaceText, const QString
 FileIterator::FileIterator()
     : m_list(QStringList()),
     m_iterator(0),
-    m_index(0)
+    m_index(-1)
 {
 }
 
-FileIterator::FileIterator(const QStringList &fileList)
+FileIterator::FileIterator(const QStringList &fileList,
+                           const QList<QTextCodec *> encodings)
     : m_list(fileList),
-    m_iterator(new QStringListIterator(m_list)),
+      m_iterator(new QStringListIterator(m_list)),
-    m_index(0)
+      m_encodings(encodings),
+      m_index(-1)
 {
 }
 
@@ -371,7 +399,14 @@ int FileIterator::maxProgress() const
 
 int FileIterator::currentProgress() const
 {
-    return m_index;
+    return m_index + 1;
+}
+
+QTextCodec * FileIterator::encoding() const
+{
+    if (m_index >= 0 && m_index < m_encodings.size())
+        return m_encodings.at(m_index);
+    return QTextCodec::codecForLocale();
 }
 
 // #pragma mark -- SubDirFileIterator
@@ -380,9 +415,11 @@ namespace {
     const int MAX_PROGRESS = 1000;
 }
 
-SubDirFileIterator::SubDirFileIterator(const QStringList &directories, const QStringList &filters)
+SubDirFileIterator::SubDirFileIterator(const QStringList &directories, const QStringList &filters,
+                                       QTextCodec *encoding)
     : m_filters(filters), m_progress(0)
 {
+    m_encoding = (encoding == 0 ? QTextCodec::codecForLocale() : encoding);
     qreal maxPer = MAX_PROGRESS/directories.count();
     foreach (const QString &directoryEntry, directories) {
         if (!directoryEntry.isEmpty()) {
@@ -457,3 +494,8 @@ int SubDirFileIterator::currentProgress() const
 {
     return qMin(qRound(m_progress), MAX_PROGRESS);
 }
+
+QTextCodec * SubDirFileIterator::encoding() const
+{
+    return m_encoding;
+}

src/libs/utils/filesearch.h

@@ -37,6 +37,7 @@
 #include <QtCore/QMap>
 #include <QtCore/QStack>
 #include <QtCore/QDir>
+#include <QtCore/QTextCodec>
 #include <QtGui/QTextDocument>
 
 namespace Utils {
@@ -45,32 +46,38 @@ class QTCREATOR_UTILS_EXPORT FileIterator
 {
 public:
     FileIterator();
-    explicit FileIterator(const QStringList &fileList);
+    explicit FileIterator(const QStringList &fileList,
+                          const QList<QTextCodec *> encodings);
     ~FileIterator();
 
     virtual bool hasNext() const;
     virtual QString next();
+    virtual QTextCodec *encoding() const;
     virtual int maxProgress() const;
     virtual int currentProgress() const;
 
 private:
     QStringList m_list;
     QStringListIterator *m_iterator;
+    QList<QTextCodec *> m_encodings;
     int m_index;
 };
 
 class QTCREATOR_UTILS_EXPORT SubDirFileIterator : public FileIterator
 {
 public:
-    SubDirFileIterator(const QStringList &directories, const QStringList &filters);
+    SubDirFileIterator(const QStringList &directories, const QStringList &filters,
+                       QTextCodec *encoding = 0);
 
     bool hasNext() const;
     QString next();
+    QTextCodec *encoding() const;
     int maxProgress() const;
     int currentProgress() const;
 
 private:
     QStringList m_filters;
+    QTextCodec *m_encoding;
     mutable QStack<QDir> m_dirs;
     mutable QStack<qreal> m_progressValues;
     mutable QStack<bool> m_processedValues;

src/plugins/projectexplorer/allprojectsfind.cpp

@@ -32,8 +32,11 @@
 #include "project.h"
 #include "session.h"
 #include "projectexplorer.h"
+#include "editorconfiguration.h"
 
 #include <utils/qtcassert.h>
+#include <texteditor/itexteditor.h>
+#include <coreplugin/editormanager/editormanager.h>
 
 #include <QtCore/QDebug>
 #include <QtCore/QFileInfo>
@@ -85,25 +88,33 @@ Utils::FileIterator *AllProjectsFind::files() const
     foreach (const QString &filter, nameFilters) {
         filterRegs << QRegExp(filter, Qt::CaseInsensitive, QRegExp::Wildcard);
     }
-    QStringList files;
+    QMap<QString, QTextCodec *> openEditorEncodings = TextEditor::ITextEditor::openedTextEditorsEncodings();
-    QStringList projectFiles;
+    QMap<QString, QTextCodec *> encodings;
     foreach (const Project *project, projects()) {
-        projectFiles = project->files(Project::AllFiles);
+        QStringList projectFiles = project->files(Project::AllFiles);
+        QStringList filteredFiles;
         if (!filterRegs.isEmpty()) {
             foreach (const QString &file, projectFiles) {
                 foreach (const QRegExp &reg, filterRegs) {
                     if (reg.exactMatch(file) || reg.exactMatch(QFileInfo(file).fileName())) {
-                        files.append(file);
+                        filteredFiles.append(file);
                         break;
                     }
                 }
             }
         } else {
-            files += projectFiles;
+            filteredFiles = projectFiles;
+        }
+        foreach (const QString &fileName, filteredFiles) {
+            QTextCodec *codec = openEditorEncodings.value(fileName);
+            if (!codec)
+                codec = project->editorConfiguration()->defaultTextCodec();
+            if (!codec)
+                codec = Core::EditorManager::instance()->defaultTextEncoding();
+            encodings.insert(fileName, codec);
         }
     }
-    files.removeDuplicates();
+    return new Utils::FileIterator(encodings.keys(), encodings.values());
-    return new Utils::FileIterator(files);
 }
 
 QWidget *AllProjectsFind::createConfigWidget()

src/plugins/texteditor/findincurrentfile.cpp

@@ -28,6 +28,7 @@
 **************************************************************************/
 
 #include "findincurrentfile.h"
+#include "itexteditor.h"
 
 #include <coreplugin/icore.h>
 #include <coreplugin/editormanager/editormanager.h>
@@ -40,6 +41,7 @@
 #include <QtGui/QVBoxLayout>
 
 using namespace Find;
+using namespace TextEditor;
 using namespace TextEditor::Internal;
 
 FindInCurrentFile::FindInCurrentFile(SearchResultWindow *resultWindow)
@@ -64,10 +66,13 @@ QString FindInCurrentFile::displayName() const
 
 Utils::FileIterator *FindInCurrentFile::files() const
 {
-    QStringList fileList;
+    Q_ASSERT(isEnabled());
-    if (isEnabled())
+    QString fileName = m_currentFile->fileName();
-        fileList << m_currentFile->fileName();
+    QMap<QString, QTextCodec *> openEditorEncodings = ITextEditor::openedTextEditorsEncodings();
-    return new Utils::FileIterator(fileList);
+    QTextCodec *codec = openEditorEncodings.value(fileName);
+    if (!codec)
+        codec = Core::EditorManager::instance()->defaultTextEncoding();
+    return new Utils::FileIterator(QStringList() << fileName, QList<QTextCodec *>() << codec);
 }
 
 bool FindInCurrentFile::isEnabled() const

src/plugins/texteditor/findinfiles.cpp

@@ -29,6 +29,8 @@
 
 #include "findinfiles.h"
 
+#include <coreplugin/editormanager/editormanager.h>
+
 #include <QtCore/QtDebug>
 #include <QtCore/QSettings>
 #include <QtCore/QDir>
@@ -66,7 +68,8 @@ void FindInFiles::findAll(const QString &txt, Find::FindFlags findFlags)
 Utils::FileIterator *FindInFiles::files() const
 {
     return new Utils::SubDirFileIterator(QStringList() << m_directory->currentText(),
-                                         fileNameFilters());
+                                         fileNameFilters(),
+                                         Core::EditorManager::instance()->defaultTextEncoding());
 }
 
 QWidget *FindInFiles::createConfigWidget()

src/plugins/texteditor/itexteditor.cpp

@@ -31,6 +31,8 @@
 
 #include <coreplugin/editormanager/editormanager.h>
 
+#include <QtCore/QTextCodec>
+
 using namespace TextEditor;
 
 QMap<QString, QString> ITextEditor::openedTextEditorsContents()
@@ -45,3 +47,16 @@ QMap<QString, QString> ITextEditor::openedTextEditorsContents()
     }
     return workingCopy;
 }
+
+QMap<QString, QTextCodec *> TextEditor::ITextEditor::openedTextEditorsEncodings()
+{
+    QMap<QString, QTextCodec *> workingCopy;
+    foreach (Core::IEditor *editor, Core::EditorManager::instance()->openedEditors()) {
+        ITextEditor *textEditor = qobject_cast<ITextEditor *>(editor);
+        if (!textEditor)
+            continue;
+        QString fileName = textEditor->file()->fileName();
+        workingCopy[fileName] = textEditor->textCodec();
+    }
+    return workingCopy;
+}

src/plugins/texteditor/itexteditor.h

@@ -119,6 +119,7 @@ public:
     virtual QTextCodec *textCodec() const = 0;
 
     static QMap<QString, QString> openedTextEditorsContents();
+    static QMap<QString, QTextCodec *> openedTextEditorsEncodings();
 
 signals:
     void contentsChanged();

tests/manual/search/latin1.txt

@@ -0,0 +1 @@
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>

tests/manual/search/search.pro

@@ -0,0 +1 @@
+OTHER_FILES = latin1.txt