Extract Core::TextFile from TextEditor::BaseTextDocument.

Use it in BaseTextEditor and Designer to preserve CRLF
of the files.

Task-number: QTCREATORBUG-5901
Change-Id: I7599ce78649a3b09f2e5118d02f8cbf3db27ed31
Reviewed-on: http://codereview.qt.nokia.com/3591
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@nokia.com>

src/libs/utils/textfileformat.cpp

@@ -0,0 +1,314 @@
+/**************************************************************************
+**
+** This file is part of Qt Creator
+**
+** Copyright (c) 2011 Nokia Corporation and/or its subsidiary(-ies).
+**
+** Contact: Nokia Corporation (info@qt.nokia.com)
+**
+**
+** GNU Lesser General Public License Usage
+**
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this file.
+** Please review the following information to ensure the GNU Lesser General
+** Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** Other Usage
+**
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at info@qt.nokia.com.
+**
+**************************************************************************/
+
+#include "textfileformat.h"
+#include "fileutils.h"
+#include "qtcassert.h"
+
+#include <QtCore/QTextCodec>
+#include <QtCore/QStringList>
+#include <QtCore/QCoreApplication>
+#include <QtCore/QDebug>
+
+enum { debug = 0 };
+
+#include <exception>
+
+namespace Utils {
+
+QDebug operator<<(QDebug d, const TextFileFormat &format)
+{
+    QDebug nsp = d.nospace();
+    nsp << "TextFileFormat: ";
+    if (format.codec) {
+        nsp << format.codec->name();
+        foreach (const QByteArray &alias, format.codec->aliases())
+            nsp << ' ' << alias;
+    } else {
+        nsp << "NULL";
+    }
+    nsp << " hasUtf8Bom=" << format.hasUtf8Bom
+        << (format.lineTerminationMode == TextFileFormat::LFLineTerminator ? " LF" : " CRLF");
+    return d;
+}
+
+/*!
+    \class Utils::TextFileFormat
+
+    \brief Describes the format of a text file and provides autodetection.
+
+    The format comprises
+    \list
+    \o Encoding represented by a pointer to a QTextCodec
+    \o Presence of an UTF8 Byte Order Marker (BOM)
+    \o Line feed storage convention
+    \endlist
+
+    The class also provides convenience functions to read text files and return them
+    as strings or string lists and to write out files.
+*/
+
+TextFileFormat::TextFileFormat() :
+    lineTerminationMode(NativeLineTerminator), hasUtf8Bom(false), codec(0)
+{
+}
+
+/*!
+    \brief Detect the format of text data.
+*/
+
+TextFileFormat TextFileFormat::detect(const QByteArray &data)
+{
+    TextFileFormat result;
+    if (data.isEmpty())
+        return result;
+    const int bytesRead = data.size();
+    const unsigned char *buf = reinterpret_cast<const unsigned char *>(data.constData());
+    // code taken from qtextstream
+    if (bytesRead >= 4 && ((buf[0] == 0xff && buf[1] == 0xfe && buf[2] == 0 && buf[3] == 0)
+                           || (buf[0] == 0 && buf[1] == 0 && buf[2] == 0xfe && buf[3] == 0xff))) {
+        result.codec = QTextCodec::codecForName("UTF-32");
+    } else if (bytesRead >= 2 && ((buf[0] == 0xff && buf[1] == 0xfe)
+                                  || (buf[0] == 0xfe && buf[1] == 0xff))) {
+        result.codec = QTextCodec::codecForName("UTF-16");
+    } else if (bytesRead >= 3 && ((buf[0] == 0xef && buf[1] == 0xbb) && buf[2] == 0xbf)) {
+        result.codec = QTextCodec::codecForName("UTF-8");
+        result.hasUtf8Bom = true;
+    }
+    // end code taken from qtextstream
+    const int newLinePos = data.indexOf('\n');
+    if (newLinePos == -1) {
+        result.lineTerminationMode = NativeLineTerminator;
+    } else if (newLinePos == 0) {
+        result.lineTerminationMode = LFLineTerminator;
+    } else {
+        result.lineTerminationMode = data.at(newLinePos - 1) == '\r' ? CRLFLineTerminator : LFLineTerminator;
+    }
+    return result;
+}
+
+/*!
+    \brief Returns a piece of text suitable as display for a encoding error.
+*/
+
+QByteArray TextFileFormat::decodingErrorSample(const QByteArray &data)
+{
+    const int p = data.indexOf('\n', 16384);
+    return p < 0 ? data : data.left(p);
+}
+
+enum { textChunkSize = 65536 };
+
+static bool verifyDecodingError(const QString &text, const QTextCodec *codec,
+                                const char *data, const int dataSize,
+                                const bool possibleHeader)
+{
+    const QByteArray verifyBuf = codec->fromUnicode(text); // slow
+    // the minSize trick lets us ignore unicode headers
+    const int minSize = qMin(verifyBuf.size(), dataSize);
+    return (minSize < dataSize - (possibleHeader? 4 : 0)
+            || memcmp(verifyBuf.constData() + verifyBuf.size() - minSize,
+                      data + dataSize - minSize,
+                      minSize));
+}
+
+// Decode a potentially large file in chunks and append it to target
+// using the append function passed on (fits QStringList and QString).
+
+template <class Target>
+bool decodeTextFileContent(const QByteArray &dataBA,
+                           const TextFileFormat &format,
+                           Target *target,
+                           void (Target::*appendFunction)(const QString &))
+{
+    QTC_ASSERT(format.codec, return  false; )
+
+    QTextCodec::ConverterState state;
+    bool hasDecodingError = false;
+
+    const char *start = dataBA.constData();
+    const char *data = start;
+    const char *end  = data + dataBA.size();
+    // Process chunkwise as QTextCodec allocates too much memory when doing it in one
+    // go. An alternative to the code below would be creating a decoder from the codec,
+    // but its failure detection does not seem be working reliably.
+    for (const char *data = start; data < end; ) {
+        const char *chunkStart = data;
+        const int chunkSize = qMin(int(textChunkSize), int(end - chunkStart));
+        QString text = format.codec->toUnicode(chunkStart, chunkSize, &state);
+        data += chunkSize;
+        // Process until the end of the current multi-byte character. Remaining might
+        // actually contain more than needed so try one-be-one. If EOF is reached with
+        // and characters remain->encoding error.
+        for ( ; state.remainingChars && data < end ; ++data)
+            text.append(format.codec->toUnicode(data, 1, &state));
+        if (state.remainingChars)
+            hasDecodingError = true;
+        if (!hasDecodingError)
+            hasDecodingError =
+                verifyDecodingError(text, format.codec, chunkStart, data - chunkStart,
+                                    chunkStart == start);
+        if (format.lineTerminationMode == TextFileFormat::CRLFLineTerminator)
+            text.remove(QLatin1Char('\r'));
+        (target->*appendFunction)(text);
+    }
+    return !hasDecodingError;
+}
+
+/*!
+    \brief Decode data to a plain string.
+*/
+
+bool TextFileFormat::decode(const QByteArray &data, QString *target) const
+{
+    target->clear();
+    return decodeTextFileContent<QString>(data, *this, target, &QString::push_back);
+}
+
+/*!
+    \brief Decode data to a list of strings.
+
+    Intended for use with progress bars loading large files.
+*/
+
+bool TextFileFormat::decode(const QByteArray &data, QStringList *target) const
+{
+    target->clear();
+    if (data.size() > textChunkSize)
+        target->reserve(5 + data.size() / textChunkSize);
+    return decodeTextFileContent<QStringList>(data, *this, target, &QStringList::append);
+}
+
+// Read text file contents to string or stringlist.
+template <class Target>
+TextFileFormat::ReadResult readTextFile(const QString &fileName, const QTextCodec *defaultCodec,
+                                        Target *target, TextFileFormat *format, QString *errorString,
+                                        QByteArray *decodingErrorSampleIn = 0)
+{
+    if (decodingErrorSampleIn)
+        decodingErrorSampleIn->clear();
+
+    QByteArray data;
+    try {
+        Utils::FileReader reader;
+        if (!reader.fetch(fileName, errorString))
+            return TextFileFormat::ReadIOError;
+        data = reader.data();
+    } catch (const std::bad_alloc &) {
+        *errorString = QCoreApplication::translate("Utils::TextFileFormat", "Out of memory.");
+        return TextFileFormat::ReadMemoryAllocationError;
+    }
+
+    *format = TextFileFormat::detect(data);
+    if (!format->codec)
+        format->codec = defaultCodec ? defaultCodec : QTextCodec::codecForLocale();
+
+    if (!format->decode(data, target)) {
+        *errorString = QCoreApplication::translate("Utils::TextFileFormat", "An encoding error was encountered.");
+        if (decodingErrorSampleIn)
+            *decodingErrorSampleIn = TextFileFormat::decodingErrorSample(data);
+        return TextFileFormat::ReadEncodingError;
+    }
+    return TextFileFormat::ReadSuccess;
+}
+
+/*!
+    \brief Read text file into a list of strings.
+*/
+
+TextFileFormat::ReadResult
+    TextFileFormat::readFile(const QString &fileName, const QTextCodec *defaultCodec,
+                             QStringList *plainTextList, TextFileFormat *format, QString *errorString,
+                             QByteArray *decodingErrorSample /* = 0 */)
+{
+    const TextFileFormat::ReadResult result =
+        readTextFile(fileName, defaultCodec,
+                     plainTextList, format, errorString, decodingErrorSample);
+    if (debug)
+        qDebug().nospace() << Q_FUNC_INFO << fileName << ' ' << *format
+                           << " returns " << result << '/' << plainTextList->size() << " chunks";
+    return result;
+}
+
+/*!
+    \brief Read text file into a string.
+*/
+
+TextFileFormat::ReadResult
+    TextFileFormat::readFile(const QString &fileName, const QTextCodec *defaultCodec,
+                             QString *plainText, TextFileFormat *format, QString *errorString,
+                             QByteArray *decodingErrorSample /* = 0 */)
+{
+    const TextFileFormat::ReadResult result =
+        readTextFile(fileName, defaultCodec,
+                     plainText, format, errorString, decodingErrorSample);
+    if (debug)
+        qDebug().nospace() << Q_FUNC_INFO << fileName << ' ' << *format
+                           << " returns " << result << '/' << plainText->size() << " characters";
+    return result;
+}
+
+/*!
+    \brief Write out a text file.
+*/
+
+bool TextFileFormat::writeFile(const QString &fileName, QString plainText, QString *errorString) const
+{
+    QTC_ASSERT(codec, return false;)
+
+    // Does the user want CRLF? If that is native,
+    // let QFile do the work, else manually add.
+    QIODevice::OpenMode fileMode = QIODevice::NotOpen;
+    if (lineTerminationMode == CRLFLineTerminator) {
+        if (NativeLineTerminator == CRLFLineTerminator) {
+            fileMode |= QIODevice::Text;
+        } else {
+            plainText.replace(QLatin1Char('\n'), QLatin1String("\r\n"));
+        }
+    }
+
+    Utils::FileSaver saver(fileName, fileMode);
+    if (saver.hasError()) {
+        *errorString = saver.errorString();
+        return false;
+    }
+    if (hasUtf8Bom && codec->name() == "UTF-8")
+        saver.write("\xef\xbb\xbf", 3);
+    saver.write(codec->fromUnicode(plainText));
+    const bool ok = saver.finalize(errorString);
+    if (debug)
+        qDebug().nospace() << Q_FUNC_INFO << fileName << ' ' << *this <<  ' ' << plainText.size()
+                           << " bytes, returns " << ok;
+    return ok;
+}
+
+} // namespace Utils

src/libs/utils/textfileformat.h

@@ -0,0 +1,96 @@
+/**************************************************************************
+**
+** This file is part of Qt Creator
+**
+** Copyright (c) 2011 Nokia Corporation and/or its subsidiary(-ies).
+**
+** Contact: Nokia Corporation (info@qt.nokia.com)
+**
+**
+** GNU Lesser General Public License Usage
+**
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this file.
+** Please review the following information to ensure the GNU Lesser General
+** Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** Other Usage
+**
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at info@qt.nokia.com.
+**
+**************************************************************************/
+
+#ifndef TEXTFILEUTILS_H
+#define TEXTFILEUTILS_H
+
+#include "utils_global.h"
+
+#include <QtCore/QtGlobal>
+
+QT_BEGIN_NAMESPACE
+class QTextCodec;
+class QStringList;
+class QString;
+class QByteArray;
+QT_END_NAMESPACE
+
+namespace Utils {
+
+class QTCREATOR_UTILS_EXPORT TextFileFormat  {
+public:
+    enum LineTerminationMode
+    {
+        LFLineTerminator,
+        CRLFLineTerminator,
+        NativeLineTerminator =
+#if defined (Q_OS_WIN)
+        CRLFLineTerminator,
+#else
+        LFLineTerminator
+#endif
+    };
+
+    enum ReadResult
+    {
+        ReadSuccess,
+        ReadEncodingError,
+        ReadMemoryAllocationError,
+        ReadIOError
+    };
+
+    TextFileFormat();
+
+    static TextFileFormat detect(const QByteArray &data);
+
+    bool decode(const QByteArray &data, QString *target) const;
+    bool decode(const QByteArray &data, QStringList *target) const;
+
+    static ReadResult readFile(const QString &fileName, const QTextCodec *defaultCodec,
+                               QStringList *plainText, TextFileFormat *format, QString *errorString,
+                               QByteArray *decodingErrorSample = 0);
+    static ReadResult readFile(const QString &fileName, const QTextCodec *defaultCodec,
+                               QString *plainText, TextFileFormat *format, QString *errorString,
+                               QByteArray *decodingErrorSample = 0);
+
+    bool writeFile(const QString &fileName, QString plainText, QString *errorString) const;
+
+    static QByteArray decodingErrorSample(const QByteArray &data);
+
+    LineTerminationMode lineTerminationMode;
+    bool hasUtf8Bom;
+    const QTextCodec *codec;
+};
+
+} // namespace Utils
+
+#endif // TEXTFILEUTILS_H

src/libs/utils/utils-lib.pri

@@ -40,6 +40,7 @@ SOURCES += $$PWD/environment.cpp \
     $$PWD/synchronousprocess.cpp \
     $$PWD/savefile.cpp \
     $$PWD/fileutils.cpp \
+    $$PWD/textfileformat.cpp \
     $$PWD/submitfieldwidget.cpp \
     $$PWD/consoleprocess.cpp \
     $$PWD/uncommentselection.cpp \
@@ -135,6 +136,7 @@ HEADERS += \
     $$PWD/synchronousprocess.h \
     $$PWD/savefile.h \
     $$PWD/fileutils.h \
+    $$PWD/textfileformat.h \
     $$PWD/submitfieldwidget.h \
     $$PWD/uncommentselection.h \
     $$PWD/parameteraction.h \

src/plugins/coreplugin/coreplugin.pro

@@ -91,7 +91,8 @@ SOURCES += mainwindow.cpp \
     mimetypemagicdialog.cpp \
     mimetypesettings.cpp \
     dialogs/promptoverwritedialog.cpp \
-    fileutils.cpp
+    fileutils.cpp \
+    textfile.cpp
 
 HEADERS += mainwindow.h \
     editmode.h \
@@ -185,7 +186,8 @@ HEADERS += mainwindow.h \
     mimetypesettings.h \
     dialogs/promptoverwritedialog.h \
     fileutils.h \
-    externaltoolmanager.h
+    externaltoolmanager.h \
+    textfile.h
 
 FORMS += dialogs/newdialog.ui \
     actionmanager/commandmappings.ui \

src/plugins/coreplugin/textfile.cpp

@@ -0,0 +1,152 @@
+/**************************************************************************
+**
+** This file is part of Qt Creator
+**
+** Copyright (c) 2011 Nokia Corporation and/or its subsidiary(-ies).
+**
+** Contact: Nokia Corporation (info@qt.nokia.com)
+**
+**
+** GNU Lesser General Public License Usage
+**
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this file.
+** Please review the following information to ensure the GNU Lesser General
+** Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** Other Usage
+**
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at info@qt.nokia.com.
+**
+**************************************************************************/
+
+#include "textfile.h"
+#include "editormanager.h"
+
+#include <QtCore/QDebug>
+#include <QtCore/QTextCodec>
+
+/*!
+    \class Core::TextFile
+    \brief Base class for text files with encoding helpers.
+
+    Stores the format obtained from read operations and uses that when writing
+    out files, thus ensuring that CRLF/encodings are preserved.
+
+    \sa Utils::TextFileUtils
+*/
+
+enum { debug = 0 };
+
+namespace Core {
+
+namespace Internal {
+class TextFilePrivate
+{
+public:
+    TextFilePrivate() : m_readResult(Utils::TextFileFormat::ReadSuccess) {}
+
+    Utils::TextFileFormat m_format;
+    Utils::TextFileFormat::ReadResult m_readResult;
+    QByteArray m_decodingErrorSample;
+};
+
+} // namespace Internal
+
+TextFile::TextFile(QObject *parent) :
+    IFile(parent), d(new Internal::TextFilePrivate)
+{
+    setCodec(Core::EditorManager::instance()->defaultTextCodec());
+}
+
+TextFile::~TextFile()
+{
+    delete d;
+}
+
+bool TextFile::hasDecodingError() const
+{
+    return d->m_readResult == Utils::TextFileFormat::ReadEncodingError;
+}
+
+QByteArray TextFile::decodingErrorSample() const
+{
+    return d->m_decodingErrorSample;
+}
+
+/*!
+    \brief Writes out text using the format obtained from the last read.
+*/
+
+bool TextFile::write(const QString &fileName, const QString &data, QString *errorMessage) const
+{
+    return write(fileName, format(), data, errorMessage);
+}
+
+/*!
+    \brief Writes out text using a custom format obtained.
+*/
+
+bool TextFile::write(const QString &fileName, const Utils::TextFileFormat &format, const QString &data, QString *errorMessage) const
+{
+    if (debug)
+        qDebug() << Q_FUNC_INFO << this << fileName;
+    return format.writeFile(fileName, data, errorMessage);
+}
+
+/*!
+    \brief Autodetect format and read in a text file.
+*/
+
+TextFile::ReadResult TextFile::read(const QString &fileName, QStringList *plainTextList, QString *errorString)
+{
+    d->m_readResult =
+        Utils::TextFileFormat::readFile(fileName, codec(),
+                                        plainTextList, &d->m_format, errorString, &d->m_decodingErrorSample);
+    return d->m_readResult;
+}
+
+/*!
+    \brief Autodetect format and read in a text file.
+*/
+
+TextFile::ReadResult TextFile::read(const QString &fileName, QString *plainText, QString *errorString)
+{
+    d->m_readResult =
+        Utils::TextFileFormat::readFile(fileName, codec(),
+                                        plainText, &d->m_format, errorString, &d->m_decodingErrorSample);
+    return d->m_readResult;
+}
+
+const QTextCodec *TextFile::codec() const
+{
+    return d->m_format.codec;
+}
+
+void TextFile::setCodec(const QTextCodec *codec)
+{
+    if (debug)
+        qDebug() << Q_FUNC_INFO << this << (codec ? codec->name() : QByteArray());
+    d->m_format.codec = codec;
+}
+
+/*!
+    \brief Returns the format obtained from the last call to read().
+*/
+
+Utils::TextFileFormat TextFile::format() const
+{
+    return d->m_format;
+}
+
+} // namespace Core

src/plugins/coreplugin/textfile.h

@@ -0,0 +1,74 @@
+/**************************************************************************
+**
+** This file is part of Qt Creator
+**
+** Copyright (c) 2011 Nokia Corporation and/or its subsidiary(-ies).
+**
+** Contact: Nokia Corporation (info@qt.nokia.com)
+**
+**
+** GNU Lesser General Public License Usage
+**
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this file.
+** Please review the following information to ensure the GNU Lesser General
+** Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** Other Usage
+**
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at info@qt.nokia.com.
+**
+**************************************************************************/
+
+#ifndef CORE_TEXTFILE_H
+#define CORE_TEXTFILE_H
+
+#include "ifile.h"
+
+#include <utils/textfileformat.h>
+
+namespace Core {
+
+namespace Internal {
+class TextFilePrivate;
+}
+
+class CORE_EXPORT TextFile : public IFile
+{
+    Q_OBJECT
+public:
+    typedef Utils::TextFileFormat::ReadResult ReadResult;
+
+    explicit TextFile(QObject *parent = 0);
+    virtual ~TextFile();
+
+    Utils::TextFileFormat format() const;
+    const QTextCodec *codec() const;
+    void setCodec(const QTextCodec *);
+
+    ReadResult read(const QString &fileName, QStringList *plainTextList, QString *errorString);
+    ReadResult read(const QString &fileName, QString *plainText, QString *errorString);
+
+    bool hasDecodingError() const;
+    QByteArray decodingErrorSample() const;
+
+    bool write(const QString &fileName, const QString &data, QString *errorMessage) const;
+    bool write(const QString &fileName, const Utils::TextFileFormat &format, const QString &data, QString *errorMessage) const;
+
+private:
+    Internal::TextFilePrivate *d;
+};
+
+} // namespace Core
+
+#endif // CORE_TEXTFILE_H

src/plugins/designer/formwindoweditor.cpp

@@ -157,19 +157,19 @@ bool FormWindowEditor::open(QString *errorString, const QString &fileName, const
     const QFileInfo fi(fileName);
     const QString absfileName = fi.absoluteFilePath();
 
-    Utils::FileReader reader;
+    QString contents;
-    if (!reader.fetch(realFileName, QIODevice::Text, errorString))
+    if (d->m_file.read(absfileName, &contents, errorString) != Utils::TextFileFormat::ReadSuccess)
         return false;
 
     form->setFileName(absfileName);
-    QByteArray contents = reader.data();
 #if QT_VERSION >= 0x050000
-    QBuffer str(&contents);
+    const QByteArray contentsBA = contents.toUtf8();
+    QBuffer str(&contentsBA);
     str.open(QIODevice::ReadOnly);
     if (!form->setContents(&str, errorString))
         return false;
 #else
-    form->setContents(QString::fromUtf8(contents));
+    form->setContents(contents);
     if (!form->mainContainer())
         return false;
 #endif

src/plugins/designer/formwindowfile.cpp

@@ -53,16 +53,19 @@
 #include <QtCore/QFileInfo>
 #include <QtCore/QByteArray>
 #include <QtCore/QDebug>
+#include <QtCore/QTextCodec>
 
 namespace Designer {
 namespace Internal {
 
 FormWindowFile::FormWindowFile(QDesignerFormWindowInterface *form, QObject *parent)
-  : Core::IFile(parent),
+  : Core::TextFile(parent),
     m_mimeType(QLatin1String(Designer::Constants::FORM_MIMETYPE)),
     m_shouldAutoSave(false),
     m_formWindow(form)
 {
+    // Designer needs UTF-8 regardless of settings.
+    setCodec(QTextCodec::codecForName("UTF-8"));
     connect(m_formWindow->core()->formWindowManager(), SIGNAL(formWindowRemoved(QDesignerFormWindowInterface*)),
             this, SLOT(slotFormWindowRemoved(QDesignerFormWindowInterface*)));
     connect(m_formWindow->commandHistory(), SIGNAL(indexChanged(int)),
@@ -190,10 +193,7 @@ bool FormWindowFile::writeFile(const QString &fileName, QString *errorString) co
 {
     if (Designer::Constants::Internal::debug)
         qDebug() << Q_FUNC_INFO << m_fileName << fileName;
-
+    return write(fileName, format(), m_formWindow->contents(), errorString);
-    Utils::FileSaver saver(fileName, QIODevice::Text);
-    saver.write(m_formWindow->contents().toUtf8());
-    return saver.finalize(errorString);
 }
 
 void FormWindowFile::setFileName(const QString &fname)

src/plugins/designer/formwindowfile.h

@@ -33,7 +33,7 @@
 #ifndef FORMWINDOWFILE_H
 #define FORMWINDOWFILE_H
 
-#include <coreplugin/ifile.h>
+#include <coreplugin/textfile.h>
 
 #include <QtCore/QPointer>
 
@@ -45,7 +45,7 @@ QT_END_NAMESPACE
 namespace Designer {
 namespace Internal {
 
-class FormWindowFile : public Core::IFile
+class FormWindowFile : public Core::TextFile
 {
     Q_OBJECT
 

src/plugins/texteditor/basetextdocument.cpp

@@ -1,4 +1,4 @@
-/**************************************************************************
+/**************************************************************************
 **
 ** This file is part of Qt Creator
 **
@@ -58,23 +58,6 @@
 #include <utils/fileutils.h>
 #include <utils/reloadpromptutils.h>
 
-namespace {
-bool verifyDecodingError(const QString &text,
-                         QTextCodec *codec,
-                         const char *data,
-                         const int dataSize,
-                         const bool possibleHeader)
-{
-    QByteArray verifyBuf = codec->fromUnicode(text); // slow
-    // the minSize trick lets us ignore unicode headers
-    int minSize = qMin(verifyBuf.size(), dataSize);
-    return (minSize < dataSize - (possibleHeader? 4 : 0)
-            || memcmp(verifyBuf.constData() + verifyBuf.size() - minSize,
-                      data + dataSize - minSize,
-                      minSize));
-}
-}
-
 namespace TextEditor {
 namespace Internal {
 
@@ -212,40 +195,17 @@ public:
     Internal::DocumentMarker *m_documentMarker;
     SyntaxHighlighter *m_highlighter;
 
-    enum LineTerminatorMode {
-        LFLineTerminator,
-        CRLFLineTerminator,
-        NativeLineTerminator =
-#if defined (Q_OS_WIN)
-        CRLFLineTerminator
-#else
-        LFLineTerminator
-#endif
-    };
-    LineTerminatorMode m_lineTerminatorMode;
-    QTextCodec *m_codec;
-    bool m_fileHasUtf8Bom;
-
     bool m_fileIsReadOnly;
-    bool m_hasDecodingError;
     bool m_hasHighlightWarning;
-    QByteArray m_decodingErrorSample;
-    static const int kChunkSize;
 
     int m_autoSaveRevision;
 };
 
-const int BaseTextDocumentPrivate::kChunkSize = 65536;
-
 BaseTextDocumentPrivate::BaseTextDocumentPrivate(BaseTextDocument *q) :
     m_document(new QTextDocument(q)),
     m_documentMarker(new Internal::DocumentMarker(m_document)),
     m_highlighter(0),
-    m_lineTerminatorMode(NativeLineTerminator),
-    m_codec(Core::EditorManager::instance()->defaultTextCodec()),
-    m_fileHasUtf8Bom(false),
     m_fileIsReadOnly(false),
-    m_hasDecodingError(false),
     m_hasHighlightWarning(false),
     m_autoSaveRevision(-1)
 {
@@ -343,26 +303,6 @@ SyntaxHighlighter *BaseTextDocument::syntaxHighlighter() const
     return d->m_highlighter;
 }
 
-bool BaseTextDocument::hasDecodingError() const
-{
-    return d->m_hasDecodingError;
-}
-
-QTextCodec *BaseTextDocument::codec() const
-{
-    return d->m_codec;
-}
-
-void BaseTextDocument::setCodec(QTextCodec *c)
-{
-    d->m_codec = c;
-}
-
-QByteArray BaseTextDocument::decodingErrorSample() const
-{
-    return d->m_decodingErrorSample;
-}
-
 ITextMarkable *BaseTextDocument::documentMarker() const
 {
     return d->m_documentMarker;
@@ -403,22 +343,21 @@ bool BaseTextDocument::save(QString *errorString, const QString &fileName, bool
     if (!fileName.isEmpty())
         fName = fileName;
 
-    Utils::FileSaver saver(fName);
+    Utils::TextFileFormat saveFormat = format();
-    if (!saver.hasError()) {
+    if (saveFormat.codec->name() == "UTF-8") {
-        QString plainText = d->m_document->toPlainText();
+        switch (d->m_extraEncodingSettings.m_utf8BomSetting) {
-
+        case TextEditor::ExtraEncodingSettings::AlwaysAdd:
-        if (d->m_lineTerminatorMode == BaseTextDocumentPrivate::CRLFLineTerminator)
+            saveFormat.hasUtf8Bom = true;
-            plainText.replace(QLatin1Char('\n'), QLatin1String("\r\n"));
+            break;
-
+        case TextEditor::ExtraEncodingSettings::OnlyKeep:
-        if (d->m_codec->name() == "UTF-8"
+            break;
-                && (d->m_extraEncodingSettings.m_utf8BomSetting == ExtraEncodingSettings::AlwaysAdd
+        case TextEditor::ExtraEncodingSettings::AlwaysDelete:
-                    || (d->m_extraEncodingSettings.m_utf8BomSetting == ExtraEncodingSettings::OnlyKeep
+            saveFormat.hasUtf8Bom = false;
-                        && d->m_fileHasUtf8Bom))) {
+            break;
-            saver.write("\xef\xbb\xbf", 3);
         }
+    } // "UTF-8"
 
-        saver.write(d->m_codec->fromUnicode(plainText));
+    const bool ok = write(fName, saveFormat, d->m_document->toPlainText(), errorString);
-    }
 
     if (autoSave && undos < d->m_document->availableUndoSteps()) {
         d->m_document->undo();
@@ -430,7 +369,7 @@ bool BaseTextDocument::save(QString *errorString, const QString &fileName, bool
         }
     }
 
-    if (!saver.finalize(errorString))
+    if (!ok)
         return false;
     d->m_autoSaveRevision = d->m_document->revision();
     if (autoSave)
@@ -442,10 +381,6 @@ bool BaseTextDocument::save(QString *errorString, const QString &fileName, bool
     d->m_document->setModified(false);
     emit titleChanged(fi.fileName());
     emit changed();
-
-    d->m_hasDecodingError = false;
-    d->m_decodingErrorSample.clear();
-
     return true;
 }
 
@@ -464,7 +399,7 @@ void BaseTextDocument::rename(const QString &newName)
 
 bool BaseTextDocument::isReadOnly() const
 {
-    if (d->m_hasDecodingError)
+    if (hasDecodingError())
         return true;
     if (d->m_fileName.isEmpty()) //have no corresponding file, so editing is ok
         return false;
@@ -492,109 +427,23 @@ void BaseTextDocument::checkPermissions()
 bool BaseTextDocument::open(QString *errorString, const QString &fileName, const QString &realFileName)
 {
     QString title = tr("untitled");
+    QStringList content;
+
+    ReadResult readResult = Utils::TextFileFormat::ReadIOError;
+
     if (!fileName.isEmpty()) {
         const QFileInfo fi(fileName);
         d->m_fileIsReadOnly = !fi.isWritable();
         d->m_fileName = QDir::cleanPath(fi.absoluteFilePath());
 
         title = fi.fileName();
-
+        readResult = read(realFileName, &content, errorString);
-        QByteArray buf;
-        try {
-            Utils::FileReader reader;
-            if (!reader.fetch(realFileName, errorString))
-                return false;
-            buf = reader.data();
-        } catch (std::bad_alloc) {
-            *errorString = tr("Out of memory");
-            return false;
-        }
-        int bytesRead = buf.size();
-
-        QTextCodec *codec = d->m_codec;
-        d->m_fileHasUtf8Bom = false;
-
-        // code taken from qtextstream
-        if (bytesRead >= 4 && ((uchar(buf[0]) == 0xff && uchar(buf[1]) == 0xfe && uchar(buf[2]) == 0 && uchar(buf[3]) == 0)
-                               || (uchar(buf[0]) == 0 && uchar(buf[1]) == 0 && uchar(buf[2]) == 0xfe && uchar(buf[3]) == 0xff))) {
-            codec = QTextCodec::codecForName("UTF-32");
-        } else if (bytesRead >= 2 && ((uchar(buf[0]) == 0xff && uchar(buf[1]) == 0xfe)
-                                      || (uchar(buf[0]) == 0xfe && uchar(buf[1]) == 0xff))) {
-            codec = QTextCodec::codecForName("UTF-16");
-        } else if (bytesRead >= 3 && ((uchar(buf[0]) == 0xef && uchar(buf[1]) == 0xbb) && uchar(buf[2]) == 0xbf)) {
-            codec = QTextCodec::codecForName("UTF-8");
-            d->m_fileHasUtf8Bom = true;
-        } else if (!codec) {
-            codec = QTextCodec::codecForLocale();
-        }
-        // end code taken from qtextstream
-
-        d->m_codec = codec;
-
-        // An alternative to the code below would be creating a decoder from the codec,
-        // but failure detection doesn't seem be working reliably.
-        QStringList content;
-        if (bytesRead <= BaseTextDocumentPrivate::kChunkSize) {
-            QString text = d->m_codec->toUnicode(buf);
-            d->m_hasDecodingError = verifyDecodingError(
-                text, d->m_codec, buf.constData(), bytesRead, true);
-            content.append(text);
-        } else {
-            // Avoid large allocation of contiguous memory.
-            QTextCodec::ConverterState state;
-            int offset = 0;
-            while (offset < bytesRead) {
-                int currentSize = qMin(BaseTextDocumentPrivate::kChunkSize, bytesRead - offset);
-                QString text = d->m_codec->toUnicode(buf.constData() + offset, currentSize, &state);
-                if (state.remainingChars) {
-                    if (currentSize < BaseTextDocumentPrivate::kChunkSize && !d->m_hasDecodingError)
-                        d->m_hasDecodingError = true;
-
-                    // Process until the end of the current multi-byte character. Remaining might
-                    // actually contain more than needed so try one-be-one.
-                    while (state.remainingChars) {
-                        text.append(d->m_codec->toUnicode(
-                            buf.constData() + offset + currentSize, 1, &state));
-                        ++currentSize;
-                    }
-                }
-
-                if (!d->m_hasDecodingError) {
-                    d->m_hasDecodingError = verifyDecodingError(
-                        text, d->m_codec, buf.constData() + offset, currentSize, offset == 0);
-                }
-
-                offset += currentSize;
-                content.append(text);
-            }
-        }
-
-        if (d->m_hasDecodingError) {
-            int p = buf.indexOf('\n', 16384);
-            if (p < 0)
-                d->m_decodingErrorSample = buf;
-            else
-                d->m_decodingErrorSample = buf.left(p);
-        } else {
-            d->m_decodingErrorSample.clear();
-        }
-        buf.clear();
-
-        foreach (const QString &text, content) {
-            int lf = text.indexOf('\n');
-            if (lf >= 0) {
-                if (lf > 0 && text.at(lf-1) == QLatin1Char('\r')) {
-                    d->m_lineTerminatorMode = BaseTextDocumentPrivate::CRLFLineTerminator;
-                } else {
-                    d->m_lineTerminatorMode = BaseTextDocumentPrivate::LFLineTerminator;
-                }
-                break;
-            }
-        }
 
         d->m_document->setModified(false);
         const int chunks = content.size();
-        if (chunks == 1) {
+        if (chunks == 0) {
+            d->m_document->setPlainText(QString());
+        } else if (chunks == 1) {
             d->m_document->setPlainText(content.at(0));
         } else {
             QFutureInterface<void> interface;
@@ -629,7 +478,7 @@ bool BaseTextDocument::open(QString *errorString, const QString &fileName, const
 bool BaseTextDocument::reload(QString *errorString, QTextCodec *codec)
 {
     QTC_ASSERT(codec, return false);
-    d->m_codec = codec;
+    setCodec(codec);
     return reload(errorString);
 }
 

src/plugins/texteditor/basetextdocument.h

@@ -35,7 +35,7 @@
 
 #include "texteditor_global.h"
 
-#include <coreplugin/ifile.h>
+#include <coreplugin/textfile.h>
 
 QT_BEGIN_NAMESPACE
 class QTextCursor;
@@ -51,7 +51,7 @@ class ExtraEncodingSettings;
 class SyntaxHighlighter;
 class BaseTextDocumentPrivate;
 
-class TEXTEDITOR_EXPORT BaseTextDocument : public Core::IFile
+class TEXTEDITOR_EXPORT BaseTextDocument : public Core::TextFile
 {
     Q_OBJECT
 
@@ -95,11 +95,6 @@ public:
     void setSyntaxHighlighter(SyntaxHighlighter *highlighter);
     SyntaxHighlighter *syntaxHighlighter() const;
 
-    bool hasDecodingError() const;
-    QTextCodec *codec() const;
-    void setCodec(QTextCodec *c);
-    QByteArray decodingErrorSample() const;
-
     bool reload(QString *errorString, QTextCodec *codec);
     void cleanWhitespace(const QTextCursor &cursor);
 

src/plugins/texteditor/basetexteditor.cpp

@@ -5615,7 +5615,8 @@ void BaseTextEditorWidget::setTextCodec(QTextCodec *codec)
 
 QTextCodec *BaseTextEditorWidget::textCodec() const
 {
-    return baseTextDocument()->codec();
+    // TODO: Fix all QTextCodec usages to be const *.
+    return const_cast<QTextCodec *>(baseTextDocument()->codec());
 }
 
 void BaseTextEditorWidget::setReadOnly(bool b)

src/plugins/vcsbase/vcsbaseeditor.cpp

@@ -387,7 +387,7 @@ void VCSBaseEditorWidget::setDiffBaseDirectory(const QString &bd)
 
 QTextCodec *VCSBaseEditorWidget::codec() const
 {
-    return baseTextDocument()->codec();
+    return const_cast<QTextCodec *>(baseTextDocument()->codec());
 }
 
 void VCSBaseEditorWidget::setCodec(QTextCodec *c)