BinEditor: Support character encodings for the text part

Change-Id: I853b5b5c4c4d523033319169e80e6f9063360c17
Reviewed-by: David Schulz <david.schulz@qt.io>

src/plugins/bineditor/bineditorconstants.h

@@ -31,6 +31,7 @@ namespace Constants {
 const char C_BINEDITOR[] = "BinEditor.BinaryEditor";
 const char C_BINEDITOR_DISPLAY_NAME[] = QT_TRANSLATE_NOOP("OpenWith::Editors", "Binary Editor");
 const char C_BINEDITOR_MIMETYPE[] = "application/octet-stream";
+const char C_ENCODING_SETTING[] = "BinEditor/TextEncoding";
 
 } // namespace Constants
 } // namespace BinEditor

src/plugins/bineditor/bineditorplugin.cpp

@@ -29,11 +29,13 @@
 #include "bineditorservice.h"
 
 #include <coreplugin/icore.h>
+#include <texteditor/codecchooser.h>
 
 #include <QCoreApplication>
 #include <QDir>
 #include <QFile>
 #include <QFileInfo>
+#include <QTextCodec>
 #include <QVariant>
 
 #include <QMenu>
@@ -343,16 +345,20 @@ class BinEditor : public IEditor
 public:
     BinEditor(BinEditorWidget *widget)
     {
+        using namespace TextEditor;
         setWidget(widget);
         m_file = new BinEditorDocument(widget);
         m_addressEdit = new QLineEdit;
         auto addressValidator = new QRegularExpressionValidator(QRegularExpression("[0-9a-fA-F]{1,16}"), m_addressEdit);
         m_addressEdit->setValidator(addressValidator);
+        m_codecChooser = new CodecChooser(CodecChooser::Filter::SingleByte);
+        m_codecChooser->prependNone();
 
         auto l = new QHBoxLayout;
         auto w = new QWidget;
         l->setContentsMargins(0, 0, 5, 0);
         l->addStretch(1);
+        l->addWidget(m_codecChooser);
         l->addWidget(m_addressEdit);
         w->setLayout(l);
 
@@ -366,9 +372,14 @@ public:
                 this, &BinEditor::updateCursorPosition);
         connect(m_addressEdit, &QLineEdit::editingFinished,
                 this, &BinEditor::jumpToAddress);
+        connect(m_codecChooser, &CodecChooser::codecChanged,
+                widget, &BinEditorWidget::setCodec);
         connect(widget, &BinEditorWidget::modificationChanged,
                 m_file, &IDocument::changed);
         updateCursorPosition(widget->cursorPosition());
+        const QVariant setting = ICore::settings()->value(Constants::C_ENCODING_SETTING);
+        if (!setting.isNull())
+            m_codecChooser->setAssignedCodec(QTextCodec::codecForName(setting.toByteArray()));
     }
 
     ~BinEditor() override
@@ -400,6 +411,7 @@ private:
     BinEditorDocument *m_file;
     QToolBar *m_toolBar;
     QLineEdit *m_addressEdit;
+    TextEditor::CodecChooser *m_codecChooser;
 };
 
 ///////////////////////////////// BinEditorPluginPrivate //////////////////////////////////

src/plugins/bineditor/bineditorwidget.cpp

@@ -24,10 +24,12 @@
 ****************************************************************************/
 
 #include "bineditorwidget.h"
+#include "bineditorconstants.h"
 #include "bineditorservice.h"
 #include "markup.h"
 
 #include <coreplugin/coreconstants.h>
+#include <coreplugin/icore.h>
 #include <coreplugin/editormanager/editormanager.h>
 #include <coreplugin/editormanager/ieditor.h>
 
@@ -46,6 +48,7 @@
 #include <QByteArrayMatcher>
 #include <QDebug>
 #include <QFile>
+#include <QTextCodec>
 #include <QFontMetrics>
 #include <QHelpEvent>
 #include <QMenu>
@@ -136,6 +139,9 @@ private:
     std::function<void(quint64, uint)> m_watchPointRequestHandler;
     std::function<void()> m_aboutToBeDestroyedHandler;
     QList<Markup> m_markup;
+
+public:
+    QTextCodec *m_codec = nullptr;
 };
 
 BinEditorWidget::BinEditorWidget(QWidget *parent)
@@ -151,6 +157,9 @@ BinEditorWidget::BinEditorWidget(QWidget *parent)
             &TextEditor::TextEditorSettings::fontSettingsChanged,
             this, &BinEditorWidget::setFontSettings);
 
+    const QByteArray setting = ICore::settings()->value(Constants::C_ENCODING_SETTING).toByteArray();
+    if (!setting.isEmpty())
+        setCodec(QTextCodec::codecForName(setting));
 }
 
 BinEditorWidget::~BinEditorWidget()
@@ -536,6 +545,19 @@ QRect BinEditorWidget::cursorRect() const
     return QRect(x, y, w, m_lineHeight);
 }
 
+QChar BinEditorWidget::displayChar(char ch) const
+{
+    const QChar qc = QLatin1Char(ch);
+    if (qc.isPrint() && qc.unicode() < 128)
+        return qc;
+    if (!d->m_codec || qc.unicode() < 32)
+        return MidpointChar;
+    const QString uc = d->m_codec->toUnicode(&ch, 1);
+    if (uc.isEmpty() || !uc.at(0).isLetterOrNumber())
+        return MidpointChar;
+    return uc.at(0);
+}
+
 Utils::optional<qint64> BinEditorWidget::posAt(const QPoint &pos, bool includeEmptyArea) const
 {
     const int xoffset = horizontalScrollBar()->value();
@@ -553,9 +575,7 @@ Utils::optional<qint64> BinEditorWidget::posAt(const QPoint &pos, bool includeEm
             const qint64 dataPos = line * m_bytesPerLine + column;
             if (dataPos < 0 || dataPos >= m_size)
                 break;
-            QChar qc(QLatin1Char(dataAt(dataPos)));
+            const QChar qc = displayChar(dataAt(dataPos));
-            if (!qc.isPrint())
-                qc = MidpointChar;
             x -= fontMetrics().horizontalAdvance(qc);
             if (x <= 0)
                 break;
@@ -843,19 +863,21 @@ void BinEditorWidget::paintEvent(QPaintEvent *e)
         bool isOld = hasOldData && !hasData;
 
         QString printable;
+        QString printableDisp;
 
         if (hasData || hasOldData) {
             for (int c = 0; c < m_bytesPerLine; ++c) {
                 qint64 pos = line * m_bytesPerLine + c;
                 if (pos >= m_size)
                     break;
-                QChar qc(QLatin1Char(dataAt(pos, isOld)));
+                const QChar qc = displayChar(dataAt(pos, isOld));
-                if (qc.unicode() >= 127 || !qc.isPrint())
-                    qc = MidpointChar;
                 printable += qc;
+                printableDisp += qc;
+                if (qc.direction() == QChar::Direction::DirR)
+                    printableDisp += QChar(0x200E); // Add LRM to avoid reversing RTL text
             }
         } else {
-            printable = QString(m_bytesPerLine, QLatin1Char(' '));
+            printableDisp = printable = QString(m_bytesPerLine, QLatin1Char(' '));
         }
 
         QRect selectionRect;
@@ -963,16 +985,16 @@ void BinEditorWidget::paintEvent(QPaintEvent *e)
                 painter.fillRect(text_x, y-m_ascent, fm.horizontalAdvance(printable), m_lineHeight,
                                  palette().highlight());
                 painter.setPen(palette().highlightedText().color());
-                painter.drawText(text_x, y, printable);
+                painter.drawText(text_x, y, printableDisp);
                 painter.restore();
         } else {
-            painter.drawText(text_x, y, printable);
+            painter.drawText(text_x, y, printableDisp);
             if (!printableSelectionRect.isEmpty()) {
                 painter.save();
                 painter.fillRect(printableSelectionRect, palette().highlight());
                 painter.setPen(palette().highlightedText().color());
                 painter.setClipRect(printableSelectionRect);
-                painter.drawText(text_x, y, printable);
+                painter.drawText(text_x, y, printableDisp);
                 painter.restore();
             }
         }
@@ -989,7 +1011,7 @@ void BinEditorWidget::paintEvent(QPaintEvent *e)
                 painter.setClipRect(cursorRect);
                 painter.fillRect(cursorRect, Qt::red);
                 painter.setPen(Qt::white);
-                painter.drawText(text_x, y, printable);
+                painter.drawText(text_x, y, printableDisp);
                 painter.restore();
             }
         }
@@ -1472,7 +1494,8 @@ void BinEditorWidget::copy(bool raw)
     QByteArray data = dataMid(selStart, selectionLength);
     if (raw) {
         data.replace(0, ' ');
-        setClipboardAndSelection(QString::fromLatin1(data));
+        QTextCodec *codec = d->m_codec ? d->m_codec : QTextCodec::codecForName("latin1");
+        setClipboardAndSelection(codec->toUnicode(data));
         return;
     }
     QString hexString;
@@ -1672,6 +1695,15 @@ void BinEditorWidget::setNewWindowRequestAllowed(bool c)
     m_canRequestNewWindow = c;
 }
 
+void BinEditorWidget::setCodec(QTextCodec *codec)
+{
+    if (codec == d->m_codec)
+        return;
+    d->m_codec = codec;
+    ICore::settings()->setValue(Constants::C_ENCODING_SETTING, codec ? codec->name() : QByteArray());
+    viewport()->update();
+}
+
 void BinEditorWidget::updateContents()
 {
     m_oldData = m_data;

src/plugins/bineditor/bineditorwidget.h

@@ -41,8 +41,12 @@
 #include <QTextDocument>
 #include <QTextFormat>
 
-QT_FORWARD_DECLARE_CLASS(QMenu)
+QT_BEGIN_NAMESPACE
-QT_FORWARD_DECLARE_CLASS(QHelpEvent)
+class QHelpEvent;
+class QMenu;
+class QTextCodec;
+QT_END_NAMESPACE
+
 
 namespace Core { class IEditor; }
 
@@ -127,6 +131,7 @@ public:
     void copy(bool raw = false);
     void setMarkup(const QList<Markup> &markup);
     void setNewWindowRequestAllowed(bool c);
+    void setCodec(QTextCodec *codec);
 
 signals:
     void modificationChanged(bool modified);
@@ -148,6 +153,7 @@ private:
     void focusOutEvent(QFocusEvent *) override;
     void timerEvent(QTimerEvent *) override;
     void contextMenuEvent(QContextMenuEvent *event) override;
+    QChar displayChar(char ch) const;
 
     friend class BinEditorWidgetPrivate;
     BinEditorWidgetPrivate *d;

src/plugins/texteditor/codecchooser.cpp

@@ -29,9 +29,20 @@
 
 #include <QTextCodec>
 
+static bool isSingleByte(int mib)
+{
+    // Encodings are listed at https://www.iana.org/assignments/character-sets/character-sets.xhtml
+    return (mib >= 0 && mib <= 16)
+            || (mib >= 81 && mib <= 85)
+            || (mib >= 109 && mib <= 112)
+            || (mib >= 2000 && mib <= 2024)
+            || (mib >= 2028 && mib <= 2100)
+            || (mib >= 2106);
+}
+
 namespace TextEditor {
 
-CodecChooser::CodecChooser()
+CodecChooser::CodecChooser(Filter filter)
 {
     QList<int> mibs = QTextCodec::availableMibs();
     Utils::sort(mibs);
@@ -40,6 +51,8 @@ CodecChooser::CodecChooser()
     if (firstNonNegative != mibs.end())
         std::rotate(mibs.begin(), firstNonNegative, mibs.end());
     for (int mib : qAsConst(mibs)) {
+        if (filter == Filter::SingleByte && !isSingleByte(mib))
+            continue;
         if (QTextCodec *codec = QTextCodec::codecForMib(mib)) {
             QString compoundName = QLatin1String(codec->name());
             const QList<QByteArray> aliases = codec->aliases();
@@ -55,6 +68,12 @@ CodecChooser::CodecChooser()
             this, [this](int index) { emit codecChanged(m_codecs.at(index)); });
 }
 
+void CodecChooser::prependNone()
+{
+    insertItem(0, "None");
+    m_codecs.prepend(nullptr);
+}
+
 QTextCodec *CodecChooser::currentCodec() const
 {
     return codecAt(currentIndex());
@@ -73,7 +92,7 @@ void CodecChooser::setAssignedCodec(QTextCodec *codec, const QString &name)
     for (int i = 0, total = m_codecs.size(); i < total; ++i) {
         if (codec != m_codecs.at(i))
             continue;
-        if (itemText(i) == name) {
+        if (name.isEmpty() || itemText(i) == name) {
             setCurrentIndex(i);
             return;
         }

src/plugins/texteditor/codecchooser.h

@@ -40,10 +40,12 @@ class TEXTEDITOR_EXPORT CodecChooser : public QComboBox
     Q_OBJECT
 
 public:
-    CodecChooser();
+    enum class Filter { All, SingleByte };
+    explicit CodecChooser(Filter filter = Filter::All);
+    void prependNone();
     QTextCodec *currentCodec() const;
     QTextCodec *codecAt(int index) const;
-    void setAssignedCodec(QTextCodec *codec, const QString &name);
+    void setAssignedCodec(QTextCodec *codec, const QString &name = {});
     QByteArray assignedCodecName() const;
 
 signals: