QmlProfiler: Add a binary trace format

Storing traces in binary form is preferable as loading and saving is
faster and the trace files are smaller.

Change-Id: Ia7340ac526d5ce9391b1e32fc48fc1fab3ffa13d
Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>

src/plugins/qmlprofiler/qmlprofilerconstants.h

@@ -40,7 +40,8 @@ const char ANALYZER[] = "Analyzer";
 
 const int QML_MIN_LEVEL = 1; // Set to 0 to remove the empty line between models in the timeline
 
-const char TraceFileExtension[] = ".qtd";
+const char QtdFileExtension[] = ".qtd";
+const char QztFileExtension[] = ".qzt";
 
 const char QmlProfilerPerspectiveId[]  = "QmlProfiler.Perspective";
 const char QmlProfilerTimelineDockId[] = "QmlProfiler.Timeline.Dock";

src/plugins/qmlprofiler/qmlprofilerdatamodel.cpp

@@ -137,20 +137,19 @@ const QVector<QmlEventType> &QmlProfilerDataModel::eventTypes() const
     return d->eventTypes;
 }
 
-void QmlProfilerDataModel::setData(qint64 traceStart, qint64 traceEnd,
+void QmlProfilerDataModel::setEventTypes(const QVector<QmlEventType> &types)
-                                   const QVector<QmlEventType> &types,
-                                   const QVector<QmlEvent> &events)
 {
     Q_D(QmlProfilerDataModel);
-    d->modelManager->traceTime()->setTime(traceStart, traceEnd);
     d->eventTypes = types;
     for (int id = 0; id < types.count(); ++id)
         d->eventTypeIds[types[id]] = id;
+}
 
-    foreach (const QmlEvent &event, events) {
+void QmlProfilerDataModel::addEvent(const QmlEvent &event)
+{
+    Q_D(QmlProfilerDataModel);
     d->modelManager->dispatch(event, d->eventTypes[event.typeIndex()]);
     d->eventStream << event;
-    }
 }
 
 void QmlProfilerDataModel::clear()

src/plugins/qmlprofiler/qmlprofilerdatamodel.h

@@ -46,11 +46,11 @@ public:
     ~QmlProfilerDataModel();
 
     const QVector<QmlEventType> &eventTypes() const;
-    void setData(qint64 traceStart, qint64 traceEnd, const QVector<QmlEventType> &types,
+    void setEventTypes(const QVector<QmlEventType> &types);
-                 const QVector<QmlEvent> &events);
 
     void clear();
     bool isEmpty() const;
+    void addEvent(const QmlEvent &event);
     void addTypedEvent(const QmlEvent &event, const QmlEventType &type);
     void replayEvents(qint64 startTime, qint64 endTime,
                       QmlProfilerModelManager::EventLoader loader) const;

src/plugins/qmlprofiler/qmlprofilermodelmanager.cpp

@@ -310,7 +310,10 @@ void QmlProfilerModelManager::save(const QString &filename)
 
     QFuture<void> result = Utils::runAsync([file, writer] (QFutureInterface<void> &future) {
         writer->setFuture(&future);
-        writer->save(file);
+        if (file->fileName().endsWith(QLatin1String(Constants::QtdFileExtension)))
+            writer->saveQtd(file);
+        else
+            writer->saveQzt(file);
         delete writer;
         file->deleteLater();
     });
@@ -321,8 +324,9 @@ void QmlProfilerModelManager::save(const QString &filename)
 
 void QmlProfilerModelManager::load(const QString &filename)
 {
+    bool isQtd = filename.endsWith(QLatin1String(Constants::QtdFileExtension));
     QFile *file = new QFile(filename, this);
-    if (!file->open(QIODevice::ReadOnly | QIODevice::Text)) {
+    if (!file->open(isQtd ? (QIODevice::ReadOnly | QIODevice::Text) : QIODevice::ReadOnly)) {
         emit error(tr("Could not open %1 for reading.").arg(filename));
         delete file;
         emit loadFinished();
@@ -338,19 +342,28 @@ void QmlProfilerModelManager::load(const QString &filename)
         emit error(message);
     }, Qt::QueuedConnection);
 
+    connect(reader, &QmlProfilerFileReader::typesLoaded,
+            d->model, &QmlProfilerDataModel::setEventTypes);
+
+    connect(reader, &QmlProfilerFileReader::notesLoaded,
+            d->notesModel, &QmlProfilerNotesModel::setNotes);
+
+    connect(reader, &QmlProfilerFileReader::qmlEventLoaded,
+            d->model, &QmlProfilerDataModel::addEvent);
+
     connect(reader, &QmlProfilerFileReader::success, this, [this, reader]() {
-        d->model->setData(reader->traceStart(), qMax(reader->traceStart(), reader->traceEnd()),
+        d->traceTime->setTime(reader->traceStart(), reader->traceEnd());
-                          reader->eventTypes(), reader->events());
-        d->notesModel->setNotes(reader->notes());
         setRecordedFeatures(reader->loadedFeatures());
-        d->traceTime->increaseEndTime(reader->events().last().timestamp());
         delete reader;
         acquiringDone();
     }, Qt::QueuedConnection);
 
-    QFuture<void> result = Utils::runAsync([file, reader] (QFutureInterface<void> &future) {
+    QFuture<void> result = Utils::runAsync([isQtd, file, reader] (QFutureInterface<void> &future) {
         reader->setFuture(&future);
-        reader->load(file);
+        if (isQtd)
+            reader->loadQtd(file);
+        else
+            reader->loadQzt(file);
         file->close();
         file->deleteLater();
     });

src/plugins/qmlprofiler/qmlprofilertool.cpp

@@ -615,13 +615,15 @@ void saveLastTraceFile(const QString &filename)
 
 void QmlProfilerTool::showSaveDialog()
 {
+    QLatin1String tFile(QtdFileExtension);
+    QLatin1String zFile(QztFileExtension);
     QString filename = QFileDialog::getSaveFileName(
                 ICore::mainWindow(), tr("Save QML Trace"),
                 QmlProfilerPlugin::globalSettings()->lastTraceFile(),
-                tr("QML traces (*%1)").arg(QLatin1String(TraceFileExtension)));
+                tr("QML traces (*%1 *%2)").arg(zFile).arg(tFile));
     if (!filename.isEmpty()) {
-        if (!filename.endsWith(QLatin1String(TraceFileExtension)))
+        if (!filename.endsWith(zFile) && !filename.endsWith(tFile))
-            filename += QLatin1String(TraceFileExtension);
+            filename += zFile;
         saveLastTraceFile(filename);
         Debugger::enableMainWindow(false);
         d->m_profilerModelManager->save(filename);
@@ -635,10 +637,12 @@ void QmlProfilerTool::showLoadDialog()
 
     Debugger::selectPerspective(QmlProfilerPerspectiveId);
 
+    QLatin1String tFile(QtdFileExtension);
+    QLatin1String zFile(QztFileExtension);
     QString filename = QFileDialog::getOpenFileName(
                 ICore::mainWindow(), tr("Load QML Trace"),
                 QmlProfilerPlugin::globalSettings()->lastTraceFile(),
-                tr("QML traces (*%1)").arg(QLatin1String(TraceFileExtension)));
+                tr("QML traces (*%1 *%2)").arg(zFile).arg(tFile));
 
     if (!filename.isEmpty()) {
         saveLastTraceFile(filename);

src/plugins/qmlprofiler/qmlprofilertracefile.cpp

@@ -33,6 +33,9 @@
 #include <QXmlStreamWriter>
 #include <QStack>
 #include <QDebug>
+#include <QFile>
+#include <QBuffer>
+#include <QDataStream>
 
 namespace QmlProfiler {
 namespace Internal {
@@ -119,6 +122,12 @@ QmlProfilerFileReader::QmlProfilerFileReader(QObject *parent) :
     m_future(0),
     m_loadedFeatures(0)
 {
+    static int meta[] = {
+        qRegisterMetaType<QmlEvent>(),
+        qRegisterMetaType<QVector<QmlEventType> >(),
+        qRegisterMetaType<QVector<QmlNote> >()
+    };
+    Q_UNUSED(meta);
 }
 
 void QmlProfilerFileReader::setFuture(QFutureInterface<void> *future)
@@ -126,7 +135,7 @@ void QmlProfilerFileReader::setFuture(QFutureInterface<void> *future)
     m_future = future;
 }
 
-bool QmlProfilerFileReader::load(QIODevice *device)
+bool QmlProfilerFileReader::loadQtd(QIODevice *device)
 {
     if (m_future) {
         m_future->setProgressRange(0, 1000);
@@ -159,6 +168,7 @@ bool QmlProfilerFileReader::load(QIODevice *device)
 
             if (elementName == _("eventData")) {
                 loadEventTypes(stream);
+                emit typesLoaded(m_eventTypes);
                 break;
             }
 
@@ -169,6 +179,7 @@ bool QmlProfilerFileReader::load(QIODevice *device)
 
             if (elementName == _("noteData")) {
                 loadNotes(stream);
+                emit notesLoaded(m_notes);
                 break;
             }
 
@@ -182,47 +193,98 @@ bool QmlProfilerFileReader::load(QIODevice *device)
         emit error(tr("Error while parsing trace data file: %1").arg(stream.errorString()));
         return false;
     } else {
-        std::sort(m_events.begin(), m_events.end(), [](const QmlEvent &a, const QmlEvent &b) {
-            return a.timestamp() < b.timestamp();
-        });
         emit success();
         return true;
     }
 }
 
+bool QmlProfilerFileReader::loadQzt(QIODevice *device)
+{
+    if (m_future) {
+        m_future->setProgressRange(0, 1000);
+        m_future->setProgressValue(0);
+    }
+
+    QDataStream stream(device);
+    stream.setVersion(QDataStream::Qt_5_5);
+
+    QByteArray magic;
+    stream >> magic;
+    if (magic != QByteArray("QMLPROFILER")) {
+        emit error(tr("Invalid magic: %1").arg(QLatin1String(magic)));
+        return false;
+    }
+
+    qint32 dataStreamVersion;
+    stream >> dataStreamVersion;
+
+    if (dataStreamVersion > QDataStream::Qt_DefaultCompiledVersion) {
+        emit error(tr("Unknown data stream version: %1").arg(dataStreamVersion));
+        return false;
+    }
+    stream.setVersion(dataStreamVersion);
+
+    stream >> m_traceStart >> m_traceEnd;
+
+    QBuffer buffer;
+    QDataStream bufferStream(&buffer);
+    bufferStream.setVersion(dataStreamVersion);
+    QByteArray data;
+    updateProgress(device);
+
+    stream >> data;
+    buffer.setData(qUncompress(data));
+    buffer.open(QIODevice::ReadOnly);
+    bufferStream >> m_eventTypes;
+    buffer.close();
+    emit typesLoaded(m_eventTypes);
+    updateProgress(device);
+
+    stream >> data;
+    buffer.setData(qUncompress(data));
+    buffer.open(QIODevice::ReadOnly);
+    bufferStream >> m_notes;
+    buffer.close();
+    emit notesLoaded(m_notes);
+    updateProgress(device);
+
+    QmlEvent event;
+    while (!stream.atEnd()) {
+        stream >> data;
+        buffer.setData(qUncompress(data));
+        buffer.open(QIODevice::ReadOnly);
+        while (!buffer.atEnd()) {
+            if (isCanceled())
+                return false;
+            bufferStream >> event;
+            if (bufferStream.status() == QDataStream::Ok) {
+                if (event.typeIndex() >= m_eventTypes.length()) {
+                    emit error(tr("Invalid type index %1").arg(event.typeIndex()));
+                    return false;
+                }
+                m_loadedFeatures |= (1ULL << m_eventTypes[event.typeIndex()].feature());
+                emit qmlEventLoaded(event);
+            } else if (bufferStream.status() == QDataStream::ReadPastEnd) {
+                break; // Apparently EOF is a character so we end up here after the last event.
+            } else if (bufferStream.status() == QDataStream::ReadCorruptData) {
+                emit error(tr("Corrupt data before position %1.").arg(device->pos()));
+                return false;
+            } else {
+                Q_UNREACHABLE();
+            }
+        }
+        buffer.close();
+        updateProgress(device);
+    }
+    emit success();
+    return true;
+}
+
 quint64 QmlProfilerFileReader::loadedFeatures() const
 {
     return m_loadedFeatures;
 }
 
-ProfileFeature featureFromType(const QmlEventType &type) {
-    if (type.rangeType < MaximumRangeType)
-        return featureFromRangeType(type.rangeType);
-
-    switch (type.message) {
-    case Event:
-        switch (type.detailType) {
-        case AnimationFrame:
-            return ProfileAnimations;
-        case Key:
-        case Mouse:
-            return ProfileInputEvents;
-        default:
-            return MaximumProfileFeature;
-        }
-    case PixmapCacheEvent:
-        return ProfilePixmapCache;
-    case SceneGraphFrame:
-        return ProfileSceneGraph;
-    case MemoryAllocation:
-        return ProfileMemory;
-    case DebugMessage:
-        return ProfileDebugMessages;
-    default:
-        return MaximumProfileFeature;
-    }
-}
-
 void QmlProfilerFileReader::loadEventTypes(QXmlStreamReader &stream)
 {
     QTC_ASSERT(stream.name() == _("eventData"), return);
@@ -248,7 +310,7 @@ void QmlProfilerFileReader::loadEventTypes(QXmlStreamReader &stream)
         switch (token) {
         case QXmlStreamReader::StartElement: {
             if (elementName == _("event")) {
-                progress(stream.device());
+                updateProgress(stream.device());
                 type = defaultEvent;
 
                 const QXmlStreamAttributes attributes = stream.attributes();
@@ -331,7 +393,7 @@ void QmlProfilerFileReader::loadEventTypes(QXmlStreamReader &stream)
                     if (typeIndex >= m_eventTypes.size())
                         m_eventTypes.resize(typeIndex + 1);
                     m_eventTypes[typeIndex] = type;
-                    ProfileFeature feature = featureFromType(type);
+                    ProfileFeature feature = type.feature();
                     if (feature != MaximumProfileFeature)
                         m_loadedFeatures |= (1ULL << static_cast<uint>(feature));
                 }
@@ -352,6 +414,7 @@ void QmlProfilerFileReader::loadEventTypes(QXmlStreamReader &stream)
 void QmlProfilerFileReader::loadEvents(QXmlStreamReader &stream)
 {
     QTC_ASSERT(stream.name() == _("profilerDataModel"), return);
+    QVector<QmlEvent> events;
 
     while (!stream.atEnd() && !stream.hasError()) {
         if (isCanceled())
@@ -363,7 +426,7 @@ void QmlProfilerFileReader::loadEvents(QXmlStreamReader &stream)
         switch (token) {
         case QXmlStreamReader::StartElement: {
             if (elementName == _("range")) {
-                progress(stream.device());
+                updateProgress(stream.device());
                 QmlEvent event;
 
                 const QXmlStreamAttributes attributes = stream.attributes();
@@ -378,12 +441,12 @@ void QmlProfilerFileReader::loadEvents(QXmlStreamReader &stream)
 
                 if (attributes.hasAttribute(_("duration"))) {
                     event.setRangeStage(RangeStart);
-                    m_events.append(event);
+                    events.append(event);
                     QmlEvent rangeEnd(event);
                     rangeEnd.setRangeStage(RangeEnd);
                     rangeEnd.setTimestamp(event.timestamp()
                                           + attributes.value(_("duration")).toLongLong());
-                    m_events.append(rangeEnd);
+                    events.append(rangeEnd);
                 } else {
                     // attributes for special events
                     if (attributes.hasAttribute(_("framerate")))
@@ -419,7 +482,7 @@ void QmlProfilerFileReader::loadEvents(QXmlStreamReader &stream)
                     if (attributes.hasAttribute(_("text")))
                         event.setString(attributes.value(_("text")).toString());
 
-                    m_events.append(event);
+                    events.append(event);
                 }
             }
             break;
@@ -427,6 +490,11 @@ void QmlProfilerFileReader::loadEvents(QXmlStreamReader &stream)
         case QXmlStreamReader::EndElement: {
             if (elementName == _("profilerDataModel")) {
                 // done reading profilerDataModel
+                std::sort(events.begin(), events.end(), [](const QmlEvent &a, const QmlEvent &b) {
+                    return a.timestamp() < b.timestamp();
+                });
+                foreach (const QmlEvent &event, events)
+                    emit qmlEventLoaded(event);
                 return;
             }
             break;
@@ -449,7 +517,7 @@ void QmlProfilerFileReader::loadNotes(QXmlStreamReader &stream)
         switch (token) {
         case QXmlStreamReader::StartElement: {
             if (elementName == _("note")) {
-                progress(stream.device());
+                updateProgress(stream.device());
                 QXmlStreamAttributes attrs = stream.attributes();
                 currentNote.startTime = attrs.value(_("startTime")).toLongLong();
                 currentNote.duration = attrs.value(_("duration")).toLongLong();
@@ -475,7 +543,7 @@ void QmlProfilerFileReader::loadNotes(QXmlStreamReader &stream)
     }
 }
 
-void QmlProfilerFileReader::progress(QIODevice *device)
+void QmlProfilerFileReader::updateProgress(QIODevice *device)
 {
     if (!m_future)
         return;
@@ -519,7 +587,7 @@ void QmlProfilerFileWriter::setFuture(QFutureInterface<void> *future)
     m_future = future;
 }
 
-void QmlProfilerFileWriter::save(QIODevice *device)
+void QmlProfilerFileWriter::saveQtd(QIODevice *device)
 {
     if (m_future) {
         m_future->setProgressRange(0, qMax(m_model->eventTypes().size() + m_notes.size(), 1));
@@ -689,6 +757,52 @@ void QmlProfilerFileWriter::save(QIODevice *device)
     stream.writeEndDocument();
 }
 
+void QmlProfilerFileWriter::saveQzt(QFile *file)
+{
+    QDataStream stream(file);
+    stream.setVersion(QDataStream::Qt_5_5);
+    stream << QByteArray("QMLPROFILER");
+    stream << static_cast<qint32>(QDataStream::Qt_DefaultCompiledVersion);
+    stream.setVersion(QDataStream::Qt_DefaultCompiledVersion);
+
+    stream << m_startTime << m_endTime;
+
+    QBuffer buffer;
+    QDataStream bufferStream(&buffer);
+    buffer.open(QIODevice::WriteOnly);
+
+    incrementProgress();
+    buffer.open(QIODevice::WriteOnly);
+    bufferStream << m_model->eventTypes();
+    stream << qCompress(buffer.data());
+    buffer.close();
+    buffer.buffer().clear();
+    incrementProgress();
+    buffer.open(QIODevice::WriteOnly);
+    bufferStream << m_notes;
+    stream << qCompress(buffer.data());
+    buffer.close();
+    buffer.buffer().clear();
+    incrementProgress();
+
+    buffer.open(QIODevice::WriteOnly);
+    m_model->replayEvents(-1, -1, [&stream, &buffer, &bufferStream](const QmlEvent &event,
+                                                                    const QmlEventType &type) {
+        Q_UNUSED(type);
+        bufferStream << event;
+        // 32MB buffer should be plenty for efficient compression
+        if (buffer.data().length() > (1 << 25)) {
+            stream << qCompress(buffer.data());
+            buffer.close();
+            buffer.buffer().clear();
+            buffer.open(QIODevice::WriteOnly);
+        }
+    });
+    stream << qCompress(buffer.data());
+    buffer.close();
+    buffer.buffer().clear();
+}
+
 void QmlProfilerFileWriter::incrementProgress()
 {
     if (!m_future)

src/plugins/qmlprofiler/qmlprofilertracefile.h

@@ -35,6 +35,7 @@
 #include <QVector>
 #include <QString>
 
+QT_FORWARD_DECLARE_CLASS(QFile)
 QT_FORWARD_DECLARE_CLASS(QIODevice)
 QT_FORWARD_DECLARE_CLASS(QXmlStreamReader)
 
@@ -50,17 +51,18 @@ public:
 
     void setFuture(QFutureInterface<void> *future);
 
-    bool load(QIODevice *device);
+    bool loadQtd(QIODevice *device);
+    bool loadQzt(QIODevice *device);
+
     quint64 loadedFeatures() const;
 
     qint64 traceStart() const { return m_traceStart; }
     qint64 traceEnd() const { return m_traceEnd; }
 
-    const QVector<QmlEventType> &eventTypes() const { return m_eventTypes; }
-    const QVector<QmlEvent> &events() const { return m_events; }
-    const QVector<QmlNote> &notes() const { return m_notes; }
-
 signals:
+    void typesLoaded(const QVector<QmlProfiler::QmlEventType> &types);
+    void notesLoaded(const QVector<QmlProfiler::QmlNote> &notes);
+    void qmlEventLoaded(const QmlProfiler::QmlEvent &event);
     void error(const QString &error);
     void success();
 
@@ -68,13 +70,12 @@ private:
     void loadEventTypes(QXmlStreamReader &reader);
     void loadEvents(QXmlStreamReader &reader);
     void loadNotes(QXmlStreamReader &reader);
-    void progress(QIODevice *device);
+    void updateProgress(QIODevice *device);
     bool isCanceled() const;
 
     qint64 m_traceStart, m_traceEnd;
     QFutureInterface<void> *m_future;
     QVector<QmlEventType> m_eventTypes;
-    QVector<QmlEvent> m_events;
     QVector<QmlNote> m_notes;
     quint64 m_loadedFeatures;
 };
@@ -92,7 +93,8 @@ public:
     void setNotes(const QVector<QmlNote> &notes);
     void setFuture(QFutureInterface<void> *future);
 
-    void save(QIODevice *device);
+    void saveQtd(QIODevice *device);
+    void saveQzt(QFile *file);
 
 private:
     void calculateMeasuredTime();