serialize AST into a single string

this saves quite some mallocs in the parsing pass.

src/shared/proparser/profileevaluator.h

@@ -50,9 +50,14 @@ class ProFileEvaluator
     class Private;
 
 public:
+    struct FunctionDef {
+        QString string;
+        int offset;
+    };
+
     struct FunctionDefs {
-        QHash<ProString, ProFunctionDef *> testFunctions;
+        QHash<ProString, FunctionDef> testFunctions;
-        QHash<ProString, ProFunctionDef *> replaceFunctions;
+        QHash<ProString, FunctionDef> replaceFunctions;
     };
 
     enum TemplateType {
@@ -109,6 +114,8 @@ private:
     friend class ProFileCache;
 };
 
+Q_DECLARE_TYPEINFO(ProFileEvaluator::FunctionDef, Q_MOVABLE_TYPE);
+
 class ProFileCache
 {
 public:

src/shared/proparser/proitems.cpp

@@ -37,7 +37,7 @@ QT_BEGIN_NAMESPACE
 using namespace ProStringConstants;
 
 // from qhash.cpp
-static uint hash(const QChar *p, int n)
+uint ProString::hash(const QChar *p, int n)
 {
     uint h = 0;
 
@@ -86,6 +86,22 @@ ProString::ProString(const char *str, OmitPreHashing) :
 {
 }
 
+ProString::ProString(const QString &str, int offset, int length) :
+        m_string(str), m_offset(offset), m_length(length)
+{
+    updatedHash();
+}
+
+ProString::ProString(const QString &str, int offset, int length, uint hash) :
+    m_string(str), m_offset(offset), m_length(length), m_hash(hash)
+{
+}
+
+ProString::ProString(const QString &str, int offset, int length, ProStringConstants::OmitPreHashing) :
+    m_string(str), m_offset(offset), m_length(length), m_hash(0x80000000)
+{
+}
+
 uint ProString::updatedHash() const
 {
      return (m_hash = hash(m_string.constData() + m_offset, m_length));
@@ -231,44 +247,8 @@ void ProStringList::removeDuplicates()
         erase(begin() + j, end());
 }
 
-void ProItem::disposeItems(ProItem *nitm)
-{
-    for (ProItem *itm; (itm = nitm); ) {
-        nitm = itm->next();
-        switch (itm->kind()) {
-        case ProItem::ConditionKind: delete static_cast<ProCondition *>(itm); break;
-        case ProItem::VariableKind: delete static_cast<ProVariable *>(itm); break;
-        case ProItem::BranchKind: delete static_cast<ProBranch *>(itm); break;
-        case ProItem::LoopKind: delete static_cast<ProLoop *>(itm); break;
-        case ProItem::FunctionDefKind: static_cast<ProFunctionDef *>(itm)->deref(); break;
-        case ProItem::OpNotKind:
-        case ProItem::OpAndKind:
-        case ProItem::OpOrKind:
-            delete itm;
-            break;
-        }
-    }
-}
-
-ProBranch::~ProBranch()
-{
-    disposeItems(m_thenItems);
-    disposeItems(m_elseItems);
-}
-
-ProLoop::~ProLoop()
-{
-    disposeItems(m_proitems);
-}
-
-ProFunctionDef::~ProFunctionDef()
-{
-    disposeItems(m_proitems);
-}
-
 ProFile::ProFile(const QString &fileName)
-    : m_proitems(0),
+    : m_refCount(1),
-      m_refCount(1),
       m_fileName(fileName)
 {
     int nameOff = fileName.lastIndexOf(QLatin1Char('/'));
@@ -279,7 +259,6 @@ ProFile::ProFile(const QString &fileName)
 
 ProFile::~ProFile()
 {
-    ProItem::disposeItems(m_proitems);
 }
 
 QT_END_NAMESPACE

src/shared/proparser/proitems.h

@@ -62,6 +62,9 @@ public:
     ProString(const QString &str, ProStringConstants::OmitPreHashing);
     explicit ProString(const char *str);
     ProString(const char *str, ProStringConstants::OmitPreHashing);
+    ProString(const QString &str, int offset, int length);
+    ProString(const QString &str, int offset, int length, uint hash);
+    ProString(const QString &str, int offset, int length, ProStringConstants::OmitPreHashing);
     QString toQString() const;
     QString &toQString(QString &tmp) const;
     bool operator==(const ProString &other) const;
@@ -79,6 +82,8 @@ public:
     ProString trimmed() const;
     void clear() { m_string.clear(); m_length = 0; }
 
+    static uint hash(const QChar *p, int n);
+
 private:
     QString m_string;
     int m_offset, m_length;
@@ -104,130 +109,38 @@ public:
     void removeDuplicates();
 };
 
-class ProItem
+// These token definitions affect both ProFileEvaluator and ProWriter
-{
+enum ProToken {
-public:
+    TokTerminator = 0,  // end of stream (possibly not included in length; must be zero)
-    enum ProItemKind {
+    TokLine,            // line marker: // +1 (2-nl) to 1st token of each line
-        ConditionKind,
+                        // - line (1)
-        OpNotKind,
+    TokAssign,          // variable =   // "A=":2 => 1+4+2=7 (8)
-        OpAndKind,
+    TokAppend,          // variable +=  // "A+=":3 => 1+4+2=7 (8)
-        OpOrKind,
+    TokAppendUnique,    // variable *=  // "A*=":3 => 1+4+2=7 (8)
-        VariableKind,
+    TokRemove,          // variable -=  // "A-=":3 => 1+4+2=7 (8)
-        BranchKind,
+    TokReplace,         // variable ~=  // "A~=":3 => 1+4+2=7 (8)
-        LoopKind,
+                        // - variable name: hash (2), length (1), chars (length)
-        FunctionDefKind
+                        // - expression: length (2), chars (length)
-    };
+    TokCondition,       // CONFIG test:   // "A":1 => 1+2=3 (4)
-
+                        // - test name: lenght (1), chars (length)
-    ProItem(ProItemKind kind) : m_kind(kind), m_lineNumber(0) {}
+    TokNot,             // '!' operator
-
+    TokAnd,             // ':' operator
-    ProItemKind kind() const { return m_kind; }
+    TokOr,              // '|' operator
-
+    TokBranch,          // branch point:   // "X:A=":4 => [5]+1+4+1+1+[7]=19 (20)
-    int lineNumber() const { return m_lineNumber; }
+                        // - then block length (2)
-    void setLineNumber(int lineNumber) { m_lineNumber = lineNumber; }
+                        // - then block + TokTerminator (then block length)
-
+                        // - else block length (2)
-    ProItem *next() const { return m_next; }
+                        // - else block + TokTerminator (else block length)
-    ProItem **nextRef() { return &m_next; }
+    TokForLoop,         // for loop:   // "for(A,B)":8 => 1+4+3+2+1=11 (12)
-
+                        // - variable name: hash (2), length (1), chars (length)
-    static void disposeItems(ProItem *nitm);
+                        // - expression: length (2), chars (length)
-
+                        // - body length (2)
-private:
+                        // - body + TokTerminator (body length)
-    ProItem *m_next;
+    TokTestDef,         // test function definition:     // "defineTest(A):":14 => 1+4+2+1=8 (9)
-    ProItemKind m_kind;
+    TokReplaceDef,      // replace function definition:  // "defineReplace(A):":17 => 1+4+2+1=8 (9)
-    int m_lineNumber;
+                        // - function name: hash (2), length (1), chars (length)
-};
+                        // - body length (2)
-
+                        // - body + TokTerminator (body length)
-class ProVariable : public ProItem
-{
-public:
-    enum VariableOperator {
-        AddOperator         = 0,
-        RemoveOperator      = 1,
-        ReplaceOperator     = 2,
-        SetOperator         = 3,
-        UniqueAddOperator   = 4
-    };
-
-    ProVariable(const ProString &name) : ProItem(VariableKind),
-            m_variableKind(SetOperator), m_variable(name) {}
-    void setVariableOperator(VariableOperator variableKind) { m_variableKind = variableKind; }
-    VariableOperator variableOperator() const { return m_variableKind; }
-    ProString variable() const { return m_variable; }
-    void setValue(const ProString &value) { m_value = value; }
-    ProString value() const { return m_value; }
-
-private:
-    VariableOperator m_variableKind;
-    ProString m_variable;
-    ProString m_value;
-};
-
-class ProCondition : public ProItem
-{
-public:
-    explicit ProCondition(const QString &text) : ProItem(ConditionKind), m_text(text) {}
-    QString text() const { return m_text; }
-
-private:
-    QString m_text;
-};
-
-class ProBranch : public ProItem
-{
-public:
-    ProBranch() : ProItem(BranchKind) {}
-    ~ProBranch();
-
-    ProItem *thenItems() const { return m_thenItems; }
-    ProItem **thenItemsRef() { return &m_thenItems; }
-    ProItem *elseItems() const { return m_elseItems; }
-    ProItem **elseItemsRef() { return &m_elseItems; }
-
-private:
-    ProItem *m_thenItems;
-    ProItem *m_elseItems;
-};
-
-class ProLoop : public ProItem
-{
-public:
-    ProLoop(const QString &var, const QString &expr)
-        : ProItem(LoopKind), m_variable(ProString(var)),
-          m_expression(ProString(expr, ProStringConstants::NoHash)) {}
-    ~ProLoop();
-
-    ProString variable() const { return m_variable; }
-    ProString expression() const { return m_expression; }
-    ProItem *items() const { return m_proitems; }
-    ProItem **itemsRef() { return &m_proitems; }
-
-private:
-    ProString m_variable;
-    ProString m_expression;
-    ProItem *m_proitems;
-};
-
-class ProFunctionDef : public ProItem
-{
-public:
-    enum FunctionType { TestFunction, ReplaceFunction };
-
-    ProFunctionDef(const QString &name, FunctionType type)
-        : ProItem(FunctionDefKind), m_name(ProString(name)), m_type(type), m_refCount(1) {}
-    ~ProFunctionDef();
-
-    ProString name() const { return m_name; }
-    FunctionType type() const { return m_type; }
-    ProItem *items() const { return m_proitems; }
-    ProItem **itemsRef() { return &m_proitems; }
-
-    void ref() { m_refCount.ref(); }
-    void deref() { if (!m_refCount.deref()) delete this; }
-
-private:
-    ProString m_name;
-    FunctionType m_type;
-    ProItemRefCount m_refCount;
-    ProItem *m_proitems;
 };
 
 class ProFile
@@ -239,15 +152,15 @@ public:
     QString displayFileName() const { return m_displayFileName; }
     QString fileName() const { return m_fileName; }
     QString directoryName() const { return m_directoryName; }
-    ProItem *items() const { return m_proitems; }
+    const QString &items() const { return m_proitems; }
-    ProItem **itemsRef() { return &m_proitems; }
+    QString *itemsRef() { return &m_proitems; }
 
     void ref() { m_refCount.ref(); }
     void deref() { if (!m_refCount.deref()) delete this; }
 
 private:
-    ProItem *m_proitems;
     ProItemRefCount m_refCount;
+    QString m_proitems;
     QString m_fileName;
     QString m_displayFileName;
     QString m_directoryName;

src/shared/proparser/prowriter.cpp

@@ -35,20 +35,98 @@
 
 using namespace Qt4ProjectManager::Internal;
 
+static uint getBlockLen(const ushort *&tokPtr)
+{
+    uint len = *tokPtr++;
+    len |= (uint)*tokPtr++ << 16;
+    return len;
+}
+
+static QString &getHashStr(const ushort *&tokPtr, QString &tmp)
+{
+    tokPtr += 2; // ignore hash
+    uint len = *tokPtr++;
+    tmp.setRawData((const QChar *)tokPtr, len);
+    tokPtr += len;
+    return tmp;
+}
+
+static void skipStr(const ushort *&tokPtr)
+{
+    uint len = *tokPtr++;
+    tokPtr += len;
+}
+
+static void skipHashStr(const ushort *&tokPtr)
+{
+    tokPtr += 2;
+    uint len = *tokPtr++;
+    tokPtr += len;
+}
+
+static void skipLongStr(const ushort *&tokPtr)
+{
+    uint len = getBlockLen(tokPtr);
+    tokPtr += len;
+}
+
+static void skipBlock(const ushort *&tokPtr)
+{
+    uint len = getBlockLen(tokPtr);
+    tokPtr += len;
+}
+
+static void skipToken(ushort tok, const ushort *&tokPtr, int &lineNo)
+{
+    switch (tok) {
+    case TokLine:
+        lineNo = *tokPtr++;
+        break;
+    case TokAssign:
+    case TokAppend:
+    case TokAppendUnique:
+    case TokRemove:
+    case TokReplace:
+        skipHashStr(tokPtr);
+        skipLongStr(tokPtr);
+        break;
+    case TokBranch:
+        skipBlock(tokPtr);
+        skipBlock(tokPtr);
+        break;
+    case TokForLoop:
+        skipHashStr(tokPtr);
+        skipLongStr(tokPtr);
+        skipBlock(tokPtr);
+        break;
+    case TokTestDef:
+    case TokReplaceDef:
+        skipHashStr(tokPtr);
+        skipBlock(tokPtr);
+        break;
+    case TokNot:
+    case TokAnd:
+    case TokOr:
+        break;
+    case TokCondition:
+        skipStr(tokPtr);
+        break;
+    default: Q_ASSERT_X(false, "skipToken", "unexpected item type");
+    }
+}
+
 void ProWriter::addFiles(ProFile *profile, QStringList *lines,
                          const QDir &proFileDir, const QStringList &filePaths,
                          const QString &var)
 {
     // Check if variable item exists as child of root item
-    for (ProItem *item = profile->items(); item; item = item->next()) {
+    const ushort *tokPtr = (const ushort *)profile->items().constData();
-        if (item->kind() == ProItem::VariableKind) {
+    int lineNo = 0;
-            ProVariable *proVar = static_cast<ProVariable*>(item);
+    QString tmp;
-            if (var == proVar->variable().toQString()
+    while (ushort tok = *tokPtr++) {
-                && proVar->variableOperator() != ProVariable::RemoveOperator
+        if (tok == TokAssign || tok == TokAppend || tok == TokAppendUnique) {
-                && proVar->variableOperator() != ProVariable::ReplaceOperator) {
+            if (var == getHashStr(tokPtr, tmp)) {
-
+                for (--lineNo; lineNo < lines->count(); lineNo++) {
-                int lineNo = proVar->lineNumber() - 1;
-                for (; lineNo < lines->count(); lineNo++) {
                     QString line = lines->at(lineNo);
                     int idx = line.indexOf(QLatin1Char('#'));
                     if (idx >= 0)
@@ -74,6 +152,9 @@ void ProWriter::addFiles(ProFile *profile, QStringList *lines,
                 lines->insert(lineNo, added);
                 return;
             }
+            skipLongStr(tokPtr);
+        } else {
+            skipToken(tok, tokPtr, lineNo);
         }
     }
 
@@ -84,17 +165,25 @@ void ProWriter::addFiles(ProFile *profile, QStringList *lines,
     *lines << added;
 }
 
-static void findProVariables(ProItem *item, const QStringList &vars,
+static void findProVariables(const ushort *tokPtr, const QStringList &vars,
-                             QList<ProVariable *> *proVars)
+                             QList<int> *proVars)
 {
-    for (; item; item = item->next()) {
+    int lineNo = 0;
-        if (item->kind() == ProItem::BranchKind) {
+    QString tmp;
-            findProVariables(static_cast<ProBranch*>(item)->thenItems(), vars, proVars);
+    while (ushort tok = *tokPtr++) {
-            findProVariables(static_cast<ProBranch*>(item)->elseItems(), vars, proVars);
+        if (tok == TokBranch) {
-        } else if (item->kind() == ProItem::VariableKind) {
+            uint blockLen = getBlockLen(tokPtr);
-            ProVariable *proVar = static_cast<ProVariable*>(item);
+            findProVariables(tokPtr, vars, proVars);
-            if (vars.contains(proVar->variable().toQString()))
+            tokPtr += blockLen;
-                *proVars << proVar;
+            blockLen = getBlockLen(tokPtr);
+            findProVariables(tokPtr, vars, proVars);
+            tokPtr += blockLen;
+        } else if (tok == TokAssign || tok == TokAppend || tok == TokAppendUnique) {
+            if (vars.contains(getHashStr(tokPtr, tmp)))
+                *proVars << lineNo;
+            skipLongStr(tokPtr);
+        } else {
+            skipToken(tok, tokPtr, lineNo);
         }
     }
 }
@@ -105,8 +194,8 @@ QStringList ProWriter::removeFiles(ProFile *profile, QStringList *lines,
 {
     QStringList notChanged = filePaths;
 
-    QList<ProVariable *> proVars;
+    QList<int> varLines;
-    findProVariables(profile->items(), vars, &proVars);
+    findProVariables((const ushort *)profile->items().constData(), vars, &varLines);
 
     // This is a tad stupid - basically, it can remove only entries which
     // the above code added.
@@ -116,109 +205,105 @@ QStringList ProWriter::removeFiles(ProFile *profile, QStringList *lines,
 
     // This code expects proVars to be sorted by the variables' appearance in the file.
     int delta = 1;
-    foreach (ProVariable *proVar, proVars) {
+    foreach (int ln, varLines) {
-        if (proVar->variableOperator() != ProVariable::RemoveOperator
+       bool first = true;
-            && proVar->variableOperator() != ProVariable::ReplaceOperator) {
+       int lineNo = ln - delta;
-
+       typedef QPair<int, int> ContPos;
-            bool first = true;
+       QList<ContPos> contPos;
-            int lineNo = proVar->lineNumber() - delta;
+       while (lineNo < lines->count()) {
-            typedef QPair<int, int> ContPos;
+           QString &line = (*lines)[lineNo];
-            QList<ContPos> contPos;
+           int lineLen = line.length();
-            while (lineNo < lines->count()) {
+           bool killed = false;
-                QString &line = (*lines)[lineNo];
+           bool saved = false;
-                int lineLen = line.length();
+           int idx = line.indexOf(QLatin1Char('#'));
-                bool killed = false;
+           if (idx >= 0)
-                bool saved = false;
+               lineLen = idx;
-                int idx = line.indexOf(QLatin1Char('#'));
+           QChar *chars = line.data();
-                if (idx >= 0)
+           forever {
-                    lineLen = idx;
+               if (!lineLen) {
-                QChar *chars = line.data();
+                   if (idx >= 0)
-                forever {
+                       goto nextLine;
-                    if (!lineLen) {
+                   goto nextVar;
-                        if (idx >= 0)
+               }
-                            goto nextLine;
+               QChar c = chars[lineLen - 1];
-                        goto nextVar;
+               if (c != QLatin1Char(' ') && c != QLatin1Char('\t'))
-                    }
+                   break;
-                    QChar c = chars[lineLen - 1];
+               lineLen--;
-                    if (c != QLatin1Char(' ') && c != QLatin1Char('\t'))
+           }
-                        break;
+           {
-                    lineLen--;
+               int contCol = -1;
-                }
+               if (chars[lineLen - 1] == QLatin1Char('\\'))
-                {
+                   contCol = --lineLen;
-                    int contCol = -1;
+               int colNo = 0;
-                    if (chars[lineLen - 1] == QLatin1Char('\\'))
+               if (first) {
-                        contCol = --lineLen;
+                   colNo = line.indexOf(QLatin1Char('=')) + 1;
-                    int colNo = 0;
+                   first = false;
-                    if (first) {
+                   saved = true;
-                        colNo = line.indexOf(QLatin1Char('=')) + 1;
+               }
-                        first = false;
+               while (colNo < lineLen) {
-                        saved = true;
+                   QChar c = chars[colNo];
-                    }
+                   if (c == QLatin1Char(' ') || c == QLatin1Char('\t')) {
-                    while (colNo < lineLen) {
+                       colNo++;
-                        QChar c = chars[colNo];
+                       continue;
-                        if (c == QLatin1Char(' ') || c == QLatin1Char('\t')) {
+                   }
-                            colNo++;
+                   int varCol = colNo;
-                            continue;
+                   while (colNo < lineLen) {
-                        }
+                       QChar c = chars[colNo];
-                        int varCol = colNo;
+                       if (c == QLatin1Char(' ') || c == QLatin1Char('\t'))
-                        while (colNo < lineLen) {
+                           break;
-                            QChar c = chars[colNo];
+                       colNo++;
-                            if (c == QLatin1Char(' ') || c == QLatin1Char('\t'))
+                   }
-                                break;
+                   QString fn = line.mid(varCol, colNo - varCol);
-                            colNo++;
+                   if (relativeFilePaths.contains(fn)) {
-                        }
+                       notChanged.removeOne(QDir::cleanPath(proFileDir.absoluteFilePath(fn)));
-                        QString fn = line.mid(varCol, colNo - varCol);
+                       if (colNo < lineLen)
-                        if (relativeFilePaths.contains(fn)) {
+                           colNo++;
-                            notChanged.removeOne(QDir::cleanPath(proFileDir.absoluteFilePath(fn)));
+                       else if (varCol)
-                            if (colNo < lineLen)
+                           varCol--;
-                                colNo++;
+                       int len = colNo - varCol;
-                            else if (varCol)
+                       colNo = varCol;
-                                varCol--;
+                       line.remove(varCol, len);
-                            int len = colNo - varCol;
+                       lineLen -= len;
-                            colNo = varCol;
+                       contCol -= len;
-                            line.remove(varCol, len);
+                       idx -= len;
-                            lineLen -= len;
+                       if (idx >= 0)
-                            contCol -= len;
+                           line.insert(idx, QLatin1String("# ") + fn + QLatin1Char(' '));
-                            idx -= len;
+                       killed = true;
-                            if (idx >= 0)
+                   } else {
-                                line.insert(idx, QLatin1String("# ") + fn + QLatin1Char(' '));
+                       saved = true;
-                            killed = true;
+                   }
-                        } else {
+               }
-                            saved = true;
+               if (saved) {
-                        }
+                   // Entries remained
-                    }
+                   contPos.clear();
-                    if (saved) {
+               } else if (killed) {
-                        // Entries remained
+                   // Entries existed, but were all removed
-                        contPos.clear();
+                   if (contCol < 0) {
-                    } else if (killed) {
+                       // This is the last line, so clear continuations leading to it
-                        // Entries existed, but were all removed
+                       foreach (const ContPos &pos, contPos) {
-                        if (contCol < 0) {
+                           QString &bline = (*lines)[pos.first];
-                            // This is the last line, so clear continuations leading to it
+                           bline.remove(pos.second, 1);
-                            foreach (const ContPos &pos, contPos) {
+                           if (pos.second == bline.length())
-                                QString &bline = (*lines)[pos.first];
+                               while (bline.endsWith(QLatin1Char(' '))
-                                bline.remove(pos.second, 1);
+                                      || bline.endsWith(QLatin1Char('\t')))
-                                if (pos.second == bline.length())
+                                   bline.chop(1);
-                                    while (bline.endsWith(QLatin1Char(' '))
+                       }
-                                           || bline.endsWith(QLatin1Char('\t')))
+                       contPos.clear();
-                                        bline.chop(1);
+                   }
-                            }
+                   if (idx < 0) {
-                            contPos.clear();
+                       // Not even a comment stayed behind, so zap the line
-                        }
+                       lines->removeAt(lineNo);
-                        if (idx < 0) {
+                       delta++;
-                            // Not even a comment stayed behind, so zap the line
+                       continue;
-                            lines->removeAt(lineNo);
+                   }
-                            delta++;
+               }
-                            continue;
+               if (contCol >= 0)
-                        }
+                   contPos.append(qMakePair(lineNo, contCol));
-                    }
+           }
-                    if (contCol >= 0)
+         nextLine:
-                        contPos.append(qMakePair(lineNo, contCol));
+           lineNo++;
-                }
+       }
-              nextLine:
+     nextVar: ;
-                lineNo++;
-            }
-          nextVar: ;
-        }
     }
     return notChanged;
 }