precompile expressions

this gives a rather significant boost, in particular for
expressions which are evaluated repeatedly (includes).

src/shared/proparser/proitems.cpp

@@ -157,6 +157,89 @@ bool ProString::operator==(const QLatin1String &other) const
     return (uc == e);
 }
 
+QChar *ProString::prepareAppend(int extraLen)
+{
+    if (m_string.isDetached() && m_length + extraLen <= m_string.capacity()) {
+        m_string.reserve(0); // Prevent the resize() below from reallocating
+        QChar *ptr = (QChar *)m_string.constData();
+        if (m_offset)
+            memmove(ptr, ptr + m_offset, m_length * 2);
+        ptr += m_length;
+        m_offset = 0;
+        m_length += extraLen;
+        m_string.resize(m_length);
+        m_hash = 0x80000000;
+        return ptr;
+    } else {
+        QString neu(m_length + extraLen, Qt::Uninitialized);
+        QChar *ptr = (QChar *)neu.constData();
+        memcpy(ptr, m_string.constData() + m_offset, m_length * 2);
+        ptr += m_length;
+        *this = ProString(neu, NoHash);
+        return ptr;
+    }
+}
+
+// If pending != 0, prefix with space if appending to non-empty non-pending
+ProString &ProString::append(const ProString &other, bool *pending)
+{
+    if (other.m_length) {
+        if (!m_length) {
+            *this = other;
+        } else {
+            QChar *ptr;
+            if (pending && !*pending) {
+                ptr = prepareAppend(1 + other.m_length);
+                *ptr++ = 32;
+            } else {
+                ptr = prepareAppend(other.m_length);
+            }
+            memcpy(ptr, other.m_string.constData() + other.m_offset, other.m_length * 2);
+        }
+        if (pending)
+            *pending = true;
+    }
+    return *this;
+}
+
+ProString &ProString::append(const ProStringList &other, bool *pending, bool skipEmpty1st)
+{
+    if (const int sz = other.size()) {
+        int startIdx = 0;
+        if (pending && !*pending && skipEmpty1st && other.at(0).isEmpty()) {
+            if (sz == 1)
+                return *this;
+            startIdx = 1;
+        }
+        if (!m_length && sz == startIdx + 1) {
+            *this = other.at(startIdx);
+        } else {
+            int totalLength = sz - startIdx;
+            for (int i = startIdx; i < sz; ++i)
+                totalLength += other.at(i).size();
+            bool putSpace = false;
+            if (pending && !*pending && m_length)
+                putSpace = true;
+            else
+                totalLength--;
+
+            QChar *ptr = prepareAppend(totalLength);
+            for (int i = startIdx; i < sz; ++i) {
+                if (putSpace)
+                    *ptr++ = 32;
+                else
+                    putSpace = true;
+                const ProString &str = other.at(i);
+                memcpy(ptr, str.m_string.constData() + str.m_offset, str.m_length * 2);
+                ptr += str.m_length;
+            }
+        }
+        if (pending)
+            *pending = true;
+    }
+    return *this;
+}
+
 QString operator+(const ProString &one, const ProString &two)
 {
     if (two.m_length) {

src/shared/proparser/proitems.h

@@ -53,6 +53,8 @@ namespace ProStringConstants {
 enum OmitPreHashing { NoHash };
 }
 
+class ProStringList;
+
 class ProString {
 public:
     ProString();
@@ -67,6 +69,9 @@ public:
     ProString(const QString &str, int offset, int length, ProStringConstants::OmitPreHashing);
     QString toQString() const;
     QString &toQString(QString &tmp) const;
+    ProString &operator+=(const ProString &other);
+    ProString &append(const ProString &other, bool *pending = 0);
+    ProString &append(const ProStringList &other, bool *pending = 0, bool skipEmpty1st = false);
     bool operator==(const ProString &other) const;
     bool operator==(const QString &other) const;
     bool operator==(const QLatin1String &other) const;
@@ -88,6 +93,7 @@ private:
     QString m_string;
     int m_offset, m_length;
     mutable uint m_hash;
+    QChar *prepareAppend(int extraLen);
     uint updatedHash() const;
     friend uint qHash(const ProString &str);
     friend QString operator+(const ProString &one, const ProString &two);
@@ -112,35 +118,66 @@ public:
 // These token definitions affect both ProFileEvaluator and ProWriter
 enum ProToken {
     TokTerminator = 0,  // end of stream (possibly not included in length; must be zero)
-    TokLine,            // line marker: // +1 (2-nl) to 1st token of each line
+    TokLine,            // line marker:
                         // - line (1)
-    TokAssign,          // variable =   // "A=":2 => 1+4+2=7 (8)
+    TokAssign,          // variable =
-    TokAppend,          // variable +=  // "A+=":3 => 1+4+2=7 (8)
+    TokAppend,          // variable +=
-    TokAppendUnique,    // variable *=  // "A*=":3 => 1+4+2=7 (8)
+    TokAppendUnique,    // variable *=
-    TokRemove,          // variable -=  // "A-=":3 => 1+4+2=7 (8)
+    TokRemove,          // variable -=
-    TokReplace,         // variable ~=  // "A~=":3 => 1+4+2=7 (8)
+    TokReplace,         // variable ~=
-                        // - variable name: hash (2), length (1), chars (length)
+                        // previous literal/expansion is a variable manipulation
-                        // - expression: length (2), chars (length)
+                        // - value expression + TokValueTerminator
-    TokCondition,       // CONFIG test:   // "A":1 => 1+2=3 (4)
+    TokValueTerminator, // assignment value terminator
-                        // - test name: lenght (1), chars (length)
+    TokLiteral,         // literal string (fully dequoted)
+                        // - length (1)
+                        // - string data (length; unterminated)
+    TokHashLiteral,     // literal string with hash (fully dequoted)
+                        // - hash (2)
+                        // - length (1)
+                        // - string data (length; unterminated)
+    TokVariable,        // qmake variable expansion
+                        // - hash (2)
+                        // - name length (1)
+                        // - name (name length; unterminated)
+    TokProperty,        // qmake property expansion
+                        // - name length (1)
+                        // - name (name length; unterminated)
+    TokEnvVar,          // environment variable expansion
+                        // - name length (1)
+                        // - name (name length; unterminated)
+    TokFuncName,        // replace function expansion
+                        // - hash (2)
+                        // - name length (1)
+                        // - name (name length; unterminated)
+                        // - ((nested expansion + TokArgSeparator)* + nested expansion)?
+                        // - TokFuncTerminator
+    TokArgSeparator,    // function argument separator
+    TokFuncTerminator,  // function argument list terminator
+    TokCondition,       // previous literal/expansion is a conditional
+    TokTestCall,        // previous literal/expansion is a test function call
+                        // - ((nested expansion + TokArgSeparator)* + nested expansion)?
+                        // - TokFuncTerminator
     TokNot,             // '!' operator
     TokAnd,             // ':' operator
     TokOr,              // '|' operator
-    TokBranch,          // branch point:   // "X:A=":4 => [5]+1+4+1+1+[7]=19 (20)
+    TokBranch,          // branch point:
                         // - then block length (2)
                         // - then block + TokTerminator (then block length)
                         // - else block length (2)
                         // - else block + TokTerminator (else block length)
-    TokForLoop,         // for loop:   // "for(A,B)":8 => 1+4+3+2+1=11 (12)
+    TokForLoop,         // for loop:
                         // - variable name: hash (2), length (1), chars (length)
-                        // - expression: length (2), chars (length)
+                        // - expression: length (2), bytes + TokValueTerminator (length)
                         // - body length (2)
                         // - body + TokTerminator (body length)
-    TokTestDef,         // test function definition:     // "defineTest(A):":14 => 1+4+2+1=8 (9)
+    TokTestDef,         // test function definition:
-    TokReplaceDef,      // replace function definition:  // "defineReplace(A):":17 => 1+4+2+1=8 (9)
+    TokReplaceDef,      // replace function definition:
                         // - function name: hash (2), length (1), chars (length)
                         // - body length (2)
                         // - body + TokTerminator (body length)
+    TokMask = 0xff,
+    TokQuoted = 0x100,  // The expression is quoted => join expanded stringlist
+    TokNewStr = 0x200   // Next stringlist element
 };
 
 class ProFile

src/shared/proparser/prowriter.cpp

@@ -42,13 +42,29 @@ static uint getBlockLen(const ushort *&tokPtr)
     return len;
 }
 
-static QString &getHashStr(const ushort *&tokPtr, QString &tmp)
+static bool getLiteral(const ushort *tokPtr, const ushort *tokEnd, QString &tmp)
 {
-    tokPtr += 2; // ignore hash
+    int count = 0;
-    uint len = *tokPtr++;
+    while (tokPtr != tokEnd) {
-    tmp.setRawData((const QChar *)tokPtr, len);
+        ushort tok = *tokPtr++;
-    tokPtr += len;
+        switch (tok & TokMask) {
-    return tmp;
+        case TokLine:
+            tokPtr++;
+            break;
+        case TokHashLiteral:
+            tokPtr += 2;
+            // fallthrough
+        case TokLiteral: {
+            uint len = *tokPtr++;
+            tmp.setRawData((const QChar *)tokPtr, len);
+            count++;
+            tokPtr += len;
+            break; }
+        default:
+            return false;
+        }
+    }
+    return count == 1;
 }
 
 static void skipStr(const ushort *&tokPtr)
@@ -64,19 +80,53 @@ static void skipHashStr(const ushort *&tokPtr)
     tokPtr += len;
 }
 
-static void skipLongStr(const ushort *&tokPtr)
-{
-    uint len = getBlockLen(tokPtr);
-    tokPtr += len;
-}
-
 static void skipBlock(const ushort *&tokPtr)
 {
     uint len = getBlockLen(tokPtr);
     tokPtr += len;
 }
 
-static void skipToken(ushort tok, const ushort *&tokPtr, int &lineNo)
+static void skipExpression(const ushort *&pTokPtr, int &lineNo)
+{
+    const ushort *tokPtr = pTokPtr;
+    forever {
+        ushort tok = *tokPtr++;
+        switch (tok) {
+        case TokLine:
+            lineNo = *tokPtr++;
+            break;
+        case TokValueTerminator:
+        case TokFuncTerminator:
+            pTokPtr = tokPtr;
+            return;
+        case TokArgSeparator:
+            break;
+        default:
+            switch (tok & TokMask) {
+            case TokLiteral:
+            case TokProperty:
+            case TokEnvVar:
+                skipStr(tokPtr);
+                break;
+            case TokHashLiteral:
+            case TokVariable:
+                skipHashStr(tokPtr);
+                break;
+            case TokFuncName:
+                skipHashStr(tokPtr);
+                pTokPtr = tokPtr;
+                skipExpression(pTokPtr, lineNo);
+                tokPtr = pTokPtr;
+                break;
+            default:
+                pTokPtr = tokPtr - 1;
+                return;
+            }
+        }
+    }
+}
+
+static const ushort *skipToken(ushort tok, const ushort *&tokPtr, int &lineNo)
 {
     switch (tok) {
     case TokLine:
@@ -87,16 +137,14 @@ static void skipToken(ushort tok, const ushort *&tokPtr, int &lineNo)
     case TokAppendUnique:
     case TokRemove:
     case TokReplace:
-        skipHashStr(tokPtr);
+    case TokTestCall:
-        skipLongStr(tokPtr);
+        skipExpression(tokPtr, lineNo);
-        break;
-    case TokBranch:
-        skipBlock(tokPtr);
-        skipBlock(tokPtr);
         break;
     case TokForLoop:
         skipHashStr(tokPtr);
-        skipLongStr(tokPtr);
+        // fallthrough
+    case TokBranch:
+        skipBlock(tokPtr);
         skipBlock(tokPtr);
         break;
     case TokTestDef:
@@ -107,12 +155,17 @@ static void skipToken(ushort tok, const ushort *&tokPtr, int &lineNo)
     case TokNot:
     case TokAnd:
     case TokOr:
-        break;
     case TokCondition:
-        skipStr(tokPtr);
         break;
-    default: Q_ASSERT_X(false, "skipToken", "unexpected item type");
+    default: {
+            const ushort *oTokPtr = --tokPtr;
+            skipExpression(tokPtr, lineNo);
+            if (tokPtr != oTokPtr)
+                return oTokPtr;
+        }
+        Q_ASSERT_X(false, "skipToken", "unexpected item type");
     }
+    return 0;
 }
 
 void ProWriter::addFiles(ProFile *profile, QStringList *lines,
@@ -123,9 +176,10 @@ void ProWriter::addFiles(ProFile *profile, QStringList *lines,
     const ushort *tokPtr = (const ushort *)profile->items().constData();
     int lineNo = 0;
     QString tmp;
+    const ushort *lastXpr = 0;
     while (ushort tok = *tokPtr++) {
         if (tok == TokAssign || tok == TokAppend || tok == TokAppendUnique) {
-            if (var == getHashStr(tokPtr, tmp)) {
+            if (getLiteral(lastXpr, tokPtr - 1, tmp) && var == tmp) {
                 for (--lineNo; lineNo < lines->count(); lineNo++) {
                     QString line = lines->at(lineNo);
                     int idx = line.indexOf(QLatin1Char('#'));
@@ -152,9 +206,9 @@ void ProWriter::addFiles(ProFile *profile, QStringList *lines,
                 lines->insert(lineNo, added);
                 return;
             }
-            skipLongStr(tokPtr);
+            skipExpression(tokPtr, lineNo);
         } else {
-            skipToken(tok, tokPtr, lineNo);
+            lastXpr = skipToken(tok, tokPtr, lineNo);
         }
     }
 
@@ -170,6 +224,7 @@ static void findProVariables(const ushort *tokPtr, const QStringList &vars,
 {
     int lineNo = 0;
     QString tmp;
+    const ushort *lastXpr = 0;
     while (ushort tok = *tokPtr++) {
         if (tok == TokBranch) {
             uint blockLen = getBlockLen(tokPtr);
@@ -179,11 +234,11 @@ static void findProVariables(const ushort *tokPtr, const QStringList &vars,
             findProVariables(tokPtr, vars, proVars);
             tokPtr += blockLen;
         } else if (tok == TokAssign || tok == TokAppend || tok == TokAppendUnique) {
-            if (vars.contains(getHashStr(tokPtr, tmp)))
+            if (getLiteral(lastXpr, tokPtr - 1, tmp) && vars.contains(tmp))
                 *proVars << lineNo;
-            skipLongStr(tokPtr);
+            skipExpression(tokPtr, lineNo);
         } else {
-            skipToken(tok, tokPtr, lineNo);
+            lastXpr = skipToken(tok, tokPtr, lineNo);
         }
     }
 }