C++: Highlight multi-line raw string literals

Task-number: QTCREATORBUG-13094
Change-Id: I4e6b8c202677f4c1cd4df95d59130ba8379e72fe
Reviewed-by: Marco Bubke <marco.bubke@theqtcompany.com>
Reviewed-by: Orgad Shaneh <orgads@gmail.com>

src/libs/3rdparty/cplusplus/Lexer.cpp

@@ -135,11 +135,18 @@ void Lexer::scan(Token *tok)
     tok->f.utf16chars = _currentCharUtf16 - _tokenStartUtf16;
 }
 
+static bool isRawStringLiteral(unsigned char kind)
+{
+    return kind >= T_FIRST_RAW_STRING_LITERAL
+        && kind <= T_LAST_RAW_STRING_LITERAL;
+}
+
 static bool isMultiLineToken(unsigned char kind)
 {
     return kind == T_EOF_SYMBOL
         || kind == T_COMMENT
-        || kind == T_DOXY_COMMENT;
+        || kind == T_DOXY_COMMENT
+        || isRawStringLiteral(kind);
 }
 
 void Lexer::scan_helper(Token *tok)
@@ -207,7 +214,12 @@ void Lexer::scan_helper(Token *tok)
         _state = 0;
         scanCppComment(originalKind);
         return;
-    } else { // strings
+    } else if (isRawStringLiteral(s._tokenKind)) {
+        tok->f.kind = s._tokenKind;
+        if (scanUntilRawStringLiteralEndSimple())
+            _state = 0;
+        return;
+    } else { // non-raw strings
         tok->f.joined = true;
         tok->f.kind = s._tokenKind;
         _state = 0;
@@ -740,6 +752,32 @@ void Lexer::scanRawStringLiteral(Token *tok, unsigned char hint)
         tok->f.kind = T_RAW_UTF8_STRING_LITERAL;
     else
         tok->f.kind = T_RAW_STRING_LITERAL;
+
+    if (!_yychar)
+        s._tokenKind = tok->f.kind;
+}
+
+// In the highlighting case we don't have any further information
+// like the delimiter or its length, so just match for: ...)..."
+bool Lexer::scanUntilRawStringLiteralEndSimple()
+{
+    bool closingParenthesisPassed = false;
+
+    while (_yychar) {
+        if (_yychar == ')') {
+            yyinp();
+            closingParenthesisPassed = true;
+        } else {
+            if (closingParenthesisPassed && _yychar == '"') {
+                yyinp();
+                return true;
+            } else {
+                yyinp();
+            }
+        }
+    }
+
+    return false;
 }
 
 void Lexer::scanCharLiteral(Token *tok, unsigned char hint)

src/libs/3rdparty/cplusplus/Lexer.h

@@ -96,6 +96,7 @@ private:
 
     void scanStringLiteral(Token *tok, unsigned char hint = 0);
     void scanRawStringLiteral(Token *tok, unsigned char hint = 0);
+    bool scanUntilRawStringLiteralEndSimple();
     void scanCharLiteral(Token *tok, unsigned char hint = 0);
     void scanUntilQuote(Token *tok, unsigned char quote);
     bool scanDigitSequence();

src/libs/3rdparty/cplusplus/Token.h

@@ -49,11 +49,13 @@ enum Kind {
     T_UTF8_STRING_LITERAL,
     T_UTF16_STRING_LITERAL,
     T_UTF32_STRING_LITERAL,
-    T_RAW_STRING_LITERAL,
+    T_FIRST_RAW_STRING_LITERAL,
+    T_RAW_STRING_LITERAL = T_FIRST_RAW_STRING_LITERAL,
     T_RAW_WIDE_STRING_LITERAL,
     T_RAW_UTF8_STRING_LITERAL,
     T_RAW_UTF16_STRING_LITERAL,
     T_RAW_UTF32_STRING_LITERAL,
+    T_LAST_RAW_STRING_LITERAL = T_RAW_UTF32_STRING_LITERAL,
     T_AT_STRING_LITERAL,
     T_ANGLE_STRING_LITERAL,
     T_LAST_STRING_LITERAL = T_ANGLE_STRING_LITERAL,