CPlusPlus: Add lexer support for new C++20 keywords

Change-Id: I2b83deb0502ebf2cdca2af774fbb2ce26e947c11
Reviewed-by: Christian Stenger <christian.stenger@qt.io>

src/libs/3rdparty/cplusplus/Keywords.cpp

@@ -603,6 +603,34 @@ static inline int classify7(const char *s, LanguageFeatures features)
       }
     }
   }
+  else if (features.cxx20Enabled && s[0] == 'c') {
+    if (s[1] == 'h') {
+      if (s[2] == 'a') {
+        if (s[3] == 'r') {
+          if (s[4] == '8') {
+            if (s[5] == '_') {
+              if (s[6] == 't') {
+                return T_CHAR8_T;
+              }
+            }
+          }
+        }
+      }
+    }
+    else if (s[1] == 'o') {
+      if (s[2] == 'n') {
+        if (s[3] == 'c') {
+          if (s[4] == 'e') {
+            if (s[5] == 'p') {
+              if (s[6] == 't') {
+                return T_CONCEPT;
+              }
+            }
+          }
+        }
+      }
+    }
+  }
   else if (s[0] == 'd') {
     if (s[1] == 'e') {
       if (s[2] == 'f') {
@@ -847,7 +875,31 @@ static inline int classify8(const char *s, LanguageFeatures features)
       }
     }
     else if (s[1] == 'o') {
-      if (s[2] == 'n') {
+      if (features.cxx20Enabled && s[2] == '_') {
+        if (s[3] == 'a') {
+          if (s[4] == 'w') {
+            if (s[5] == 'a') {
+              if (s[6] == 'i') {
+                if (s[7] == 't') {
+                  return T_CO_AWAIT;
+                }
+              }
+            }
+          }
+        }
+        else if (s[3] == 'y') {
+          if (s[4] == 'i') {
+            if (s[5] == 'e') {
+              if (s[6] == 'l') {
+                if (s[7] == 'd') {
+                  return T_CO_YIELD;
+                }
+              }
+            }
+          }
+        }
+      }
+      else if (s[2] == 'n') {
         if (s[3] == 't') {
           if (s[4] == 'i') {
             if (s[5] == 'n') {
@@ -945,6 +997,19 @@ static inline int classify8(const char *s, LanguageFeatures features)
           }
         }
       }
+      else if (features.cxx20Enabled && s[2] == 'q') {
+        if (s[3] == 'u') {
+          if (s[4] == 'i') {
+            if (s[5] == 'r') {
+              if (s[6] == 'e') {
+                if (s[7] == 's') {
+                  return T_REQUIRES;
+                }
+              }
+            }
+          }
+        }
+      }
     }
   }
   else if (features.cxxEnabled && s[0] == 't') {
@@ -1097,13 +1162,35 @@ static inline int classify9(const char *s, LanguageFeatures features)
       }
     }
   }
-  else if (features.cxx11Enabled && s[0] == 'c') {
+  else if (s[0] == 'c') {
     if (s[1] == 'o') {
-      if (s[2] == 'n') {
+      if (features.cxx20Enabled && s[2] == '_') {
+        if (s[3] == 'r') {
+          if (s[4] == 'e') {
+            if (s[5] == 't') {
+              if (s[6] == 'u') {
+                if (s[7] == 'r') {
+                  if (s[8] == 'n') {
+                    return T_CO_RETURN;
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+      else if (s[2] == 'n') {
         if (s[3] == 's') {
           if (s[4] == 't') {
             if (s[5] == 'e') {
-              if (s[6] == 'x') {
+              if (features.cxx20Enabled && s[6] == 'v') {
+                if (s[7] == 'a') {
+                  if (s[8] == 'l') {
+                    return T_CONSTEVAL;
+                  }
+                }
+              }
+              else if (features.cxx11Enabled && s[6] == 'x') {
                 if (s[7] == 'p') {
                   if (s[8] == 'r') {
                     return T_CONSTEXPR;
@@ -1111,6 +1198,15 @@ static inline int classify9(const char *s, LanguageFeatures features)
                 }
               }
             }
+            else if (features.cxx20Enabled && s[5] == 'i') {
+              if (s[6] == 'n') {
+                if (s[7] == 'i') {
+                  if (s[8] == 't') {
+                    return T_CONSTINIT;
+                  }
+                }
+              }
+            }
           }
         }
       }

src/libs/3rdparty/cplusplus/Keywords.kwgen

@@ -128,6 +128,16 @@ nullptr
 static_assert
 thread_local
 
+%pre-check=features.cxx20Enabled
+char8_t
+concept
+consteval
+constinit
+co_await
+co_return
+co_yield
+requires
+
 %pre-check=features.qtKeywordsEnabled
 emit
 foreach

src/libs/3rdparty/cplusplus/Token.cpp

@@ -120,9 +120,15 @@ const char *token_names[] = {
     ("case"),
     ("catch"),
     ("class"),
+    ("co_await"),
+    ("co_return"),
+    ("co_yield"),
+    ("concept"),
     ("const"),
     ("const_cast"),
+    ("consteval"),
     ("constexpr"),
+    ("constinit"),
     ("continue"),
     ("decltype"),
     ("default"),
@@ -151,6 +157,7 @@ const char *token_names[] = {
     ("public"),
     ("register"),
     ("reinterpret_cast"),
+    ("requires"),
     ("return"),
     ("sizeof"),
     ("static"),
@@ -210,6 +217,7 @@ const char *token_names[] = {
     // Primitive types
     ("bool"),
     ("char"),
+    ("char8_t"),
     ("char16_t"),
     ("char32_t"),
     ("double"),

src/libs/3rdparty/cplusplus/Token.h

@@ -130,9 +130,15 @@ enum Kind {
     T_CASE,
     T_CATCH,
     T_CLASS,
+    T_CO_AWAIT,
+    T_CO_RETURN,
+    T_CO_YIELD,
+    T_CONCEPT,
     T_CONST,
     T_CONST_CAST,
+    T_CONSTEVAL,
     T_CONSTEXPR,
+    T_CONSTINIT,
     T_CONTINUE,
     T_DECLTYPE,
     T_DEFAULT,
@@ -161,6 +167,7 @@ enum Kind {
     T_PUBLIC,
     T_REGISTER,
     T_REINTERPRET_CAST,
+    T_REQUIRES,
     T_RETURN,
     T_SIZEOF,
     T_STATIC,
@@ -223,6 +230,7 @@ enum Kind {
     T_FIRST_PRIMITIVE,
     T_BOOL = T_FIRST_PRIMITIVE,
     T_CHAR,
+    T_CHAR8_T,
     T_CHAR16_T,
     T_CHAR32_T,
     T_DOUBLE,

src/libs/cplusplus/cplusplus.qbs

@@ -41,6 +41,7 @@ Project {
                 "FullySpecifiedType.cpp",
                 "FullySpecifiedType.h",
                 "Keywords.cpp",
+                "Keywords.kwgen",
                 "Lexer.cpp",
                 "Lexer.h",
                 "LiteralTable.h",

src/plugins/cppeditor/cppcodemodelinspectordumper.cpp

@@ -254,12 +254,19 @@ QString Utils::toString(CPlusPlus::Kind kind)
     TOKEN(T_CASE);
     TOKEN(T_CATCH);
     TOKEN(T_CHAR);
+    TOKEN(T_CHAR8_T);
     TOKEN(T_CHAR16_T);
     TOKEN(T_CHAR32_T);
     TOKEN(T_CLASS);
+    TOKEN(T_CO_AWAIT);
+    TOKEN(T_CO_RETURN);
+    TOKEN(T_CO_YIELD);
+    TOKEN(T_CONCEPT);
     TOKEN_AND_ALIASES(T_CONST, T___CONST/T___CONST__);
     TOKEN(T_CONST_CAST);
     TOKEN(T_CONSTEXPR);
+    TOKEN(T_CONSTEVAL);
+    TOKEN(T_CONSTINIT);
     TOKEN(T_CONTINUE);
     TOKEN_AND_ALIASES(T_DECLTYPE, T___DECLTYPE);
     TOKEN(T_DEFAULT);
@@ -292,6 +299,7 @@ QString Utils::toString(CPlusPlus::Kind kind)
     TOKEN(T_PUBLIC);
     TOKEN(T_REGISTER);
     TOKEN(T_REINTERPRET_CAST);
+    TOKEN(T_REQUIRES);
     TOKEN(T_RETURN);
     TOKEN(T_SHORT);
     TOKEN(T_SIGNED);

src/plugins/cppeditor/cppcompletion_test.cpp

@@ -409,16 +409,16 @@ static void enumTestCase(const QByteArray &tag, const QByteArray &source,
                          const QByteArray &prefix = QByteArray())
 {
     QByteArray fullSource = source;
-    fullSource.replace('$', "enum E { val1, val2, val3 };");
+    fullSource.replace('$', "enum E { value1, value2, value3 };");
-    QTest::newRow(tag) << fullSource << (prefix + "val")
+    QTest::newRow(tag) << fullSource << (prefix + "value")
-            << QStringList({"val1", "val2", "val3"});
+            << QStringList({"value1", "value2", "value3"});
 
     QTest::newRow(QByteArray{tag + "_cxx11"}) << fullSource << QByteArray{prefix + "E::"}
-            << QStringList({"E", "val1", "val2", "val3"});
+            << QStringList({"E", "value1", "value2", "value3"});
 
     fullSource.replace("enum E ", "enum ");
-    QTest::newRow(QByteArray{tag + "_anon"}) << fullSource << QByteArray{prefix + "val"}
+    QTest::newRow(QByteArray{tag + "_anon"}) << fullSource << QByteArray{prefix + "value"}
-            << QStringList({"val1", "val2", "val3"});
+            << QStringList({"value1", "value2", "value3"});
 }
 
 void CompletionTest::testCompletion_data()

tests/auto/cplusplus/lexer/tst_lexer.cpp

@@ -43,6 +43,7 @@ public:
 private slots:
     void basic();
     void basic_data();
+    void cxx20();
     void incremental();
     void incremental_data();
     void literals();
@@ -250,6 +251,38 @@ void tst_SimpleLexer::basic_data()
     QTest::newRow(source) << source << expectedTokenKindList;
 }
 
+void tst_SimpleLexer::cxx20()
+{
+    LanguageFeatures features;
+    features.cxxEnabled = features.cxx11Enabled = features.cxx14Enabled
+        = features.cxx20Enabled = true;
+    const QString source = R"(
+template<typename T> concept IsPointer = requires(T p) { *p; };
+SomeType coroutine()
+{
+    constinit const char8_t = 'c';
+    if consteval {} else {}
+    co_await std::suspend_always{};
+    co_yield 1;
+    co_return;
+}
+)";
+    const TokenKindList expectedTokens = {
+        T_TEMPLATE, T_LESS, T_TYPENAME, T_IDENTIFIER, T_GREATER, T_CONCEPT, T_IDENTIFIER, T_EQUAL,
+        T_REQUIRES, T_LPAREN, T_IDENTIFIER, T_IDENTIFIER, T_RPAREN, T_LBRACE, T_STAR, T_IDENTIFIER,
+        T_SEMICOLON, T_RBRACE, T_SEMICOLON,
+        T_IDENTIFIER, T_IDENTIFIER, T_LPAREN, T_RPAREN,
+        T_LBRACE,
+        T_CONSTINIT, T_CONST, T_CHAR8_T, T_EQUAL, T_CHAR_LITERAL, T_SEMICOLON,
+        T_IF, T_CONSTEVAL, T_LBRACE, T_RBRACE, T_ELSE, T_LBRACE, T_RBRACE,
+        T_CO_AWAIT, T_IDENTIFIER, T_COLON_COLON, T_IDENTIFIER, T_LBRACE, T_RBRACE, T_SEMICOLON,
+        T_CO_YIELD, T_NUMERIC_LITERAL, T_SEMICOLON,
+        T_CO_RETURN, T_SEMICOLON,
+        T_RBRACE
+    };
+    run(source.toUtf8(), toTokens(expectedTokens), false, CompareKind, false, features);
+}
+
 void tst_SimpleLexer::literals()
 {
     QFETCH(QByteArray, source);