feedc0de/qt-creator
1
0
Fork 0
forked from qt-creator/qt-creator
Code Pull Requests Activity

C++: Fix cplusplus-keywordgen to generate Keywords.cpp

Browse Source 
Fix kewordgen to generate current Keywords.cpp with minimal diff,
but also to be compatible with old usage and kwgen files.

Add new parameters:
%no-namespace-for-tokens - not use namespace for tokens
%pre-check-argument - set type and name of additional variable to use
%function-name - name of a function ("classify" by default)
%pre-check - name of a variable to check for a keyword (i.e. cxxEnabled)
Now "%%" is also used to end and restart keywords parsing to be able to
generate additional keywords for classifyOperator()

Automatically add checks for LanguageFeatures to generated code
Pass input and output files with command line arguments, print usage

Update Keywords.kwgen to include all keywords currently used

Add missing aliases to Token.h

Change-Id: I6cc84e150e0d797277204032fc05ce9cfbd01f58
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@qt.io>
This commit is contained in:
Volodymyr Zibarov
2020-05-26 22:05:11 +03:00
parent 3563c457a5
commit df6d95c3e1
4 changed files with 660 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

450
src/libs/3rdparty/cplusplus/Keywords.cpp vendored
View File

@@ -23,6 +23,12 @@
namespace CPlusPlus { namespace CPlusPlus {
// === following code is generated with cplusplus-keywordgen tool
// === from source file: Keywords.kwgen
// === keywords begin
static inline int classify2(const char *s, LanguageFeatures) static inline int classify2(const char *s, LanguageFeatures)
{ {
if (s[0] == 'd') { if (s[0] == 'd') {
@@ -132,13 +138,6 @@ static inline int classify4(const char *s, LanguageFeatures features)
} }
} }
} }
else if (s[1] == 'n') {
if (s[2] == 'u') {
if (s[3] == 'm') {
return T_ENUM;
}
}
}
else if (features.qtKeywordsEnabled && s[1] == 'm') { else if (features.qtKeywordsEnabled && s[1] == 'm') {
if (s[2] == 'i') { if (s[2] == 'i') {
if (s[3] == 't') { if (s[3] == 't') {
@@ -146,6 +145,13 @@ static inline int classify4(const char *s, LanguageFeatures features)
} }
} }
} }
else if (s[1] == 'n') {
if (s[2] == 'u') {
if (s[3] == 'm') {
return T_ENUM;
}
}
}
} }
else if (s[0] == 'g') { else if (s[0] == 'g') {
if (s[1] == 'o') { if (s[1] == 'o') {
@@ -165,15 +171,15 @@ static inline int classify4(const char *s, LanguageFeatures features)
} }
} }
} }
else if (s[0] == 't') { else if (features.cxxEnabled && s[0] == 't') {
if (features.cxxEnabled && s[1] == 'h') { if (s[1] == 'h') {
if (s[2] == 'i') { if (s[2] == 'i') {
if (s[3] == 's') { if (s[3] == 's') {
return T_THIS; return T_THIS;
} }
} }
} }
else if (features.cxxEnabled && s[1] == 'r') { else if (s[1] == 'r') {
if (s[2] == 'u') { if (s[2] == 'u') {
if (s[3] == 'e') { if (s[3] == 'e') {
return T_TRUE; return T_TRUE;
@@ -285,13 +291,11 @@ static inline int classify5(const char *s, LanguageFeatures features)
} }
} }
} }
else if (features.qtKeywordsEnabled) { else if (features.qtKeywordsEnabled && s[1] == 'l') {
if (s[1] == 'l') { if (s[2] == 'o') {
if (s[2] == 'o') { if (s[3] == 't') {
if (s[3] == 't') { if (s[4] == 's') {
if (s[4] == 's') { return T_SLOTS;
return T_Q_SLOTS;
}
} }
} }
} }
@@ -513,22 +517,18 @@ static inline int classify6(const char *s, LanguageFeatures features)
} }
} }
} }
else if (features.qtKeywordsEnabled && s[0] == 'S') { else if (features.qtKeywordsEnabled && s[0] == 'Q') {
if (s[1] == 'I') { if (s[1] == '_') {
if (s[2] == 'G') { if (s[2] == 'E') {
if (s[3] == 'N') { if (s[3] == 'M') {
if (s[4] == 'A') { if (s[4] == 'I') {
if (s[5] == 'L') { if (s[5] == 'T') {
return T_SIGNAL; return T_Q_EMIT;
} }
} }
} }
} }
} else if (s[2] == 'S') {
}
else if (features.qtKeywordsEnabled && s[0] == 'Q') {
if (s[1] == '_') {
if (s[2] == 'S') {
if (s[3] == 'L') { if (s[3] == 'L') {
if (s[4] == 'O') { if (s[4] == 'O') {
if (s[5] == 'T') { if (s[5] == 'T') {
@@ -537,11 +537,15 @@ static inline int classify6(const char *s, LanguageFeatures features)
} }
} }
} }
else if (s[2] == 'E') { }
if (s[3] == 'M') { }
if (s[4] == 'I') { else if (features.qtKeywordsEnabled && s[0] == 'S') {
if (s[5] == 'T') { if (s[1] == 'I') {
return T_Q_EMIT; if (s[2] == 'G') {
if (s[3] == 'N') {
if (s[4] == 'A') {
if (s[5] == 'L') {
return T_SIGNAL;
} }
} }
} }
@@ -614,6 +618,21 @@ static inline int classify7(const char *s, LanguageFeatures features)
} }
} }
} }
else if (features.qtKeywordsEnabled && s[0] == 'f') {
if (s[1] == 'o') {
if (s[2] == 'r') {
if (s[3] == 'e') {
if (s[4] == 'a') {
if (s[5] == 'c') {
if (s[6] == 'h') {
return T_FOREACH;
}
}
}
}
}
}
}
else if (features.cxxEnabled && s[0] == 'm') { else if (features.cxxEnabled && s[0] == 'm') {
if (s[1] == 'u') { if (s[1] == 'u') {
if (s[2] == 't') { if (s[2] == 't') {
@@ -659,21 +678,6 @@ static inline int classify7(const char *s, LanguageFeatures features)
} }
} }
} }
else if (features.qtKeywordsEnabled && s[0] == 'f') {
if (s[1] == 'o') {
if (s[2] == 'r') {
if (s[3] == 'e') {
if (s[4] == 'a') {
if (s[5] == 'c') {
if (s[6] == 'h') {
return T_Q_FOREACH;
}
}
}
}
}
}
}
else if (features.qtKeywordsEnabled && s[0] == 's') { else if (features.qtKeywordsEnabled && s[0] == 's') {
if (s[1] == 'i') { if (s[1] == 'i') {
if (s[2] == 'g') { if (s[2] == 'g') {
@@ -681,7 +685,7 @@ static inline int classify7(const char *s, LanguageFeatures features)
if (s[4] == 'a') { if (s[4] == 'a') {
if (s[5] == 'l') { if (s[5] == 'l') {
if (s[6] == 's') { if (s[6] == 's') {
return T_Q_SIGNALS; return T_SIGNALS;
} }
} }
} }
@@ -736,18 +740,7 @@ static inline int classify7(const char *s, LanguageFeatures features)
} }
else if (features.qtEnabled && s[0] == 'Q') { else if (features.qtEnabled && s[0] == 'Q') {
if (s[1] == '_') { if (s[1] == '_') {
if (s[2] == 'S') { if (s[2] == 'E') {
if (s[3] == 'L') {
if (s[4] == 'O') {
if (s[5] == 'T') {
if (s[6] == 'S') {
return T_Q_SLOTS;
}
}
}
}
}
else if (s[2] == 'E') {
if (s[3] == 'N') { if (s[3] == 'N') {
if (s[4] == 'U') { if (s[4] == 'U') {
if (s[5] == 'M') { if (s[5] == 'M') {
@@ -769,6 +762,17 @@ static inline int classify7(const char *s, LanguageFeatures features)
} }
} }
} }
else if (s[2] == 'S') {
if (s[3] == 'L') {
if (s[4] == 'O') {
if (s[5] == 'T') {
if (s[6] == 'S') {
return T_Q_SLOTS;
}
}
}
}
}
} }
} }
return T_IDENTIFIER; return T_IDENTIFIER;
@@ -792,18 +796,7 @@ static inline int classify8(const char *s, LanguageFeatures features)
} }
} }
else if (s[2] == 't') { else if (s[2] == 't') {
if (s[3] == 'y') { if (s[3] == 'h') {
if (s[4] == 'p') {
if (s[5] == 'e') {
if (s[6] == 'o') {
if (s[7] == 'f') {
return T___TYPEOF;
}
}
}
}
}
else if (s[3] == 'h') {
if (s[4] == 'r') { if (s[4] == 'r') {
if (s[5] == 'e') { if (s[5] == 'e') {
if (s[6] == 'a') { if (s[6] == 'a') {
@@ -814,11 +807,46 @@ static inline int classify8(const char *s, LanguageFeatures features)
} }
} }
} }
else if (s[3] == 'y') {
if (s[4] == 'p') {
if (s[5] == 'e') {
if (s[6] == 'o') {
if (s[7] == 'f') {
return T___TYPEOF;
}
}
}
}
}
} }
} }
} }
else if (s[0] == 'c') { else if (s[0] == 'c') {
if (s[1] == 'o') { if (features.cxx11Enabled && s[1] == 'h') {
if (s[2] == 'a') {
if (s[3] == 'r') {
if (s[4] == '1') {
if (s[5] == '6') {
if (s[6] == '_') {
if (s[7] == 't') {
return T_CHAR16_T;
}
}
}
}
else if (s[4] == '3') {
if (s[5] == '2') {
if (s[6] == '_') {
if (s[7] == 't') {
return T_CHAR32_T;
}
}
}
}
}
}
}
else if (s[1] == 'o') {
if (s[2] == 'n') { if (s[2] == 'n') {
if (s[3] == 't') { if (s[3] == 't') {
if (s[4] == 'i') { if (s[4] == 'i') {
@@ -832,28 +860,6 @@ static inline int classify8(const char *s, LanguageFeatures features)
} }
} }
} }
} else if (features.cxx11Enabled && s[1] == 'h') {
if (s[2] == 'a') {
if (s[3] == 'r') {
if (s[4] == '1') {
if (s[5] == '6') {
if (s[6] == '_') {
if (s[7] == 't') {
return T_CHAR16_T;
}
}
}
} else if (s[4] == '3') {
if (s[5] == '2') {
if (s[6] == '_') {
if (s[7] == 't') {
return T_CHAR32_T;
}
}
}
}
}
}
} }
} }
else if (features.cxx11Enabled && s[0] == 'd') { else if (features.cxx11Enabled && s[0] == 'd') {
@@ -1133,21 +1139,7 @@ static inline int classify9(const char *s, LanguageFeatures features)
} }
else if (features.qtEnabled && s[0] == 'Q') { else if (features.qtEnabled && s[0] == 'Q') {
if (s[1] == '_') { if (s[1] == '_') {
if (s[2] == 'S') { if (s[2] == 'F') {
if (s[3] == 'I') {
if (s[4] == 'G') {
if (s[5] == 'N') {
if (s[6] == 'A') {
if (s[7] == 'L') {
if (s[8] == 'S') {
return T_Q_SIGNALS;
}
}
}
}
}
}
} else if (s[2] == 'F') {
if (s[3] == 'O') { if (s[3] == 'O') {
if (s[4] == 'R') { if (s[4] == 'R') {
if (s[5] == 'E') { if (s[5] == 'E') {
@@ -1162,6 +1154,21 @@ static inline int classify9(const char *s, LanguageFeatures features)
} }
} }
} }
else if (s[2] == 'S') {
if (s[3] == 'I') {
if (s[4] == 'G') {
if (s[5] == 'N') {
if (s[6] == 'A') {
if (s[7] == 'L') {
if (s[8] == 'S') {
return T_Q_SIGNALS;
}
}
}
}
}
}
}
} }
} }
return T_IDENTIFIER; return T_IDENTIFIER;
@@ -1171,24 +1178,7 @@ static inline int classify10(const char *s, LanguageFeatures features)
{ {
if (s[0] == '_') { if (s[0] == '_') {
if (s[1] == '_') { if (s[1] == '_') {
if (s[2] == 'i') { if (features.cxxEnabled && s[2] == 'd') {
if (s[3] == 'n') {
if (s[4] == 'l') {
if (s[5] == 'i') {
if (s[6] == 'n') {
if (s[7] == 'e') {
if (s[8] == '_') {
if (s[9] == '_') {
return T___INLINE__;
}
}
}
}
}
}
}
}
else if (features.cxxEnabled && s[2] == 'd') {
if (s[3] == 'e') { if (s[3] == 'e') {
if (s[4] == 'c') { if (s[4] == 'c') {
if (s[5] == 'l') { if (s[5] == 'l') {
@@ -1205,6 +1195,23 @@ static inline int classify10(const char *s, LanguageFeatures features)
} }
} }
} }
else if (s[2] == 'i') {
if (s[3] == 'n') {
if (s[4] == 'l') {
if (s[5] == 'i') {
if (s[6] == 'n') {
if (s[7] == 'e') {
if (s[8] == '_') {
if (s[9] == '_') {
return T___INLINE__;
}
}
}
}
}
}
}
}
else if (s[2] == 't') { else if (s[2] == 't') {
if (s[3] == 'y') { if (s[3] == 'y') {
if (s[4] == 'p') { if (s[4] == 'p') {
@@ -1272,7 +1279,7 @@ static inline int classify10(const char *s, LanguageFeatures features)
if (s[7] == 'I') { if (s[7] == 'I') {
if (s[8] == 'D') { if (s[8] == 'D') {
if (s[9] == 'E') { if (s[9] == 'E') {
return T_Q_PROPERTY; // Q_OVERRIDE is just an alias for Q_PROPERTY return T_Q_OVERRIDE;
} }
} }
} }
@@ -1308,23 +1315,7 @@ static inline int classify11(const char *s, LanguageFeatures features)
if (s[0] == '_') { if (s[0] == '_') {
if (s[1] == '_') { if (s[1] == '_') {
if (s[2] == 'a') { if (s[2] == 'a') {
if (s[3] == 't') { if (s[3] == 'l') {
if (s[4] == 't') {
if (s[5] == 'r') {
if (s[6] == 'i') {
if (s[7] == 'b') {
if (s[8] == 'u') {
if (s[9] == 't') {
if (s[10] == 'e') {
return T___ATTRIBUTE;
}
}
}
}
}
}
}
} else if (s[3] == 'l') {
if (s[4] == 'i') { if (s[4] == 'i') {
if (s[5] == 'g') { if (s[5] == 'g') {
if (s[6] == 'n') { if (s[6] == 'n') {
@@ -1341,6 +1332,23 @@ static inline int classify11(const char *s, LanguageFeatures features)
} }
} }
} }
else if (s[3] == 't') {
if (s[4] == 't') {
if (s[5] == 'r') {
if (s[6] == 'i') {
if (s[7] == 'b') {
if (s[8] == 'u') {
if (s[9] == 't') {
if (s[10] == 'e') {
return T___ATTRIBUTE;
}
}
}
}
}
}
}
}
} }
} }
} }
@@ -1420,31 +1428,6 @@ static inline int classify12(const char *s, LanguageFeatures features)
} }
} }
} }
else if (features.qtEnabled && s[0] == 'Q') {
if (s[1] == '_') {
if (s[2] == 'I') {
if (s[3] == 'N') {
if (s[4] == 'T') {
if (s[5] == 'E') {
if (s[6] == 'R') {
if (s[7] == 'F') {
if (s[8] == 'A') {
if (s[9] == 'C') {
if (s[10] == 'E') {
if (s[11] == 'S') {
return T_Q_INTERFACES;
}
}
}
}
}
}
}
}
}
}
}
}
else if (features.cxxEnabled && s[0] == 'd') { else if (features.cxxEnabled && s[0] == 'd') {
if (s[1] == 'y') { if (s[1] == 'y') {
if (s[2] == 'n') { if (s[2] == 'n') {
@@ -1495,6 +1478,31 @@ static inline int classify12(const char *s, LanguageFeatures features)
} }
} }
} }
else if (features.qtEnabled && s[0] == 'Q') {
if (s[1] == '_') {
if (s[2] == 'I') {
if (s[3] == 'N') {
if (s[4] == 'T') {
if (s[5] == 'E') {
if (s[6] == 'R') {
if (s[7] == 'F') {
if (s[8] == 'A') {
if (s[9] == 'C') {
if (s[10] == 'E') {
if (s[11] == 'S') {
return T_Q_INTERFACES;
}
}
}
}
}
}
}
}
}
}
}
}
return T_IDENTIFIER; return T_IDENTIFIER;
} }
@@ -1526,7 +1534,8 @@ static inline int classify13(const char *s, LanguageFeatures features)
} }
} }
} }
} else if (features.cxx11Enabled && s[0] == 's') { }
else if (features.cxx11Enabled && s[0] == 's') {
if (s[1] == 't') { if (s[1] == 't') {
if (s[2] == 'a') { if (s[2] == 'a') {
if (s[3] == 't') { if (s[3] == 't') {
@@ -1556,6 +1565,40 @@ static inline int classify13(const char *s, LanguageFeatures features)
return T_IDENTIFIER; return T_IDENTIFIER;
} }
static inline int classify14(const char *s, LanguageFeatures features)
{
if (features.qtEnabled && s[0] == 'Q') {
if (s[1] == '_') {
if (s[2] == 'P') {
if (s[3] == 'R') {
if (s[4] == 'I') {
if (s[5] == 'V') {
if (s[6] == 'A') {
if (s[7] == 'T') {
if (s[8] == 'E') {
if (s[9] == '_') {
if (s[10] == 'S') {
if (s[11] == 'L') {
if (s[12] == 'O') {
if (s[13] == 'T') {
return T_Q_PRIVATE_SLOT;
}
}
}
}
}
}
}
}
}
}
}
}
}
}
return T_IDENTIFIER;
}
static inline int classify16(const char *s, LanguageFeatures features) static inline int classify16(const char *s, LanguageFeatures features)
{ {
if (features.cxxEnabled && s[0] == 'r') { if (features.cxxEnabled && s[0] == 'r') {
@@ -1594,40 +1637,6 @@ static inline int classify16(const char *s, LanguageFeatures features)
return T_IDENTIFIER; return T_IDENTIFIER;
} }
static inline int classify14(const char *s, LanguageFeatures features)
{
if (features.qtEnabled && s[0] == 'Q') {
if (s[1] == '_') {
if (s[2] == 'P') {
if (s[3] == 'R') {
if (s[4] == 'I') {
if (s[5] == 'V') {
if (s[6] == 'A') {
if (s[7] == 'T') {
if (s[8] == 'E') {
if (s[9] == '_') {
if (s[10] == 'S') {
if (s[11] == 'L') {
if (s[12] == 'O') {
if (s[13] == 'T') {
return T_Q_PRIVATE_SLOT;
}
}
}
}
}
}
}
}
}
}
}
}
}
}
return T_IDENTIFIER;
}
static inline int classify18(const char *s, LanguageFeatures features) static inline int classify18(const char *s, LanguageFeatures features)
{ {
if (features.qtEnabled && s[0] == 'Q') { if (features.qtEnabled && s[0] == 'Q') {
@@ -1714,8 +1723,8 @@ static inline int classify19(const char *s, LanguageFeatures features)
return T_IDENTIFIER; return T_IDENTIFIER;
} }
int Lexer::classify(const char *s, int n, LanguageFeatures features)
int Lexer::classify(const char *s, int n, LanguageFeatures features) { {
switch (n) { switch (n) {
case 2: return classify2(s, features); case 2: return classify2(s, features);
case 3: return classify3(s, features); case 3: return classify3(s, features);
@@ -1737,7 +1746,13 @@ int Lexer::classify(const char *s, int n, LanguageFeatures features) {
} // switch } // switch
} }
static inline int classifyOperator2(const char *s) { // === keywords end
// === keywords begin
static inline int classifyOperator2(const char *s)
{
if (s[0] == 'o') { if (s[0] == 'o') {
if (s[1] == 'r') { if (s[1] == 'r') {
return T_OR; return T_OR;
@@ -1746,7 +1761,8 @@ static inline int classifyOperator2(const char *s) {
return T_IDENTIFIER; return T_IDENTIFIER;
} }
static inline int classifyOperator3(const char *s) { static inline int classifyOperator3(const char *s)
{
if (s[0] == 'a') { if (s[0] == 'a') {
if (s[1] == 'n') { if (s[1] == 'n') {
if (s[2] == 'd') { if (s[2] == 'd') {
@@ -1771,7 +1787,8 @@ static inline int classifyOperator3(const char *s) {
return T_IDENTIFIER; return T_IDENTIFIER;
} }
static inline int classifyOperator5(const char *s) { static inline int classifyOperator5(const char *s)
{
if (s[0] == 'b') { if (s[0] == 'b') {
if (s[1] == 'i') { if (s[1] == 'i') {
if (s[2] == 't') { if (s[2] == 't') {
@@ -1808,7 +1825,8 @@ static inline int classifyOperator5(const char *s) {
return T_IDENTIFIER; return T_IDENTIFIER;
} }
static inline int classifyOperator6(const char *s) { static inline int classifyOperator6(const char *s)
{
if (s[0] == 'a') { if (s[0] == 'a') {
if (s[1] == 'n') { if (s[1] == 'n') {
if (s[2] == 'd') { if (s[2] == 'd') {
@@ -1864,7 +1882,8 @@ static inline int classifyOperator6(const char *s) {
return T_IDENTIFIER; return T_IDENTIFIER;
} }
int Lexer::classifyOperator(const char *s, int n) { int Lexer::classifyOperator(const char *s, int n)
{
switch (n) { switch (n) {
case 2: return classifyOperator2(s); case 2: return classifyOperator2(s);
case 3: return classifyOperator3(s); case 3: return classifyOperator3(s);
@@ -1874,5 +1893,6 @@ int Lexer::classifyOperator(const char *s, int n) {
} // switch } // switch
} }
// === keywords end
} // namespace CPlusPlus } // namespace CPlusPlus

145
src/libs/3rdparty/cplusplus/Keywords.kwgen vendored
View File

@@ -1,55 +1,99 @@
// Copyright (c) 2008 Roberto Raggi <roberto.raggi@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#include "Lexer.h" #include "Lexer.h"
#include "Token.h" #include "Token.h"
namespace CPlusPlus {
%token-prefix=T_ %token-prefix=T_
%toupper %toupper
%no-enums %no-enums
%namespace=Lexer %namespace=Lexer
%no-namespace-for-tokens
%pre-check-argument=LanguageFeatures features
%function-name=classify
%% %%
__asm __asm
__asm__ __asm__
__attribute __attribute
__attribute__ __attribute__
__alignof__
__const __const
__const__ __const__
__inline __inline
__inline__ __inline__
__thread
__typeof __typeof
__typeof__ __typeof__
__volatile __volatile
__volatile__ __volatile__
asm asm
auto
bool
break break
case case
catch
char char
class
const const
const_cast
continue continue
default default
delete
do do
double double
dynamic_cast
else else
enum enum
explicit
export
extern extern
false
float float
for for
friend
goto goto
if if
inline inline
int int
long long
register
return
short
signed
sizeof
static
struct
switch
typedef
typeof
union
unsigned
void
volatile
while
%pre-check=features.cxxEnabled
__decltype
auto
bool
catch
class
const_cast
delete
dynamic_cast
explicit
export
false
friend
mutable mutable
namespace namespace
new new
@@ -57,30 +101,79 @@ operator
private private
protected protected
public public
register
reinterpret_cast reinterpret_cast
return
short
signed
sizeof
static
static_cast static_cast
struct
switch
template template
this this
throw throw
true true
try try
typedef
typeid typeid
typename typename
typeof
union
unsigned
using using
virtual virtual
void
volatile
wchar_t wchar_t
while
%pre-check=features.cxx11Enabled
alignas
alignof
char16_t
char32_t
constexpr
decltype
noexcept
nullptr
static_assert
thread_local
%pre-check=features.qtKeywordsEnabled
emit
foreach
Q_EMIT
Q_SLOT
SIGNAL
signals
slots
%pre-check=features.qtMocRunEnabled
Q_Q
Q_D
%pre-check=features.qtEnabled
Q_DECLARE_INTERFACE
Q_ENUMS
Q_FLAGS
Q_FOREACH
Q_GADGET
Q_INVOKABLE
Q_INTERFACES
Q_OBJECT
Q_OVERRIDE
Q_PRIVATE_PROPERTY
Q_PRIVATE_SLOT
Q_PROPERTY
Q_SIGNAL
Q_SIGNALS
Q_SLOTS
SLOT
%%
%pre-check-argument=
%function-name=classifyOperator
%%
or
and
not
xor
bitor
compl
or_eq
and_eq
bitand
not_eq
xor_eq
%%
} // namespace CPlusPlus

5
src/libs/3rdparty/cplusplus/Token.h vendored
View File

@@ -293,6 +293,11 @@ enum Kind {
T___ATTRIBUTE = T___ATTRIBUTE__, T___ATTRIBUTE = T___ATTRIBUTE__,
T___ALIGNOF__ = T_ALIGNOF, T___ALIGNOF__ = T_ALIGNOF,
T_SLOTS = T_Q_SLOTS,
T_FOREACH = T_Q_FOREACH,
T_SIGNALS = T_Q_SIGNALS,
T_Q_OVERRIDE = T_Q_PROPERTY,
}; };
class CPLUSPLUS_EXPORT Token class CPLUSPLUS_EXPORT Token

401
src/tools/3rdparty/cplusplus-keywordgen/cplusplus-keywordgen.cpp vendored
View File

@@ -17,23 +17,29 @@
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN // IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// ### TODO: Rewrite me.
#include <algorithm> #include <algorithm>
#include <cctype> #include <cctype>
#include <cstdlib> #include <cstdlib>
#include <fstream>
#include <functional> #include <functional>
#include <iostream> #include <iostream>
#include <list> #include <list>
#include <map> #include <map>
#include <set> #include <set>
#include <sstream>
#include <string> #include <string>
#include <vector> #include <vector>
class State; class State;
class DottedItem; class DottedItem;
typedef std::list<std::string> RuleList; struct Rule
{
std::string keyword;
std::string preCheck;
};
typedef std::list<Rule> RuleList;
typedef RuleList::iterator RulePtr; typedef RuleList::iterator RulePtr;
typedef std::list<State> StateList; typedef std::list<State> StateList;
typedef StateList::iterator StatePtr; typedef StateList::iterator StatePtr;
@@ -60,7 +66,7 @@ public:
bool operator!=(const DottedItem &other) const { return !operator==(other); } bool operator!=(const DottedItem &other) const { return !operator==(other); }
bool terminal() const { return dot == rule->end(); } bool terminal() const { return dot == rule->keyword.end(); }
DottedItem next() const DottedItem next() const
{ {
@@ -101,14 +107,52 @@ public:
return intern(State(n.begin(), n.end())); return intern(State(n.begin(), n.end()));
} }
std::set<char> firsts() std::vector<char> firsts()
{ {
std::set<char> s; std::set<char> charsSet;
for (DottedItemPtr it = first_item(); it != last_item(); ++it) { for (DottedItemPtr it = first_item(); it != last_item(); ++it) {
if (!it->terminal()) if (!it->terminal())
s.insert(*it->dot); charsSet.insert(*it->dot);
} }
return s; std::vector<char> charsOrderedUpperToBack; // to minimize Keywords.cpp diff
charsOrderedUpperToBack.reserve(charsSet.size());
for (char c : charsSet) {
charsOrderedUpperToBack.push_back(c);
}
std::stable_partition(charsOrderedUpperToBack.begin(),
charsOrderedUpperToBack.end(),
[](char c) {
return !std::isupper(static_cast<unsigned char>(c));
});
return charsOrderedUpperToBack;
}
bool hasPreChecks()
{
for (DottedItemPtr it = first_item(); it != last_item(); ++it) {
if (!it->rule->preCheck.empty()) {
return true;
}
}
return false;
}
std::string commonPreCheck(char ch)
{
std::string result;
for (DottedItemPtr it = first_item(); it != last_item(); ++it) {
if (!it->terminal() && *it->dot == ch) {
if (result.empty()) {
if (it->rule->preCheck.empty()) {
return "";
}
result = (it->rule->preCheck);
} else if (result != it->rule->preCheck) {
return "";
}
}
}
return result;
} }
size_t item_count() const { return _items.size(); } size_t item_count() const { return _items.size(); }
@@ -127,7 +171,7 @@ public:
{ {
std::vector<DottedItem> items; std::vector<DottedItem> items;
for (; first != last; ++first) for (; first != last; ++first)
items.push_back(DottedItem(first, first->begin())); items.push_back(DottedItem(first, first->keyword.begin()));
return intern(State(items.begin(), items.end())); return intern(State(items.begin(), items.end()));
} }
@@ -151,6 +195,15 @@ static std::string option_token_prefix = "Token_";
static std::string option_char_type = "char"; static std::string option_char_type = "char";
static std::string option_unicode_function = ""; static std::string option_unicode_function = "";
static std::string option_preCheck_arg_type;
static std::string option_preCheck_arg_name;
static std::string option_tokens_namespace;
static std::string option_function_name = "classify";
std::stringstream input;
std::stringstream output;
std::string token_id(const std::string &id) std::string token_id(const std::string &id)
{ {
std::string token = option_token_prefix; std::string token = option_token_prefix;
@@ -167,7 +220,7 @@ std::string token_id(const std::string &id)
bool starts_with(const std::string &line, const std::string &text) bool starts_with(const std::string &line, const std::string &text)
{ {
if (text.length() < line.length()) { if (text.length() <= line.length()) {
return std::equal(line.begin(), line.begin() + text.size(), text.begin()); return std::equal(line.begin(), line.begin() + text.size(), text.begin());
} }
return false; return false;
@@ -176,16 +229,27 @@ bool starts_with(const std::string &line, const std::string &text)
void doit(State &state) void doit(State &state)
{ {
static int depth{0}; static int depth{0};
static int preCheckDepth{0};
++depth; ++depth;
std::string indent(depth * 2, ' '); std::string indent(depth * 2, ' ');
std::set<char> firsts = state.firsts(); std::vector<char> firsts = state.firsts();
for (std::set<char>::iterator it = firsts.begin(); it != firsts.end(); ++it) { for (std::vector<char>::iterator it = firsts.begin(); it != firsts.end(); ++it) {
std::string _else = it == firsts.begin() ? "" : "else "; std::string _else = it == firsts.begin() ? "" : "else ";
std::cout << indent << _else << "if (s[" << (depth - 1) << "]" << option_unicode_function output << indent << _else << "if (";
<< " == '" << *it << "') {" << std::endl; if (preCheckDepth == 0) {
std::string commonPreCheck = state.commonPreCheck(*it);
if (!commonPreCheck.empty()) {
output << commonPreCheck << " && ";
preCheckDepth++;
}
} else if (preCheckDepth > 0) {
preCheckDepth++;
}
output << "s[" << (depth - 1) << "]" << option_unicode_function << " == '" << *it << "'";
output << ") {" << std::endl;
State &next_state = state.next(*it); State &next_state = state.next(*it);
bool found = false; bool found = false;
@@ -196,58 +260,76 @@ void doit(State &state)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
found = true; found = true;
std::cout << indent << " return " << option_namespace_name << token_id(*item->rule) output << indent << " return " << option_tokens_namespace
<< ";" << std::endl; << token_id(item->rule->keyword) << ";" << std::endl;
} }
} }
if (!found) if (!found)
doit(next_state); doit(next_state);
std::cout << indent << "}" << std::endl; if (preCheckDepth > 0)
preCheckDepth--;
output << indent << "}" << std::endl;
} }
--depth; --depth;
} }
void gen_classify_n(State &start_state, int N) void gen_classify_n(State &start_state, size_t N)
{ {
std::cout << "static inline int classify" << N << "(const " << option_char_type << " *s) {" output << "static inline int " << option_function_name << N << "(const " << option_char_type
<< std::endl; << " *s";
if (!option_preCheck_arg_type.empty()) {
output << ", " << option_preCheck_arg_type;
if (start_state.hasPreChecks()) {
output << " " << option_preCheck_arg_name;
}
}
output << ")" << std::endl << "{" << std::endl;
doit(start_state); doit(start_state);
std::cout << " return " << option_namespace_name << token_id("identifier") << ";" << std::endl output << " return " << option_tokens_namespace << token_id("identifier") << ";" << std::endl
<< "}" << std::endl << "}" << std::endl
<< std::endl; << std::endl;
} }
void gen_classify(const std::multimap<size_t, std::string> &keywords) void gen_classify(const std::multimap<size_t, Rule> &keywords)
{ {
std::cout << "int " << option_namespace_name << "classify(const " << option_char_type output << "int " << option_namespace_name << option_function_name << "(const "
<< " *s, int n) {" << std::endl << option_char_type << " *s, int n";
<< " switch (n) {" << std::endl; if (!option_preCheck_arg_type.empty()) {
std::multimap<size_t, std::string>::const_iterator it = keywords.begin(); output << ", " << option_preCheck_arg_type << " " << option_preCheck_arg_name;
}
output << ")" << std::endl;
output << "{" << std::endl << " switch (n) {" << std::endl;
std::multimap<size_t, Rule>::const_iterator it = keywords.begin();
while (it != keywords.end()) { while (it != keywords.end()) {
size_t size = it->first; size_t size = it->first;
std::cout << " case " << size << ": return classify" << size << "(s);" << std::endl; output << " case " << size << ": return " << option_function_name << size << "(s";
if (!option_preCheck_arg_type.empty()) {
output << ", " << option_preCheck_arg_name;
}
output << ");" << std::endl;
do { do {
++it; ++it;
} while (it != keywords.end() && it->first == size); } while (it != keywords.end() && it->first == size);
} }
std::cout << " default: return " << option_namespace_name << token_id("identifier") << ";" output << " default: return " << option_tokens_namespace << token_id("identifier") << ";"
<< std::endl << std::endl
<< " } // switch" << std::endl << " } // switch" << std::endl
<< "}" << std::endl << "}" << std::endl
<< std::endl; << std::endl;
} }
void gen_enums(const std::multimap<size_t, std::string> &keywords) void gen_enums(const std::multimap<size_t, Rule> &keywords)
{ {
std::cout << "enum {" << std::endl; output << "enum {" << std::endl;
std::multimap<size_t, std::string>::const_iterator it = keywords.begin(); std::multimap<size_t, Rule>::const_iterator it = keywords.begin();
for (; it != keywords.end(); ++it) { for (; it != keywords.end(); ++it) {
std::cout << " " << token_id(it->second) << "," << std::endl; output << " " << token_id(it->second.keyword) << "," << std::endl;
} }
std::cout << " " << token_id("identifier") << std::endl << "};" << std::endl << std::endl; output << " " << token_id("identifier") << std::endl << "};" << std::endl << std::endl;
} }
inline bool not_whitespace_p(char ch) inline bool not_whitespace_p(char ch)
@@ -258,6 +340,11 @@ inline bool not_whitespace_p(char ch)
int main(int argc, char *argv[]) int main(int argc, char *argv[])
{ {
const std::string ns = "--namespace="; const std::string ns = "--namespace=";
const std::string inputFileOpt = "--input";
const std::string outputFileOpt = "--output";
std::string inputFilename;
std::string outputFilename;
for (int i = 0; i < argc; ++i) { for (int i = 0; i < argc; ++i) {
const std::string arg(argv[i]); const std::string arg(argv[i]);
@@ -266,13 +353,33 @@ int main(int argc, char *argv[])
else if (starts_with(arg, ns)) { else if (starts_with(arg, ns)) {
option_namespace_name.assign(arg.begin() + ns.size(), arg.end()); option_namespace_name.assign(arg.begin() + ns.size(), arg.end());
option_namespace_name += "::"; option_namespace_name += "::";
} else if (arg == inputFileOpt && i + 1 < argc) {
inputFilename = argv[i + 1];
++i;
} else if (arg == outputFileOpt && i + 1 < argc) {
outputFilename = argv[i + 1];
++i;
}else if (arg == "--help" || arg == "-h") {
std::cout << "usage: cplusplus-keywordgen [--input <kwgen file>] [--output <cpp file>]"
<< std::endl;
std::cout << "\t If no input or output specified: std::cin/cout will be used"
<< std::endl;
exit(EXIT_SUCCESS);
} }
} }
std::multimap<size_t, std::string> keywords; if (inputFilename.empty()) {
std::string textline; std::string textline;
while (getline(std::cin, textline)) {
bool readKeywords = false; input << textline << std::endl;
}
} else {
std::ifstream fileInput(inputFilename, std::ios_base::in);
std::string textline;
while (getline(fileInput, textline)) {
input << textline << std::endl;
}
}
const std::string opt_no_enums = "%no-enums"; const std::string opt_no_enums = "%no-enums";
const std::string opt_toupper = "%toupper"; const std::string opt_toupper = "%toupper";
@@ -281,73 +388,167 @@ int main(int argc, char *argv[])
const std::string opt_char_type = "%char-type="; const std::string opt_char_type = "%char-type=";
const std::string opt_unicode_function = "%unicode-function="; const std::string opt_unicode_function = "%unicode-function=";
while (getline(std::cin, textline)) { const std::string opt_preCheck_arg = "%pre-check-argument=";
// remove trailing spaces const std::string opt_function_name = "%function-name=";
textline.assign(textline.begin(),
const std::string opt_no_namespace_for_tokens = "%no-namespace-for-tokens";
// this may be only in keywords section
const std::string preCheckOpt = "%pre-check=";
bool useNamespaceForTokens = true;
bool finished = false;
while (!finished) {
finished = true;
bool readKeywords = false;
std::string preCheckValue;
std::multimap<size_t, Rule> keywords;
std::string textline;
while (getline(input, textline)) {
// remove trailing spaces
textline
.assign(textline.begin(),
std::find_if(textline.rbegin(), textline.rend(), not_whitespace_p).base()); std::find_if(textline.rbegin(), textline.rend(), not_whitespace_p).base());
if (!readKeywords) { if (!readKeywords) {
if (textline.size() >= 2 && textline[0] == '%') { if (textline.size() >= 2 && textline[0] == '%') {
if (textline[1] == '%') { if (textline[1] == '%') {
readKeywords = true; readKeywords = true;
} else if (textline == opt_no_enums) {
option_no_enums = true; static bool generatedMessageAdded=false;
} else if (textline == opt_toupper) { if(!generatedMessageAdded){
option_toupper = true; generatedMessageAdded=true;
} else if (starts_with(textline, opt_tok_prefix)) { output
option_token_prefix.assign(textline.begin() + opt_tok_prefix.size(), << "// === following code is generated with cplusplus-keywordgen tool"
textline.end()); << std::endl;
} else if (starts_with(textline, opt_char_type)) { for (auto it = inputFilename.rbegin(); it != inputFilename.rend(); ++it) {
option_char_type.assign(textline.begin() + opt_char_type.size(), textline.end()); if (*it == '\\' || *it == '/') {
} else if (starts_with(textline, opt_unicode_function)) { output
option_unicode_function.assign(textline.begin() + opt_unicode_function.size(), << "// === from source file: "
<< inputFilename.substr(std::distance(it, inputFilename.rend()))
<< std::endl;
break;
}
}
output << std::endl;
}
output << "// === keywords begin" << std::endl;
output << std::endl;
} else if (textline == opt_no_enums) {
option_no_enums = true;
} else if (textline == opt_toupper) {
option_toupper = true;
} else if (starts_with(textline, opt_tok_prefix)) {
option_token_prefix.assign(textline.begin() + opt_tok_prefix.size(),
textline.end()); textline.end());
} else if (starts_with(textline, opt_ns)) { } else if (starts_with(textline, opt_char_type)) {
option_namespace_name.assign(textline.begin() + opt_ns.size(), textline.end()); option_char_type.assign(textline.begin() + opt_char_type.size(),
option_namespace_name += "::"; textline.end());
} else if (starts_with(textline, opt_unicode_function)) {
option_unicode_function.assign(textline.begin()
+ opt_unicode_function.size(),
textline.end());
} else if (starts_with(textline, opt_ns)) {
option_namespace_name.assign(textline.begin() + opt_ns.size(),
textline.end());
option_namespace_name += "::";
if (useNamespaceForTokens) {
option_tokens_namespace = option_namespace_name;
}
} else if (starts_with(textline, opt_preCheck_arg)) {
std::string::size_type spacePos = textline.find(' ',
opt_preCheck_arg.size());
if (spacePos == std::string::npos) {
option_preCheck_arg_type.clear();
option_preCheck_arg_name.clear();
} else {
option_preCheck_arg_type
= textline.substr(opt_preCheck_arg.size(),
spacePos - opt_preCheck_arg.size());
option_preCheck_arg_name = textline.substr(spacePos + 1);
}
} else if (starts_with(textline, opt_function_name)) {
option_function_name.assign(textline.begin() + opt_function_name.size(),
textline.end());
} else if (textline == opt_no_namespace_for_tokens) {
useNamespaceForTokens = false;
option_tokens_namespace.clear();
}
continue;
}
output << textline << std::endl;
} else {
if (textline.empty())
continue;
if (textline == "%%") {
finished = false;
break;
} }
continue; if (starts_with(textline, preCheckOpt)) {
} preCheckValue = textline.substr(preCheckOpt.size());
std::cout << textline << std::endl;
} else {
if (textline.empty())
continue;
std::string::iterator start = textline.begin();
while (start != textline.end() && std::isspace(*start))
++start;
std::string::iterator stop = start;
while (stop != textline.end() && (std::isalnum(*stop) || *stop == '_'))
++stop;
if (start != stop) {
std::string keyword(start, stop);
if (keyword == "identifier") {
std::cerr << "*** Error. `identifier' is reserved" << std::endl;
exit(EXIT_FAILURE);
} }
keywords.insert(std::make_pair(keyword.size(), keyword)); std::string::iterator start = textline.begin();
while (start != textline.end() && std::isspace(*start))
++start;
std::string::iterator stop = start;
while (stop != textline.end() && (std::isalnum(*stop) || *stop == '_'))
++stop;
if (start != stop) {
Rule rule;
rule.keyword.assign(start, stop);
if (rule.keyword == "identifier") {
std::cerr << "*** Error. `identifier' is reserved" << std::endl;
exit(EXIT_FAILURE);
}
rule.preCheck = preCheckValue;
keywords.insert(std::make_pair(rule.keyword.size(), rule));
}
} }
} }
if (readKeywords) {
if (!option_no_enums)
gen_enums(keywords);
std::multimap<size_t, Rule>::iterator it = keywords.begin();
while (it != keywords.end()) {
size_t size = it->first;
RuleList rules;
do {
rules.push_back(it->second);
++it;
} while (it != keywords.end() && it->first == size);
gen_classify_n(State::start(rules.begin(), rules.end()), size);
State::reset();
}
gen_classify(keywords);
output << "// === keywords end" << std::endl;
}
} }
if (!option_no_enums) if (outputFilename.empty()) {
gen_enums(keywords); std::string textline;
while (getline(output, textline)) {
std::multimap<size_t, std::string>::iterator it = keywords.begin(); std::cout << textline << std::endl;
while (it != keywords.end()) { }
size_t size = it->first; } else {
RuleList rules; std::ofstream outFile(outputFilename, std::ios_base::out);
do { std::string textline;
rules.push_back(it->second); while (getline(output, textline)) {
++it; outFile << textline << std::endl;
} while (it != keywords.end() && it->first == size); }
gen_classify_n(State::start(rules.begin(), rules.end()), size); std::cout << "Generated: " << outputFilename << std::endl;
State::reset();
} }
gen_classify(keywords);
} }