boost/wave
2
0
Fork 0
mirror of https://github.com/boostorg/wave.git synced 2026-01-19 04:42:16 +00:00
Code Issues Projects Releases Wiki Activity

Tweak PR #241 for correctness and tidy code

Browse Source 
- Fix RE2C code for numbers (binary and digit separators)
- Revert to RE2C version 1.0.2, for now
- Revamp token ids to minimize changes
- Restore existing and more accurate name OPTSIGN in slex
- Add binary literal support to lexertl
- Fix xlex support for size_t literals
- Add test tokens for octal, binary, and hex literals
This commit is contained in:
Jeff Trull
2025-10-24 18:28:10 -07:00
committed by Jeff Trull
parent 50b322dcc9
commit 2408633f16
10 changed files with 11447 additions and 10838 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/boost/wave/cpplexer/re2clex/cpp.re
View File

@@ -23,10 +23,11 @@ OctalDigit = [0-7];
Digit = [0-9];
HexDigit = [a-fA-F0-9];
BinaryDigit = [01];
Integer = (("0" [xX] HexDigit (("'" HexDigit) | HexDigit)*) | ("0" [bB] BinaryDigit (("'" BinaryDigit) | BinaryDigit)*) | ("0" OctalDigit ((("'" OctalDigit) | OctalDigit)*)) | ([1-9] ("'" Digit | Digit)*));
Integer = (("0" [xX] HexDigit ([']? HexDigit)*) | ("0" ([']? OctalDigit)*) | ([1-9] ([']? Digit)*) | ("0" [bB] BinaryDigit ([']? BinaryDigit)*));
ExponentStart = [Ee] [+-];
ExponentPart = [Ee] [+-]? Digit (("'" Digit)| Digit)*;
FractionalConstant = (Digit? ((Digit "'" Digit)|Digit)* "." Digit (("'" Digit)| Digit)*) | (Digit (("'" Digit)| Digit)* ".");
SeparatedDigits = Digit ([']? Digit)*;
ExponentPart = [Ee] [+-]? SeparatedDigits;
FractionalConstant = (SeparatedDigits? "." SeparatedDigits) | (SeparatedDigits ".");
FloatingSuffix = [fF] [lL]? | [lL] [fF]?;
IntegerSuffix = [uU] [lL]? | [lL] [uU]?;
LongIntegerSuffix = [uU] ("ll" | "LL") | ("ll" | "LL") [uU]?;

10882
include/boost/wave/cpplexer/re2clex/cpp_re.inc
View File

File diff suppressed because it is too large Load Diff

7
include/boost/wave/cpplexer/re2clex/strict_cpp.re
View File

@@ -23,10 +23,11 @@ OctalDigit = [0-7];
Digit = [0-9];
HexDigit = [a-fA-F0-9];
BinaryDigit = [01];
Integer = (("0" [xX] HexDigit (("'" HexDigit) | HexDigit)*) | ("0" [bB] BinaryDigit (("'" BinaryDigit) | BinaryDigit)*) | ("0" OctalDigit ((("'" OctalDigit) | OctalDigit)*)) | ([1-9] ("'" Digit | Digit)*));
Integer = (("0" [xX] HexDigit ([']? HexDigit)*) | ("0" ([']? OctalDigit)*) | ([1-9] ([']? Digit)*) | ("0" [bB] BinaryDigit ([']? BinaryDigit)*));
ExponentStart = [Ee] [+-];
ExponentPart = [Ee] [+-]? Digit (("'" Digit)| Digit)*;
FractionalConstant = (Digit? ((Digit "'" Digit)|Digit)* "." Digit (("'" Digit)| Digit)*) | (Digit (("'" Digit)| Digit)* ".");
SeparatedDigits = Digit ([']? Digit)*;
ExponentPart = [Ee] [+-]? SeparatedDigits;
FractionalConstant = (SeparatedDigits? "." SeparatedDigits) | (SeparatedDigits ".");
FloatingSuffix = [fF] [lL]? | [lL] [fF]?;
IntegerSuffix = [uU] [lL]? | [lL] [uU]?;
LongIntegerSuffix = [uU] ("ll" | "LL") | ("ll" | "LL") [uU]?;

11006
include/boost/wave/cpplexer/re2clex/strict_cpp_re.inc
View File

File diff suppressed because it is too large Load Diff

138
include/boost/wave/token_ids.hpp
View File

@@ -237,85 +237,87 @@ enum token_id : std::uint32_t {
T_HEXAINT = TOKEN_FROM_ID(383, IntegerLiteralTokenType),
T_INTLIT = TOKEN_FROM_ID(384, IntegerLiteralTokenType),
T_LONGINTLIT = TOKEN_FROM_ID(385, IntegerLiteralTokenType),
T_SIZETLIT = TOKEN_FROM_ID(386, IntegerLiteralTokenType),
T_FLOATLIT = TOKEN_FROM_ID(387, FloatingLiteralTokenType),
T_FIXEDPOINTLIT = TOKEN_FROM_ID(387, FloatingLiteralTokenType|AltTokenType), // IDL specific
T_CCOMMENT = TOKEN_FROM_ID(388, WhiteSpaceTokenType|AltTokenType),
T_CPPCOMMENT = TOKEN_FROM_ID(389, WhiteSpaceTokenType|AltTokenType),
T_CHARLIT = TOKEN_FROM_ID(390, CharacterLiteralTokenType),
T_STRINGLIT = TOKEN_FROM_ID(391, StringLiteralTokenType),
T_CONTLINE = TOKEN_FROM_ID(392, EOLTokenType),
T_SPACE = TOKEN_FROM_ID(393, WhiteSpaceTokenType),
T_SPACE2 = TOKEN_FROM_ID(394, WhiteSpaceTokenType),
T_NEWLINE = TOKEN_FROM_ID(395, EOLTokenType),
T_GENERATEDNEWLINE = TOKEN_FROM_ID(395, EOLTokenType|AltTokenType),
T_POUND_POUND = TOKEN_FROM_ID(396, OperatorTokenType),
T_POUND_POUND_ALT = TOKEN_FROM_ID(396, OperatorTokenType|AltTokenType),
T_POUND_POUND_TRIGRAPH = TOKEN_FROM_ID(396, OperatorTokenType|TriGraphTokenType),
T_POUND = TOKEN_FROM_ID(397, OperatorTokenType),
T_POUND_ALT = TOKEN_FROM_ID(397, OperatorTokenType|AltTokenType),
T_POUND_TRIGRAPH = TOKEN_FROM_ID(397, OperatorTokenType|TriGraphTokenType),
T_ANY = TOKEN_FROM_ID(398, UnknownTokenType),
T_ANY_TRIGRAPH = TOKEN_FROM_ID(398, UnknownTokenType|TriGraphTokenType),
T_PP_INCLUDE = TOKEN_FROM_ID(399, PPTokenType),
T_PP_QHEADER = TOKEN_FROM_ID(400, PPTokenType),
T_PP_HHEADER = TOKEN_FROM_ID(401, PPTokenType),
T_PP_INCLUDE_NEXT = TOKEN_FROM_ID(399, PPTokenType|AltTokenType),
T_PP_QHEADER_NEXT = TOKEN_FROM_ID(400, PPTokenType|AltTokenType),
T_PP_HHEADER_NEXT = TOKEN_FROM_ID(401, PPTokenType|AltTokenType),
T_EOF = TOKEN_FROM_ID(402, EOFTokenType), // end of file reached
T_EOI = TOKEN_FROM_ID(403, EOFTokenType), // end of input reached
T_PP_NUMBER = TOKEN_FROM_ID(404, InternalTokenType),
T_FLOATLIT = TOKEN_FROM_ID(386, FloatingLiteralTokenType),
T_FIXEDPOINTLIT = TOKEN_FROM_ID(386, FloatingLiteralTokenType|AltTokenType), // IDL specific
T_CCOMMENT = TOKEN_FROM_ID(387, WhiteSpaceTokenType|AltTokenType),
T_CPPCOMMENT = TOKEN_FROM_ID(388, WhiteSpaceTokenType|AltTokenType),
T_CHARLIT = TOKEN_FROM_ID(389, CharacterLiteralTokenType),
T_STRINGLIT = TOKEN_FROM_ID(390, StringLiteralTokenType),
T_CONTLINE = TOKEN_FROM_ID(391, EOLTokenType),
T_SPACE = TOKEN_FROM_ID(392, WhiteSpaceTokenType),
T_SPACE2 = TOKEN_FROM_ID(393, WhiteSpaceTokenType),
T_NEWLINE = TOKEN_FROM_ID(394, EOLTokenType),
T_GENERATEDNEWLINE = TOKEN_FROM_ID(394, EOLTokenType|AltTokenType),
T_POUND_POUND = TOKEN_FROM_ID(395, OperatorTokenType),
T_POUND_POUND_ALT = TOKEN_FROM_ID(395, OperatorTokenType|AltTokenType),
T_POUND_POUND_TRIGRAPH = TOKEN_FROM_ID(395, OperatorTokenType|TriGraphTokenType),
T_POUND = TOKEN_FROM_ID(396, OperatorTokenType),
T_POUND_ALT = TOKEN_FROM_ID(396, OperatorTokenType|AltTokenType),
T_POUND_TRIGRAPH = TOKEN_FROM_ID(396, OperatorTokenType|TriGraphTokenType),
T_ANY = TOKEN_FROM_ID(397, UnknownTokenType),
T_ANY_TRIGRAPH = TOKEN_FROM_ID(397, UnknownTokenType|TriGraphTokenType),
T_PP_INCLUDE = TOKEN_FROM_ID(398, PPTokenType),
T_PP_QHEADER = TOKEN_FROM_ID(399, PPTokenType),
T_PP_HHEADER = TOKEN_FROM_ID(400, PPTokenType),
T_PP_INCLUDE_NEXT = TOKEN_FROM_ID(398, PPTokenType|AltTokenType),
T_PP_QHEADER_NEXT = TOKEN_FROM_ID(399, PPTokenType|AltTokenType),
T_PP_HHEADER_NEXT = TOKEN_FROM_ID(400, PPTokenType|AltTokenType),
T_EOF = TOKEN_FROM_ID(401, EOFTokenType), // end of file reached
T_EOI = TOKEN_FROM_ID(402, EOFTokenType), // end of input reached
T_PP_NUMBER = TOKEN_FROM_ID(403, InternalTokenType),
// MS extensions
T_MSEXT_INT8 = TOKEN_FROM_ID(405, KeywordTokenType),
T_MSEXT_INT16 = TOKEN_FROM_ID(406, KeywordTokenType),
T_MSEXT_INT32 = TOKEN_FROM_ID(407, KeywordTokenType),
T_MSEXT_INT64 = TOKEN_FROM_ID(408, KeywordTokenType),
T_MSEXT_BASED = TOKEN_FROM_ID(409, KeywordTokenType),
T_MSEXT_DECLSPEC = TOKEN_FROM_ID(410, KeywordTokenType),
T_MSEXT_CDECL = TOKEN_FROM_ID(411, KeywordTokenType),
T_MSEXT_FASTCALL = TOKEN_FROM_ID(412, KeywordTokenType),
T_MSEXT_STDCALL = TOKEN_FROM_ID(413, KeywordTokenType),
T_MSEXT_TRY = TOKEN_FROM_ID(414, KeywordTokenType),
T_MSEXT_EXCEPT = TOKEN_FROM_ID(415, KeywordTokenType),
T_MSEXT_FINALLY = TOKEN_FROM_ID(416, KeywordTokenType),
T_MSEXT_LEAVE = TOKEN_FROM_ID(417, KeywordTokenType),
T_MSEXT_INLINE = TOKEN_FROM_ID(418, KeywordTokenType),
T_MSEXT_ASM = TOKEN_FROM_ID(419, KeywordTokenType),
T_MSEXT_INT8 = TOKEN_FROM_ID(404, KeywordTokenType),
T_MSEXT_INT16 = TOKEN_FROM_ID(405, KeywordTokenType),
T_MSEXT_INT32 = TOKEN_FROM_ID(406, KeywordTokenType),
T_MSEXT_INT64 = TOKEN_FROM_ID(407, KeywordTokenType),
T_MSEXT_BASED = TOKEN_FROM_ID(408, KeywordTokenType),
T_MSEXT_DECLSPEC = TOKEN_FROM_ID(409, KeywordTokenType),
T_MSEXT_CDECL = TOKEN_FROM_ID(410, KeywordTokenType),
T_MSEXT_FASTCALL = TOKEN_FROM_ID(411, KeywordTokenType),
T_MSEXT_STDCALL = TOKEN_FROM_ID(412, KeywordTokenType),
T_MSEXT_TRY = TOKEN_FROM_ID(413, KeywordTokenType),
T_MSEXT_EXCEPT = TOKEN_FROM_ID(414, KeywordTokenType),
T_MSEXT_FINALLY = TOKEN_FROM_ID(415, KeywordTokenType),
T_MSEXT_LEAVE = TOKEN_FROM_ID(416, KeywordTokenType),
T_MSEXT_INLINE = TOKEN_FROM_ID(417, KeywordTokenType),
T_MSEXT_ASM = TOKEN_FROM_ID(418, KeywordTokenType),
T_MSEXT_PP_REGION = TOKEN_FROM_ID(420, PPTokenType),
T_MSEXT_PP_ENDREGION = TOKEN_FROM_ID(421, PPTokenType),
T_MSEXT_PP_REGION = TOKEN_FROM_ID(419, PPTokenType),
T_MSEXT_PP_ENDREGION = TOKEN_FROM_ID(420, PPTokenType),
// import is needed to be a keyword for the C++ module Standards proposal
T_IMPORT = TOKEN_FROM_ID(422, KeywordTokenType),
T_IMPORT = TOKEN_FROM_ID(421, KeywordTokenType),
// C++11 keywords
T_ALIGNAS = TOKEN_FROM_ID(423, KeywordTokenType),
T_ALIGNOF = TOKEN_FROM_ID(424, KeywordTokenType),
T_CHAR16_T = TOKEN_FROM_ID(425, KeywordTokenType),
T_CHAR32_T = TOKEN_FROM_ID(426, KeywordTokenType),
T_CONSTEXPR = TOKEN_FROM_ID(427, KeywordTokenType),
T_DECLTYPE = TOKEN_FROM_ID(428, KeywordTokenType),
T_NOEXCEPT = TOKEN_FROM_ID(429, KeywordTokenType),
T_NULLPTR = TOKEN_FROM_ID(430, KeywordTokenType),
T_STATICASSERT = TOKEN_FROM_ID(431, KeywordTokenType),
T_THREADLOCAL = TOKEN_FROM_ID(432, KeywordTokenType),
T_RAWSTRINGLIT = TOKEN_FROM_ID(433, StringLiteralTokenType),
T_ALIGNAS = TOKEN_FROM_ID(422, KeywordTokenType),
T_ALIGNOF = TOKEN_FROM_ID(423, KeywordTokenType),
T_CHAR16_T = TOKEN_FROM_ID(424, KeywordTokenType),
T_CHAR32_T = TOKEN_FROM_ID(425, KeywordTokenType),
T_CONSTEXPR = TOKEN_FROM_ID(426, KeywordTokenType),
T_DECLTYPE = TOKEN_FROM_ID(427, KeywordTokenType),
T_NOEXCEPT = TOKEN_FROM_ID(428, KeywordTokenType),
T_NULLPTR = TOKEN_FROM_ID(429, KeywordTokenType),
T_STATICASSERT = TOKEN_FROM_ID(430, KeywordTokenType),
T_THREADLOCAL = TOKEN_FROM_ID(431, KeywordTokenType),
T_RAWSTRINGLIT = TOKEN_FROM_ID(432, StringLiteralTokenType),
// C++20 keywords
T_CHAR8_T = TOKEN_FROM_ID(434, KeywordTokenType),
T_CONCEPT = TOKEN_FROM_ID(435, KeywordTokenType),
T_CONSTEVAL = TOKEN_FROM_ID(436, KeywordTokenType),
T_CONSTINIT = TOKEN_FROM_ID(437, KeywordTokenType),
T_CO_AWAIT = TOKEN_FROM_ID(438, KeywordTokenType),
T_CO_RETURN = TOKEN_FROM_ID(439, KeywordTokenType),
T_CO_YIELD = TOKEN_FROM_ID(440, KeywordTokenType),
T_REQUIRES = TOKEN_FROM_ID(441, KeywordTokenType),
T_MODULE = TOKEN_FROM_ID(442, KeywordTokenType),
T_CHAR8_T = TOKEN_FROM_ID(433, KeywordTokenType),
T_CONCEPT = TOKEN_FROM_ID(434, KeywordTokenType),
T_CONSTEVAL = TOKEN_FROM_ID(435, KeywordTokenType),
T_CONSTINIT = TOKEN_FROM_ID(436, KeywordTokenType),
T_CO_AWAIT = TOKEN_FROM_ID(437, KeywordTokenType),
T_CO_RETURN = TOKEN_FROM_ID(438, KeywordTokenType),
T_CO_YIELD = TOKEN_FROM_ID(439, KeywordTokenType),
T_REQUIRES = TOKEN_FROM_ID(440, KeywordTokenType),
T_MODULE = TOKEN_FROM_ID(441, KeywordTokenType),
// C++20 operators
T_SPACESHIP = TOKEN_FROM_ID(443, OperatorTokenType),
T_SPACESHIP = TOKEN_FROM_ID(442, OperatorTokenType),
// c++23 features
T_SIZETLIT = TOKEN_FROM_ID(443, IntegerLiteralTokenType),
T_LAST_TOKEN_ID,
T_LAST_TOKEN = ID_FROM_TOKEN(T_LAST_TOKEN_ID & ~PPTokenFlag),

6
samples/cpp_tokens/slex/cpp_slex_lexer.hpp
View File

@@ -134,10 +134,10 @@ private:
#define OCTALDIGIT "[0-7]"
#define DIGIT "[0-9]"
#define HEXDIGIT "[0-9a-fA-F]"
#define OPTSIGN "[-+]?"
#define BINARYDIGIT "[01]"
#define SIGN "[-+]?"
#define EXPSTART "[eE]" "[-+]"
#define EXPONENT "(" "[eE]" SIGN DIGIT "('" DIGIT OR DIGIT ")*" ")"
#define EXPONENT "(" "[eE]" OPTSIGN DIGIT "('" DIGIT OR DIGIT ")*" ")"
#define NONDIGIT "[a-zA-Z_]"
#define INTEGER "(" \
"(0x|0X)" HEXDIGIT "('" HEXDIGIT OR HEXDIGIT ")*" OR \
@@ -509,7 +509,7 @@ lexer<IteratorT, PositionT>::init_data_cpp2a[INIT_DATA_CPP2A_SIZE] =
#undef OCTALDIGIT
#undef HEXDIGIT
#undef NONDIGIT
#undef SIGN
#undef OPTSIGN
#undef EXPSTART
#undef EXPONENT
#undef LONGINTEGER_SUFFIX

6
samples/list_includes/lexertl/lexertl_lexer.hpp
View File

@@ -60,7 +60,7 @@ namespace boost { namespace wave { namespace cpplexer { namespace lexertl
#define INIT_DATA_PP_NUMBER_SIZE 2
#define INIT_DATA_CPP0X_SIZE 15
#define INIT_DATA_CPP2A_SIZE 11
#define INIT_MACRO_DATA_SIZE 29
#define INIT_MACRO_DATA_SIZE 30
#endif // #if BOOST_WAVE_LEXERTL_USE_STATIC_TABLES == 0
// this is just a hack to have a unique token id not otherwise used by Wave
@@ -159,13 +159,15 @@ lexertl<Iterator, Position>::init_macro_data[INIT_MACRO_DATA_SIZE] =
MACRO_DATA("OCTALDIGIT", "[0-7]"),
MACRO_DATA("DIGIT", "[0-9]"),
MACRO_DATA("HEXDIGIT", "[0-9a-fA-F]"),
MACRO_DATA("BINARYDIGIT", "[0-1]"),
MACRO_DATA("OPTSIGN", "[-+]?"),
MACRO_DATA("EXPSTART", "[eE][-+]"),
MACRO_DATA("EXPONENT", "([eE]{OPTSIGN}{DIGIT}('{DIGIT}|{DIGIT})*)"),
MACRO_DATA("NONDIGIT", "[a-zA-Z_]"),
MACRO_DATA("INTEGER", "(" "(0x|0X){HEXDIGIT}('{HEXDIGIT}|{HEXDIGIT})*"
OR "0('{OCTALDIGIT}|{OCTALDIGIT})*"
OR "[1-9]('{DIGIT}|{DIGIT})*" ")"),
OR "[1-9]('{DIGIT}|{DIGIT})*"
OR "(0b|0B){BINARYDIGIT}('{BINARYDIGIT}|{BINARYDIGIT})*" ")"),
MACRO_DATA("INTEGER_SUFFIX", "(" "[uU][lL]?" OR "[lL][uU]?" ")"),
MACRO_DATA("SIZET_SUFFIX", "(" "[uU]?[zZ]" OR "[zZ][uU]?" ")"),
#if BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0

2
samples/token_statistics/xlex/xlex_lexer.hpp
View File

@@ -147,7 +147,7 @@ private:
")"
#define INTEGER_SUFFIX "(" "[uU][lL]?|[lL][uU]?" ")"
#define SIZET_SUFFIX "(" "[uU]?[zZ]|[zZ][uU]?" ")"
#define SIZET_SUFFIX "(" "[uU][zZ]|[zZ][uU]?" ")"
#if BOOST_WAVE_SUPPORT_MS_EXTENSIONS != 0
#define LONGINTEGER_SUFFIX "(" "[uU]" "(" "ll" OR "LL" ")" OR \
"(" "ll" OR "LL" ")" "[uU]" "?" OR \

228
src/token_ids.cpp
View File

@@ -171,69 +171,69 @@ static char const *tok_names[] = {
/* 383 */ "HEXAINT",
/* 384 */ "INTLIT",
/* 385 */ "LONGINTLIT",
/* 386 */ "SIZETLIT",
/* 387 */ "FLOATLIT",
/* 388 */ "CCOMMENT",
/* 389 */ "CPPCOMMENT",
/* 390 */ "CHARLIT",
/* 391 */ "STRINGLIT",
/* 392 */ "CONTLINE",
/* 393 */ "SPACE",
/* 394 */ "SPACE2",
/* 395 */ "NEWLINE",
/* 396 */ "POUND_POUND",
/* 397 */ "POUND",
/* 398 */ "ANY",
/* 399 */ "PP_INCLUDE",
/* 400 */ "PP_QHEADER",
/* 401 */ "PP_HHEADER",
/* 402 */ "EOF",
/* 403 */ "EOI",
/* 404 */ "PP_NUMBER",
/* 386 */ "FLOATLIT",
/* 387 */ "CCOMMENT",
/* 388 */ "CPPCOMMENT",
/* 389 */ "CHARLIT",
/* 390 */ "STRINGLIT",
/* 391 */ "CONTLINE",
/* 392 */ "SPACE",
/* 393 */ "SPACE2",
/* 394 */ "NEWLINE",
/* 395 */ "POUND_POUND",
/* 396 */ "POUND",
/* 397 */ "ANY",
/* 398 */ "PP_INCLUDE",
/* 399 */ "PP_QHEADER",
/* 400 */ "PP_HHEADER",
/* 401 */ "EOF",
/* 402 */ "EOI",
/* 403 */ "PP_NUMBER",
// MS extensions
/* 405 */ "MSEXT_INT8",
/* 406 */ "MSEXT_INT16",
/* 407 */ "MSEXT_INT32",
/* 408 */ "MSEXT_INT64",
/* 409 */ "MSEXT_BASED",
/* 410 */ "MSEXT_DECLSPEC",
/* 411 */ "MSEXT_CDECL",
/* 412 */ "MSEXT_FASTCALL",
/* 413 */ "MSEXT_STDCALL",
/* 414 */ "MSEXT_TRY",
/* 415 */ "MSEXT_EXCEPT",
/* 416 */ "MSEXT_FINALLY",
/* 417 */ "MSEXT_LEAVE",
/* 418 */ "MSEXT_INLINE",
/* 419 */ "MSEXT_ASM",
/* 420 */ "MSEXT_REGION",
/* 421 */ "MSEXT_ENDREGION",
/* 404 */ "MSEXT_INT8",
/* 405 */ "MSEXT_INT16",
/* 406 */ "MSEXT_INT32",
/* 407 */ "MSEXT_INT64",
/* 408 */ "MSEXT_BASED",
/* 409 */ "MSEXT_DECLSPEC",
/* 410 */ "MSEXT_CDECL",
/* 411 */ "MSEXT_FASTCALL",
/* 412 */ "MSEXT_STDCALL",
/* 413 */ "MSEXT_TRY",
/* 414 */ "MSEXT_EXCEPT",
/* 415 */ "MSEXT_FINALLY",
/* 416 */ "MSEXT_LEAVE",
/* 417 */ "MSEXT_INLINE",
/* 418 */ "MSEXT_ASM",
/* 419 */ "MSEXT_REGION",
/* 420 */ "MSEXT_ENDREGION",
/* 422 */ "IMPORT",
/* 421 */ "IMPORT",
/* 423 */ "ALIGNAS",
/* 424 */ "ALIGNOF",
/* 425 */ "CHAR16_T",
/* 426 */ "CHAR32_T",
/* 427 */ "CONSTEXPR",
/* 428 */ "DECLTYPE",
/* 429 */ "NOEXCEPT",
/* 430 */ "NULLPTR",
/* 431 */ "STATIC_ASSERT",
/* 432 */ "THREADLOCAL",
/* 433 */ "RAWSTRINGLIT",
/* 422 */ "ALIGNAS",
/* 423 */ "ALIGNOF",
/* 424 */ "CHAR16_T",
/* 425 */ "CHAR32_T",
/* 426 */ "CONSTEXPR",
/* 427 */ "DECLTYPE",
/* 428 */ "NOEXCEPT",
/* 429 */ "NULLPTR",
/* 430 */ "STATIC_ASSERT",
/* 431 */ "THREADLOCAL",
/* 432 */ "RAWSTRINGLIT",
/* 434 */ "T_CHAR8_T",
/* 435 */ "T_CONCEPT",
/* 436 */ "T_CONSTEVAL",
/* 437 */ "T_CONSTINIT",
/* 438 */ "T_CO_AWAIT",
/* 439 */ "T_CO_RETURN",
/* 440 */ "T_CO_YIELD",
/* 441 */ "T_REQUIRES",
/* 442 */ "T_MODULE",
/* 443 */ "T_SPACESHIP",
/* 433 */ "T_CHAR8_T",
/* 434 */ "T_CONCEPT",
/* 435 */ "T_CONSTEVAL",
/* 436 */ "T_CONSTINIT",
/* 437 */ "T_CO_AWAIT",
/* 438 */ "T_CO_RETURN",
/* 439 */ "T_CO_YIELD",
/* 440 */ "T_REQUIRES",
/* 441 */ "T_MODULE",
/* 442 */ "T_SPACESHIP",
/* 443 */ "T_SIZETLIT",
};
// make sure, I have not forgotten any commas (as I did more than once)
@@ -385,69 +385,69 @@ static char const *tok_values[] = {
/* 383 */ "", // hexlit
/* 384 */ "", // intlit
/* 385 */ "", // longintlit
/* 386 */ "", // sizetintlit
/* 387 */ "", // floatlit
/* 388 */ "", // ccomment
/* 389 */ "", // cppcomment
/* 390 */ "", // charlit
/* 391 */ "", // stringlit
/* 392 */ "", // contline
/* 393 */ "", // space
/* 394 */ "", // space2
/* 395 */ "\n",
/* 396 */ "##",
/* 397 */ "#",
/* 398 */ "", // any
/* 386 */ "", // floatlit
/* 387 */ "", // ccomment
/* 388 */ "", // cppcomment
/* 389 */ "", // charlit
/* 390 */ "", // stringlit
/* 391 */ "", // contline
/* 392 */ "", // space
/* 393 */ "", // space2
/* 394 */ "\n",
/* 395 */ "##",
/* 396 */ "#",
/* 397 */ "", // any
/* 398 */ "#include",
/* 399 */ "#include",
/* 400 */ "#include",
/* 401 */ "#include",
/* 402 */ "", // eof
/* 403 */ "", // eoi
/* 404 */ "", // pp-number
/* 401 */ "", // eof
/* 402 */ "", // eoi
/* 403 */ "", // pp-number
// MS extensions
/* 405 */ "__int8",
/* 406 */ "__int16",
/* 407 */ "__int32",
/* 408 */ "__int64",
/* 409 */ "__based",
/* 410 */ "__declspec",
/* 411 */ "__cdecl",
/* 412 */ "__fastcall",
/* 413 */ "__stdcall",
/* 414 */ "__try",
/* 415 */ "__except",
/* 416 */ "__finally",
/* 417 */ "__leave",
/* 418 */ "__inline",
/* 419 */ "__asm",
/* 420 */ "#region",
/* 421 */ "#endregion",
/* 404 */ "__int8",
/* 405 */ "__int16",
/* 406 */ "__int32",
/* 407 */ "__int64",
/* 408 */ "__based",
/* 409 */ "__declspec",
/* 410 */ "__cdecl",
/* 411 */ "__fastcall",
/* 412 */ "__stdcall",
/* 413 */ "__try",
/* 414 */ "__except",
/* 415 */ "__finally",
/* 416 */ "__leave",
/* 417 */ "__inline",
/* 418 */ "__asm",
/* 419 */ "#region",
/* 420 */ "#endregion",
/* 422 */ "import",
/* 421 */ "import",
/* 423 */ "alignas",
/* 424 */ "alignof",
/* 425 */ "char16_t",
/* 426 */ "char32_t",
/* 427 */ "constexpr",
/* 428 */ "decltype",
/* 429 */ "noexcept",
/* 430 */ "nullptr",
/* 431 */ "static_assert",
/* 432 */ "thread_local",
/* 433 */ "", // extrawstringlit
/* 422 */ "alignas",
/* 423 */ "alignof",
/* 424 */ "char16_t",
/* 425 */ "char32_t",
/* 426 */ "constexpr",
/* 427 */ "decltype",
/* 428 */ "noexcept",
/* 429 */ "nullptr",
/* 430 */ "static_assert",
/* 431 */ "thread_local",
/* 432 */ "", // extrawstringlit
/* 434 */ "char8_t",
/* 435 */ "concept",
/* 436 */ "consteval",
/* 437 */ "constinit",
/* 438 */ "co_await",
/* 439 */ "co_return",
/* 440 */ "co_yield",
/* 441 */ "requires",
/* 442 */ "module",
/* 443 */ "<=>",
/* 433 */ "char8_t",
/* 434 */ "concept",
/* 435 */ "consteval",
/* 436 */ "constinit",
/* 437 */ "co_await",
/* 438 */ "co_return",
/* 439 */ "co_yield",
/* 440 */ "requires",
/* 441 */ "module",
/* 442 */ "<=>",
/* 443 */ "", // sizetlit
};
// make sure, I have not forgotten any commas (as I did more than once)

3
test/testlexers/cpp_tokens.hpp
View File

@@ -267,6 +267,9 @@ lexem const lexems[] =
{ "1'2'34l", boost::wave::T_INTLIT },
{ "1234ul", boost::wave::T_INTLIT },
{ "1234LU", boost::wave::T_INTLIT },
{ "0x12'ab78", boost::wave::T_INTLIT },
{ "013'77", boost::wave::T_INTLIT },
{ "0b10'00101", boost::wave::T_INTLIT },
{ "1234z", boost::wave::T_SIZETLIT},
{ "1234zU", boost::wave::T_SIZETLIT},
{ "1234UZ", boost::wave::T_SIZETLIT},