boost/wave
2
0
Fork 0
mirror of https://github.com/boostorg/wave.git synced 2026-01-19 04:42:16 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #22 from jefftrull/feature/fixing_12603

Browse Source 
Repair C++11 raw string literal lexing
This commit is contained in:
Hartmut Kaiser
2017-11-28 14:35:49 -06:00
committed by GitHub
parent 2a6a1f906f c74492a208
commit 62e29280cc
6 changed files with 1191 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
include/boost/wave/cpplexer/re2clex/cpp.re
View File

@@ -258,7 +258,10 @@ NonDigit = [a-zA-Z_$] | UniversalChar;
"L"? "R" ["]
{
if (s->act_in_cpp0x_mode)
{
rawstringdelim = "";
goto extrawstringlit;
}
--YYCURSOR;
BOOST_WAVE_RET(T_IDENTIFIER);
}
@@ -282,7 +285,10 @@ NonDigit = [a-zA-Z_$] | UniversalChar;
([uU] | "u8") "R" ["]
{
if (s->act_in_cpp0x_mode)
{
rawstringdelim = "";
goto extrawstringlit;
}
--YYCURSOR;
BOOST_WAVE_RET(T_IDENTIFIER);
}
@@ -503,19 +509,68 @@ extstringlit:
extrawstringlit:
{
// we have consumed the double quote but not the lparen
// at this point we may see a delimiter
/*!re2c
(EscapeSequence | UniversalChar | any\[\r\n\\"])
* {
(*s->error_proc)(s, lexing_exception::generic_lexing_error,
"Invalid character in raw string delimiter ('%c')", yych);
}
// delimiters are any character but parentheses, backslash, and whitespace
any\[()\\\t\v\f\r\n]
{
rawstringdelim += yych;
if (rawstringdelim.size() > 16)
{
(*s->error_proc)(s, lexing_exception::generic_lexing_error,
"Raw string delimiter of excessive length (\"%s\") in input stream",
rawstringdelim.c_str());
}
goto extrawstringlit;
}
"("
{
rawstringdelim = ")" + rawstringdelim;
goto extrawstringbody;
}
*/
}
extrawstringbody:
{
/*!re2c
* {
(*s->error_proc)(s, lexing_exception::generic_lexing_error,
"Invalid character in raw string body ('%c')", yych);
}
Newline
{
s->line += count_backslash_newlines(s, cursor) +1;
cursor.column = 1;
goto extrawstringlit;
goto extrawstringbody;
}
["] { BOOST_WAVE_RET(T_RAWSTRINGLIT); }
(EscapeSequence | UniversalChar | any\["])
{
goto extrawstringbody;
}
["]
{
// check to see if we have completed a delimiter
if (string_type((char *)(YYCURSOR - rawstringdelim.size() - 1),
(char *)(YYCURSOR - 1)) == rawstringdelim)
{
BOOST_WAVE_RET(T_RAWSTRINGLIT);
} else {
goto extrawstringbody;
}
}
*/
}

3
include/boost/wave/cpplexer/re2clex/cpp_re.hpp
View File

@@ -378,6 +378,9 @@ boost::wave::token_id scan(Scanner<Iterator> *s)
uchar_wrapper marker (s->ptr);
uchar_wrapper limit (s->lim);
typedef BOOST_WAVE_STRINGTYPE string_type;
string_type rawstringdelim; // for use with C++11 raw string literals
// include the correct Re2C token definition rules
#if BOOST_WAVE_USE_STRICT_LEXER != 0
#include "strict_cpp_re.inc"

929
include/boost/wave/cpplexer/re2clex/cpp_re.inc
View File

File diff suppressed because it is too large Load Diff

63
include/boost/wave/cpplexer/re2clex/strict_cpp.re
View File

@@ -257,7 +257,10 @@ NonDigit = [a-zA-Z_] | UniversalChar;
"L"? "R" ["]
{
if (s->act_in_cpp0x_mode)
{
rawstringdelim = "";
goto extrawstringlit;
}
--YYCURSOR;
BOOST_WAVE_RET(T_IDENTIFIER);
}
@@ -281,7 +284,10 @@ NonDigit = [a-zA-Z_] | UniversalChar;
([uU] | "u8") "R" ["]
{
if (s->act_in_cpp0x_mode)
{
rawstringdelim = "";
goto extrawstringlit;
}
--YYCURSOR;
BOOST_WAVE_RET(T_IDENTIFIER);
}
@@ -368,7 +374,7 @@ ccomment:
{
BOOST_WAVE_UPDATE_CURSOR(); // adjust the input cursor
(*s->error_proc)(s, lexing_exception::generic_lexing_warning,
"unterminated 'C' style comment");
"Unterminated 'C' style comment");
}
else
{
@@ -502,19 +508,68 @@ extstringlit:
extrawstringlit:
{
// we have consumed the double quote but not the lparen
// at this point we may see a delimiter
/*!re2c
(EscapeSequence | UniversalChar | any\[\r\n\\"])
* {
(*s->error_proc)(s, lexing_exception::generic_lexing_error,
"Invalid character in raw string delimiter ('%c')", yych);
}
// delimiters are any character but parentheses, backslash, and whitespace
any\[()\\\t\v\f\r\n]
{
rawstringdelim += yych;
if (rawstringdelim.size() > 16)
{
(*s->error_proc)(s, lexing_exception::generic_lexing_error,
"Raw string delimiter of excessive length (\"%s\") in input stream",
rawstringdelim.c_str());
}
goto extrawstringlit;
}
"("
{
rawstringdelim = ")" + rawstringdelim;
goto extrawstringbody;
}
*/
}
extrawstringbody:
{
/*!re2c
* {
(*s->error_proc)(s, lexing_exception::generic_lexing_error,
"Invalid character in raw string body ('%c')", yych);
}
Newline
{
s->line += count_backslash_newlines(s, cursor) +1;
cursor.column = 1;
goto extrawstringlit;
goto extrawstringbody;
}
["] { BOOST_WAVE_RET(T_RAWSTRINGLIT); }
(EscapeSequence | UniversalChar | any\["])
{
goto extrawstringbody;
}
["]
{
// check to see if we have completed a delimiter
if (string_type((char *)(YYCURSOR - rawstringdelim.size() - 1),
(char *)(YYCURSOR - 1)) == rawstringdelim)
{
BOOST_WAVE_RET(T_RAWSTRINGLIT);
} else {
goto extrawstringbody;
}
}
*/
}

931
include/boost/wave/cpplexer/re2clex/strict_cpp_re.inc
View File

File diff suppressed because it is too large Load Diff

30
test/testwave/testfiles/t_7_001.cpp
View File

@@ -11,29 +11,39 @@
//O -Werror
//R #line 17 "t_7_001.cpp"
//R R"de
//R R"(de
//R fg
//R h"
R"de
//R h)"
R"(de
fg
h"
h)"
//R #line 22 "t_7_001.cpp"
"abc" //R "abc"
R"abc" //R R"abc"
R"(abc)" //R R"(abc)"
//R #line 28 "t_7_001.cpp"
//R uR"de fg
//R h"
uR"de \
//R uR"(de fg
//R h)"
uR"(de \
fg
h"
h)"
//R #line 33 "t_7_001.cpp"
u"abc" //R u"abc"
U"def" //R U"def"
u8"ghi" //R u8"ghi"
//R #line 39 "t_7_001.cpp"
//R #line 40 "t_7_001.cpp"
//R R"delim("quoted text
//R with newline")delim"
R"delim("quoted text
with newline")delim"
//R #line 45 "t_7_001.cpp"
//R R"de"lim(some text)de"lim"
R"de"lim(some text)de"lim"
//R #line 49 "t_7_001.cpp"
//R no_newline_at_end_of_file
no_newline_at_end_of_file