In the out-argument overload of rule_parser::call(), always parse into a local

attr_type attribute ('attr'), rather than the one supplied ('retval').  Then,
take the parsed attribute and either combine it with retval (if they are both
containers), or assign attr to retval (if attr_type is not none).

Reproducer by Andreas Buhr.

Fixes #294.

.github/workflows/macos-15.yml

@@ -1,4 +1,4 @@
-name: macos-12 - Clang 14
+name: macos-15 - Clang
 
 on:
   push:
@@ -14,9 +14,9 @@ jobs:
   build:
     strategy:
       matrix:
-        cxx_std: [17]
+        cxx_std: [17, 20]
 
-    runs-on: macos-12
+    runs-on: macos-15
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/ubuntu.yml

@@ -18,10 +18,6 @@ jobs:
         compiler_version: [g++-10, g++-11]
         cxx_std: [17, 20]
         os: [ubuntu-22.04]
-        include:
-          - compiler_version: g++-9
-            cxx_std: 17
-            os: ubuntu-20.04
 
     runs-on: ${{ matrix.os }}
 

.github/workflows/windows.yml

@@ -24,13 +24,13 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Configure CMake
-        # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
+        # Configure CMake in a 'build' subdirectory. Visual Studio is a multi-config generator, so we don't use CMAKE_BUILD_TYPE.
         # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-        run: cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCXX_STD=${{ matrix.cxx_std }}
+        run: cmake -B build -DCXX_STD=${{ matrix.cxx_std }}
 
       - name: Build
         working-directory: build
-        run: cmake --build . -- /p:CL_MPcount=4
+        run: cmake --build . --config ${{ env.BUILD_TYPE }} -- /p:CL_MPcount=4
 
       - name: Test
         working-directory: build/test

README.md

@@ -57,7 +57,7 @@ Master status:
 
 [![Windows MSVC](https://github.com/tzlaine/parser/actions/workflows/windows.yml/badge.svg?branch=master)](https://github.com/tzlaine/parser/actions/workflows/windows.yml)
 
-[![macos-12 - Clang 14](https://github.com/tzlaine/parser/actions/workflows/macos-12.yml/badge.svg?branch=master)](https://github.com/tzlaine/parser/actions/workflows/macos-12.yml)
+[![macos-13 - Clang 14](https://github.com/tzlaine/parser/actions/workflows/macos-13.yml/badge.svg?branch=master)](https://github.com/tzlaine/parser/actions/workflows/macos-13.yml)
 
 Develop status:
 
@@ -67,6 +67,6 @@ Develop status:
 
 [![Windows MSVC](https://github.com/tzlaine/parser/actions/workflows/windows.yml/badge.svg?branch=develop)](https://github.com/tzlaine/parser/actions/workflows/windows.yml)
 
-[![macos-12 - Clang 14](https://github.com/tzlaine/parser/actions/workflows/macos-12.yml/badge.svg?branch=develop)](https://github.com/tzlaine/parser/actions/workflows/macos-12.yml)
+[![macos-13 - Clang 14](https://github.com/tzlaine/parser/actions/workflows/macos-13.yml/badge.svg?branch=develop)](https://github.com/tzlaine/parser/actions/workflows/macos-13.yml)
 
 [![License](https://img.shields.io/badge/license-boost-brightgreen.svg)](LICENSE_1_0.txt)

build.jam

@@ -0,0 +1,24 @@
+# Copyright René Ferdinand Rivera Morell 2025
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+
+require-b2 5.2 ;
+
+constant boost_dependencies :
+    /boost/assert//boost_assert
+    /boost/charconv//boost_charconv
+    /boost/hana//boost_hana
+    /boost/type_index//boost_type_index ;
+
+project /boost/parser
+    ;
+
+explicit
+    [ alias boost_parser : : :
+        : <library>$(boost_dependencies) <include>include ]
+    [ alias all : boost_parser test ]
+    ;
+
+call-if : boost-library parser
+    ;

doc/Jamfile.v2

@@ -39,7 +39,7 @@ rule run_doxygen ( files * : name : expand ? )
 
 }
 
-run_doxygen [ glob $(here)/../../../boost/parser/*.hpp : $(here)/../../../boost/parser/concepts.hpp ] : "Headers" ;
+run_doxygen [ glob $(here)/../include/boost/parser/*.hpp : $(here)/../include/boost/parser/concepts.hpp ] : "Headers" ;
 
 install images_standalone : [ glob *.png ] : <location>html/parser/img ;
 explicit images_standalone ;

doc/parser.qbk

@@ -42,7 +42,6 @@
 [import ../test/parser.cpp]
 [import ../test/parser_rule.cpp]
 [import ../test/parser_quoted_string.cpp]
-[import ../test/lexer_and_parser.cpp]
 
 [import ../include/boost/parser/concepts.hpp]
 [import ../include/boost/parser/error_handling_fwd.hpp]
@@ -110,17 +109,8 @@
 [def _trans_replace_vs_    [classref boost::parser::transform_replace_view `boost::parser::transform_replace_view`s]]
 
 
-[def _lex_                 [classref boost::parser::lexer_t `boost::parser::lexer_t`]]
-[def _tok_                 [classref boost::parser::token `boost::parser::token`]]
-[def _toks_                [classref boost::parser::token `boost::parser::token`s]]
-[def _tok_spec_            [classref boost::parser::token_spec_t `boost::parser::token_spec_t`]]
-[def _tok_specs_           [classref boost::parser::token_spec_t `boost::parser::token_spec_t`s]]
-[def _tok_chs_             [globalref boost::parser::token_chars `boost::parser::token_chars`]]
-[def _to_tok_              [globalref boost::parser::to_tokens `boost::parser::to_tokens`]]
-[def _tok_v_               [classref boost::parser::tokens_view `boost::parser::tokens_view`]]
-[def _ch_id_               [globalref boost::parser::character_id `boost::parser::character_id`]]
-
 [def _std_str_             `std::string`]
+[def _std_strs_            `std::string`s]
 [def _std_vec_char_        `std::vector<char>`]
 [def _std_vec_char32_      `std::vector<char32_t>`]
 
@@ -212,6 +202,7 @@
 [def _merge_               [globalref boost::parser::merge `merge[]`]]
 [def _sep_                 [globalref boost::parser::separate `separate[]`]]
 [def _transform_           [globalref boost::parser::transform `transform(f)[]`]]
+[def _delimiter_           [globalref boost::parser::delimiter `delimiter(p)[]`]]
 
 [def _omit_np_             [globalref boost::parser::omit `omit`]]
 [def _raw_np_              [globalref boost::parser::raw `raw`]]
@@ -222,11 +213,13 @@
 [def _merge_np_            [globalref boost::parser::merge `merge`]]
 [def _sep_np_              [globalref boost::parser::separate `separate`]]
 [def _transform_np_        [globalref boost::parser::transform `transform`]]
+[def _delimiter_np_        [globalref boost::parser::delimiter `delimiter`]]
 
 [def _blank_               [globalref boost::parser::blank `blank`]]
 [def _control_             [globalref boost::parser::control `control`]]
 [def _digit_               [globalref boost::parser::digit `digit`]]
 [def _punct_               [globalref boost::parser::punct `punct`]]
+[def _symb_                [globalref boost::parser::symb `symb`]]
 [def _hex_digit_           [globalref boost::parser::hex_digit `hex_digit`]]
 [def _lower_               [globalref boost::parser::lower `lower`]]
 [def _upper_               [globalref boost::parser::upper `upper`]]
@@ -250,6 +243,7 @@
 [def _more_about_rules_    [link boost_parser.tutorial.more_about_rules More About Rules]]
 [def _unicode_             [link boost_parser.tutorial.unicode_support Unicode Support]]
 [def _concepts_            [link boost_parser.concepts Concepts]]
+[def _seq_parser_example_  [link boost_parser.tutorial.attribute_generation.a_sequence_parser_attribute_example A sequence parser attribute example]]
 [def _ex_json_             [link boost_parser.extended_examples.parsing_json Parsing JSON]]
 [def _ex_cb_json_          [link boost_parser.extended_examples.parsing_json_with_callbacks Parsing JSON With Callbacks]]
 [def _rationale_           [link boost_parser.rationale Rationale]]
@@ -264,12 +258,6 @@
 [def _udls_                [@https://en.cppreference.com/w/cpp/language/user_literal UDLs]]
 [def _yaml_                [@https://yaml.org/spec/1.2/spec.html YAML 1.2]]
 
-[def _nttp_                [@https://en.cppreference.com/w/cpp/language/template_parameters NTTP]]
-[def _nttps_               [@https://en.cppreference.com/w/cpp/language/template_parameters NTTPs]]
-
-[def _ctre_                [@https://github.com/hanickadot/compile-time-regular-expressions CTRE]]
-[def _pcre_                [@https://www.pcre.org PCRE]]
-
 [def _Spirit_              [@https://www.boost.org/doc/libs/release/libs/spirit Boost.Spirit]]
 [def _spirit_reals_        [@https://www.boost.org/doc/libs/release/libs/spirit/doc/html/spirit/qi/reference/numeric/real.html real number parsers]]
 

doc/rationale.qbk

@@ -202,7 +202,7 @@ It has a different API, and other code that operates on text expects a string
 instead of some other container.  Arrays of characters are already considered
 special by the standard library and common practice in C++.
 
-Second, When you write a parser that parses multiple characters in a row, you
+Second, when you write a parser that parses multiple characters in a row, you
 are typically trying to produce a string attribute, rather than a few
 individual character values.  When you use multiple non-character parsers in a
 row, you are typically trying to produce multiple values.  For instance:
@@ -217,7 +217,7 @@ I've rarely written a parser like `parser_2` and wanted a
 `std::vector<std::string>`.
 
 _Parser_ therefore makes the common case the default behavior, and provides
-you with the _merge_ and _sep_ directives to let you opt-in to generating the
+you with the _merge_ and _sep_ directives to let you opt in to generating the
 less-common attributes.
 
 [heading Attribute compatibility rules are more strict than in Spirit]
@@ -261,13 +261,12 @@ Also, Spirit-style looseness is more complicated than `parser` above
 indicates.  Remember, `int_ | eps` and `-int_` are supposed to be semantically
 equivalent.  To do otherwise this would be a profound violation of the
 principle of least surprise.  So, if they're equivalent, we would need to
-apply the same rule to `int_ | eps`.  Also, we would probably need to apply it
-to `if_(cond)[int_]`, which is also a `std::optional<int>`.  This is a lot to
-remember, and this is complicated to implement and maintain.
+apply the same rule to `int_ | eps`.  This is a lot to remember, and this is
+complicated to implement and maintain.
 
 I've been using Spirit 1 and later Spirit 2 since they were released.  I did
 not know about the particular looseness discussed here; a user pointed it out
-on Github.  In many years of using these libraries, I never fully learned all
+on GitHub.  In many years of using these libraries, I never fully learned all
 the attribute-compatibility rules, and was often surprised by them.
 
 Having a small set of rules that the user can internalize is vital; if the
@@ -299,7 +298,7 @@ that it succeeds (if it had failed, it would have cleared its attribute).  It
 does not know that there is nothing after it that could continue the parse,
 nor that it is being used in to do a full parse.  So, the over-all parse
 fails, but the part of the parse that fills in the out-param attribute does
-not know do clear its attribute.
+not know to clear its attribute.
 
 This is why the explicit clearing behavior happens at the end of _p_.  This is
 not without its downsides, though.  Consider this.
@@ -315,7 +314,7 @@ not without its downsides, though.  Consider this.
 Here, the explicit clearing replaces the previous value of `3`, even though
 the parser never touched the value!  Destroying users' variables' state
 without need may seem like a bad idea, but consider the alternative _emdash_
-In the previous example, we had spurious values left in the out-param
+in the previous example, we had spurious values left in the out-param
 attribute.  Here, without clearing, we would have had a value left in the
 out-param attribute, not because it was a partial result of the parse, but
 because the parse never touched it.  This is certain to be confusing, or at
@@ -325,4 +324,24 @@ always equal to `A()` if the parser fails.  It is equal to whatever the parser
 sets it to _emdash_ or its previous value, if the parser does not mutate it
 _emdash_ if the parse succeeds.
 
+[heading There are no _Spirit_-style character class parsers]
+
+_Spirit_ has these character class parsers that recognize the same set of
+characters as the C standard library's character class functions.  For
+instance, _Spirit_'s `alnum` recognizes the characters recognized by
+`std::isalnum()`, its `punct` recognizes the characters recognized by
+`std::ispunct()`, etc.
+
+The problem with this is that those `std::is*()` functions are badly broken.
+They do not even work correctly for ASCII values.  This is because they use
+the C standard library's locale mechanism, which can be set to anything the
+current platform supports, and can be set by any code anywhere in your
+program; the locale is mutable global state.  So, even if you use the default
+C locale in your program, if you link against a library that sets the locale
+to something that breaks ASCII character recognition (an EBCDIC locale, for
+instance), your program is now incorrect, regardless of the code you wrote.
+
+For this reason, I firmly believe that no one, anywhere, should use those C
+functions in production code, and I am not supporting their use via _Parser_.
+
 [endsect]

doc/tables.qbk

@@ -80,7 +80,7 @@ the input they match unless otherwise stated in the table below.]
     [[ _attr_np_`(arg0)` ]
      [ Always matches, and consumes no input.  Generates the attribute `_RES_np_(arg0)`. ]
      [ `decltype(_RES_np_(arg0))`. ]
-     [ An important use case for `_attr_` is to provide a default attribute value as a trailing alternative.  For instance, an *optional* comma-delmited list is: `int_ % ',' | attr(std::vector<int>)`.  Without the "`| attr(...)`", at least one `int_` match would be required. ]]
+     [ An important use case for `_attr_` is to provide a default attribute value as a trailing alternative.  For instance, an *optional* comma-delimited list is: `int_ % ',' | attr(std::vector<int>)`.  Without the "`| attr(...)`", at least one `int_` match would be required. ]]
 
     [[ _ch_ ]
      [ Matches any single code point. ]
@@ -132,8 +132,13 @@ the input they match unless otherwise stated in the table below.]
      [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
      []]
 
+    [[ `_symb_` ]
+     [ Matches a single symbol code point. ]
+     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
+     []]
+
     [[ `_hex_digit_` ]
-     [ Matches a single hexidecimal digit code point. ]
+     [ Matches a single hexadecimal digit code point. ]
      [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
      []]
 
@@ -225,7 +230,7 @@ the input they match unless otherwise stated in the table below.]
     [[ _ui_ ]
      [ Matches an unsigned integral value. ]
      [ `unsigned int` ]
-     []]
+     [ To specify a base/radix of `N`, use _ui_`.base<N>()`.  To specify exactly `D` digits, use _ui_`.digits<D>()`.  To specify a minimum of `LO` digits and a maximum of `HI` digits, use _ui_`.digits<LO, HI>()`.  These calls can be chained, as in _ui_`.base<2>().digits<8>()`. ]]
 
     [[ `_ui_(arg0)` ]
      [ Matches exactly the unsigned integral value `_RES_np_(arg0)`. ]
@@ -265,7 +270,7 @@ the input they match unless otherwise stated in the table below.]
     [[ _i_ ]
      [ Matches a signed integral value. ]
      [ `int` ]
-     []]
+     [ To specify a base/radix of `N`, use _i_`.base<N>()`.  To specify exactly `D` digits, use _i_`.digits<D>()`.  To specify a minimum of `LO` digits and a maximum of `HI` digits, use _i_`.digits<LO, HI>()`.  These calls can be chained, as in _i_`.base<2>().digits<8>()`. ]]
 
     [[ `_i_(arg0)` ]
      [ Matches exactly the signed integral value `_RES_np_(arg0)`. ]
@@ -314,7 +319,7 @@ the input they match unless otherwise stated in the table below.]
 
     [[ `_if_np_(pred)[p]` ]
      [ Equivalent to `_e_(pred) >> p`. ]
-     [ `std::optional<_ATTR_np_(p)>` ]
+     [ `_ATTR_np_(p)` ]
      [ It is an error to write `_if_np_(pred)`.  That is, it is an error to omit the conditionally matched parser `p`. ]]
 
     [[ `_sw_np_(arg0)(arg1, p1)(arg2, p2) ...` ]
@@ -323,9 +328,9 @@ the input they match unless otherwise stated in the table below.]
      [ It is an error to write `_sw_np_(arg0)`.  That is, it is an error to omit the conditionally matched parsers `p1`, `p2`, .... ]]
 
     [[ _symbols_t_ ]
-     [ _symbols_ is an associative container of key, value pairs.  Each key is a _std_str_ and each value has type `T`.  In the Unicode parsing path, the strings are considered to be UTF-8 encoded; in the non-Unicode path, no encoding is assumed.  _symbols_ Matches the longest prefix `pre` of the input that is equal to one of the keys `k`.  If the length `len` of `pre` is zero, and there is no zero-length key, it does not match the input.  If `len` is positive, the generated attribute is the value associated with `k`.]
+     [ _symbols_ is an associative container of key, value pairs.  Each key is a _std_str_ and each value has type `T`.  In the Unicode parsing path, the strings are considered to be UTF-8 encoded; in the non-Unicode path, no encoding is assumed.  _symbols_ matches the longest prefix `pre` of the input that is equal to one of the keys `k`.  If the length `len` of `pre` is zero, and there is no zero-length key, it does not match the input.  If `len` is positive, the generated attribute is the value associated with `k`.]
      [ `T` ]
-     [ Unlike the other entries in this table, _symbols_ is a type, not an object. ]]
+     [ Unlike the other entries in this table, _symbols_ is a type, not an object.  Inside of skippers, all _symbols_ will appear empty. ]]
 
     [[ _quot_str_ ]
      [ Matches `'"'`, followed by zero or more characters, followed by `'"'`. ]
@@ -363,7 +368,7 @@ character type (or use _attr_ to do so).]
 ]
 
 [template table_combining_operations
-Here are all the operator overloaded for parsers.  In the tables below:
+Here are all the operators overloaded for parsers.  In the tables below:
 
 * `c` is a character of type `char` or `char32_t`;
 
@@ -385,16 +390,16 @@ consume the input they match unless otherwise stated in the table below.]
     [[`*p`] [ Parses using `p` repeatedly until `p` no longer matches; always matches. ] [`std::string` if `_ATTR_np_(p)` is `char` or `char32_t`, otherwise `std::vector<_ATTR_np_(p)>`] [ Matching _e_ an unlimited number of times creates an infinite loop, which is undefined behavior in C++.  _Parser_ will assert in debug mode when it encounters `*_e_` (this applies to unconditional _e_ only). ]]
     [[`+p`] [ Parses using `p` repeatedly until `p` no longer matches; matches iff `p` matches at least once. ] [`std::string` if `_ATTR_np_(p)` is `char` or `char32_t`, otherwise `std::vector<_ATTR_np_(p)>`] [ Matching _e_ an unlimited number of times creates an infinite loop, which is undefined behavior in C++.  _Parser_ will assert in debug mode when it encounters `+_e_` (this applies to unconditional _e_ only). ]]
     [[`-p`] [ Equivalent to `p | _e_`. ] [`std::optional<_ATTR_np_(p)>`] []]
-    [[`p1 >> p2`] [ Matches iff `p1` matches and then `p2` matches. ] [`_bp_tup_<_ATTR_np_(p1), _ATTR_np_(p2)>` (See note.)] [ `>>` is associative; `p1 >> p2 >> p3`, `(p1 >> p2) >> p3`, and `p1 >> (p2 >> p3)` are all equivalent.  This attribute type only applies to the case where `p1` and `p2` both generate attributes; see _attr_gen_ for the full rules. ]]
+    [[`p1 >> p2`] [ Matches iff `p1` matches and then `p2` matches. ] [`_bp_tup_<_ATTR_np_(p1), _ATTR_np_(p2)>` (See note.)] [ `>>` is associative; `p1 >> p2 >> p3`, `(p1 >> p2) >> p3`, and `p1 >> (p2 >> p3)` are all equivalent.  This attribute type only applies to the case where `p1` and `p2` both generate attributes; see _attr_gen_ for the full rules.  Differs in precedence from `operator>`. ]]
     [[`p >> c`] [ Equivalent to `p >> lit(c)`. ] [`_ATTR_np_(p)`] []]
     [[`p >> r`] [ Equivalent to `p >> lit(r)`. ] [`_ATTR_np_(p)`] []]
-    [[`p1 > p2`] [ Matches iff `p1` matches and then `p2` matches.  No back-tracking is allowed after `p1` matches; if `p1` matches but then `p2` does not, the top-level parse fails. ] [`_bp_tup_<_ATTR_np_(p1), _ATTR_np_(p2)>` (See note.)] [ `>` is associative; `p1 > p2 > p3`, `(p1 > p2) > p3`, and `p1 > (p2 > p3)` are all equivalent.  This attribute type only applies to the case where `p1` and `p2` both generate attributes; see _attr_gen_ for the full rules. ]]
+    [[`p1 > p2`] [ Matches iff `p1` matches and then `p2` matches.  No back-tracking is allowed after `p1` matches; if `p1` matches but then `p2` does not, the top-level parse fails. ] [`_bp_tup_<_ATTR_np_(p1), _ATTR_np_(p2)>` (See note.)] [ `>` is associative; `p1 > p2 > p3`, `(p1 > p2) > p3`, and `p1 > (p2 > p3)` are all equivalent.  This attribute type only applies to the case where `p1` and `p2` both generate attributes; see _attr_gen_ for the full rules.  Differs in precedence from `operator>>`. ]]
     [[`p > c`] [ Equivalent to `p > lit(c)`. ] [`_ATTR_np_(p)`] []]
     [[`p > r`] [ Equivalent to `p > lit(r)`. ] [`_ATTR_np_(p)`] []]
     [[`p1 | p2`] [ Matches iff either `p1` matches or `p2` matches. ] [`std::variant<_ATTR_np_(p1), _ATTR_np_(p2)>` (See note.)] [ `|` is associative; `p1 | p2 | p3`, `(p1 | p2) | p3`, and `p1 | (p2 | p3)` are all equivalent.  This attribute type only applies to the case where `p1` and `p2` both generate attributes, and where the attribute types are different; see _attr_gen_ for the full rules. ]]
     [[`p | c`] [ Equivalent to `p | lit(c)`. ] [`_ATTR_np_(p)`] []]
     [[`p | r`] [ Equivalent to `p | lit(r)`. ] [`_ATTR_np_(p)`] []]
-    [[`p1 || p2`] [ Matches iff `p1` matches and `p2` matches, regardless of the order they match in. ] [`_bp_tup_<_ATTR_np_(p1), _ATTR_np_(p2)>`] [ `||` is associative; `p1 || p2 || p3`, `(p1 || p2) || p3`, and `p1 || (p2 || p3)` are all equivalent.  It is an error to include a _e_ (conditional or non-conditional) in an `operator||` expression.  Though the parsers are matched in any order, the attribute elements are always in the order written in the `operator||` expression. ]]
+    [[`p1 || p2`] [ Matches iff `p1` matches and `p2` matches, regardless of the order they match in. ] [`_bp_tup_<_ATTR_np_(p1), _ATTR_np_(p2)>`] [ `||` is associative; `p1 || p2 || p3`, `(p1 || p2) || p3`, and `p1 || (p2 || p3)` are all equivalent.  It is an error to include an _e_ (conditional or non-conditional) in an `operator||` expression.  Though the parsers are matched in any order, the attribute elements are always in the order written in the `operator||` expression. ]]
     [[`p1 - p2`] [ Equivalent to `!p2 >> p1`. ] [`_ATTR_np_(p1)`] []]
     [[`p - c`] [ Equivalent to `p - lit(c)`. ] [`_ATTR_np_(p)`] []]
     [[`p - r`] [ Equivalent to `p - lit(r)`. ] [`_ATTR_np_(p)`] []]
@@ -424,6 +429,15 @@ because, for any parser `p`, `_e_ | p` is equivalent to _e_, since _e_ always
 matches.  This is not true for _e_ parameterized with a condition.  For any
 condition `cond`, `_e_(cond)` is allowed to appear anywhere within an
 alternative parser.
+
+[important The C++ operators `>` and `>>` have different precedences.  This
+will sometimes come up in warnings from your compiler.  No matter how you do
+or do not parenthesize chains of parsers separated by `>` and `>>`, the
+resulting expression evaluates the same.  Feel free to add parentheses if your
+compiler complains.  More broadly, keep the C++ operator precedence rules in
+mind when writing your parsers _emdash_ the simplest thing to write may not
+have your intended semantics. ]
+
 ]
 
 [template table_attribute_generation
@@ -491,7 +505,7 @@ attribute type is `char32_t`:
     static_assert(std::is_same_v<decltype(result), std::optional<char32_t>>));
 
 The good news is that usually you don't parse characters individually.  When
-you parse with _ch_, you usually parse repetition of then, which will produce
+you parse with _ch_, you usually parse repetition of them, which will produce
 a _std_str_, regardless of whether you're in Unicode parsing mode or not.  If
 you do need to parse individual characters, and want to lock down their
 attribute type, you can use _cp_ and/or _cu_ to enforce a non-polymorphic
@@ -537,13 +551,13 @@ tables below:
     [[`p1 || p2`]                       [`_bp_tup_<_ATTR_np_(p1), _ATTR_np_(p2)>`]]
     [[`p1 || p2 || p3`]                 [`_bp_tup_<_ATTR_np_(p1), _ATTR_np_(p2), _ATTR_np_(p3)>`]]
 
-    [[`p1 % p2`]                        [`std::string` if `_ATTR_np_(p)` is `char` or `char32_t`, otherwise `std::vector<_ATTR_np_(p1)>`]]
+    [[`p1 % p2`]                        [`std::string` if `_ATTR_np_(p1)` is `char` or `char32_t`, otherwise `std::vector<_ATTR_np_(p1)>`]]
 
     [[`p[a]`]                           [None.]]
 
     [[`_rpt_np_(arg0)[p]`]              [`std::string` if `_ATTR_np_(p)` is `char` or `char32_t`, otherwise `std::vector<_ATTR_np_(p)>`]]
     [[`_rpt_np_(arg0, arg1)[p]`]        [`std::string` if `_ATTR_np_(p)` is `char` or `char32_t`, otherwise `std::vector<_ATTR_np_(p)>`]]
-    [[`_if_np_(pred)[p]`]               [`std::optional<_ATTR_np_(p)>`]]
+    [[`_if_np_(pred)[p]`]               [`_ATTR_np_(p)`]]
     [[`_sw_np_(arg0)(arg1, p1)(arg2, p2)...`]
      [`std::variant<_ATTR_np_(p1), _ATTR_np_(p2), ...>`]]
 ]
@@ -595,220 +609,3 @@ same attribute generation rules.
     [[`p1 | p2[a] | p3`]             [`std::optional<std::variant<_ATTR_np_(p1), _ATTR_np_(p3)>>`]]
 ]
 ]
-
-[template table_token_parsers_and_their_semantics
-This table lists all the _Parser_ parsers usable during token parsing.  For
-the callable parsers, a separate entry exists for each possible arity of
-arguments.  For a parser `p`, if there is no entry for `p` without arguments,
-`p` is a function, and cannot itself be used as a parser; it must be called.
-In the table below:
-
-* each entry is a global object usable directly in your parsers, unless
-  otherwise noted;
-
-* "code point" is used to refer to the elements of the input range, which
-  assumes that the parse is being done in the Unicode-aware code path (if the
-  parse is being done in the non-Unicode code path, read "code point" as
-  "`char`");
-
-* _RES_ is a notional macro that expands to the resolution of parse argument
-  or evaluation of a parse predicate (see _parsers_uses_);
-
-* "`_RES_np_(pred) == true`" is a shorthand notation for "`_RES_np_(pred)` is
-  contextually convertible to `bool` and `true`"; likewise for `false`;
-
-* `c` is a character of some character type;
-
-* `str` is a string literal of type `CharType const[]`, for some character
-  type `Char\Type`;
-
-* `pred` is a parse predicate;
-
-* `arg0`, `arg1`, `arg2`, ... are parse arguments;
-
-* `a` is a semantic action;
-
-* `r` is an object whose type models `parsable_range`;
-
-* `tok` is a token parser created using _tok_spec_; and
-
-* `p`, `p1`, `p2`, ... are parsers.
-
-[note The definition of `parsable_range` is:
-
-[parsable_range_concept]
-
-]
-
-[note Some of the parsers in this table consume no input.  All parsers consume
-the input they match unless otherwise stated in the table below.]
-
-[table Token Parsers and Their Semantics
-    [[Parser] [Semantics] [Attribute Type] [Notes]]
-
-    [[ `tok` ]
-      [ Matches any token with the same ID as `tok`. ]
-      [ The attribute type given when specifying `tok`, or a string view if unspecified.  The attribute type must be a specialization of `std::basic_string_view`, an integral type, or a floating point type. ]
-      []]
-
-    [[ `tok(arg0)` ]
-     [ Matches exactly the value `_RES_np_(arg0)`. ]
-     [ The attribute type given when specifying `tok`.  The attribute type must be a an integral type or a floating point type. ]
-     [ This case applies only when `arg0` is *not* a range. ]]
-
-    [[ `tok(r)` ]
-     [ Matches exactly the value `r`. ]
-     [ The attribute type given when specifying `tok`.  The attribute type must be a specialization of `std::basic_string_view`. ]
-     [ This overload does *not* take parse arguments. ]]
-
-    [[ _e_ ]
-      [ Matches /epsilon/, the empty string.  Always matches, and consumes no input. ]
-      [ None. ]
-      [ Matching _e_ an unlimited number of times creates an infinite loop, which is undefined behavior in C++.  _Parser_ will assert in debug mode when it encounters `*_e_`, `+_e_`, etc (this applies to unconditional _e_ only). ]]
-
-    [[ `_e_(pred)` ]
-     [ Fails to match the input if `_RES_np_(pred) == false`.  Otherwise, the semantics are those of _e_. ]
-     [ None. ]
-     []]
-
-    [[ _ws_ ]
-     [ Matches a single whitespace code point (see note), according to the Unicode White_Space property. ]
-     [ None. ]
-     [ For more info, see the [@https://www.unicode.org/Public/UCD/latest/ucd/PropList.txt Unicode properties].  _ws_ may consume one code point or two.  It only consumes two code points when it matches `"\r\n"`. ]]
-
-    [[ _eol_ ]
-     [ Matches a single newline (see note), following the "hard" line breaks in the Unicode line breaking algorithm. ]
-     [ None. ]
-     [ For more info, see the [@https://unicode.org/reports/tr14 Unicode Line Breaking Algorithm].  _eol_ may consume one code point or two.  It only consumes two code points when it matches `"\r\n"`. ]]
-
-    [[ _eoi_ ]
-     [ Matches only at the end of input, and consumes no input. ]
-     [ None. ]
-     []]
-
-    [[ _attr_np_`(arg0)` ]
-     [ Always matches, and consumes no input.  Generates the attribute `_RES_np_(arg0)`. ]
-     [ `decltype(_RES_np_(arg0))`. ]
-     [ An important use case for `_attr_` is to provide a default attribute value as a trailing alternative.  For instance, an *optional* comma-delmited list is: `int_ % ',' | attr(std::vector<int>)`.  Without the "`| attr(...)`", at least one `int_` match would be required. ]]
-
-    [[ _ch_ ]
-     [ Matches any single code point. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See _attr_gen_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_ch_(arg0)` ]
-     [ Matches exactly the code point `_RES_np_(arg0)`. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See _attr_gen_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_ch_(arg0, arg1)` ]
-     [ Matches the next code point `n` in the input, if `_RES_np_(arg0) <= n && n <= _RES_np_(arg1)`. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See _attr_gen_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_ch_(r)` ]
-     [ Matches the next code point `n` in the input, if `n` is one of the code points in `r`. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See _attr_gen_. ]
-     [ `r` is taken to be in a UTF encoding.  The exact UTF used depends on `r`'s element type.  If you do not pass UTF encoded ranges for `r`, the behavior of _ch_ is undefined.  Note that ASCII is a subset of UTF-8, so ASCII is fine.  EBCDIC is not.  `r` is not copied; a reference to it is taken.  The lifetime of `_ch_(r)` must be within the lifetime of `r`.  This overload of _ch_ does *not* take parse arguments.  Only matches tokens with the ID _ch_id_. ]]
-
-    [[ _cp_ ]
-     [ Matches a single code point. ]
-     [ `char32_t` ]
-     [ Similar to _ch_, but with a fixed `char32_t` attribute type; _cp_ has all the same call operator overloads as _ch_, though they are not repeated here, for brevity.  Only matches tokens with the ID _ch_id_. ]]
-
-    [[ _cu_ ]
-     [ Matches a single code point. ]
-     [ `char` ]
-     [ Similar to _ch_, but with a fixed `char` attribute type; _cu_ has all the same call operator overloads as _ch_, though they are not repeated here, for brevity.  Even though the name "`cu`" suggests that this parser match at the code unit level, it does not.  The name refers to the attribute type generated, much like the names _i_ versus _ui_.  Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_blank_` ]
-     [ Equivalent to `_ws_ - _eol_`. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_control_` ]
-     [ Matches a single control-character code point. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_digit_` ]
-     [ Matches a single decimal digit code point. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_punct_` ]
-     [ Matches a single punctuation code point. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_hex_digit_` ]
-     [ Matches a single hexidecimal digit code point. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_lower_` ]
-     [ Matches a single lower-case code point. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `_upper_` ]
-     [ Matches a single upper-case code point. ]
-     [ The code point type in Unicode parsing, or `char` in non-Unicode parsing.  See the entry for _ch_. ]
-     [ Only matches tokens with the ID _ch_id_. ]]
-
-    [[ _lit_np_`(c)`]
-     [ Matches exactly the given code point `c`. ]
-     [ None. ]
-     [_lit_ does *not* take parse arguments.  Only matches tokens with the ID _ch_id_. ]]
-
-    [[ `c_l` ]
-     [ Matches exactly the given code point `c`. ]
-     [ None. ]
-     [ This is a _udl_ that represents `_lit_np_(c)`, for example `'F'_l`.  Only matches tokens with the ID _ch_id_. ]]
-
-    [[ _lit_np_`(r)`]
-     [ Matches exactly the given string `r`. ]
-     [ None. ]
-     [ _lit_ does *not* take parse arguments.  _str_ matches the entire token or not at all.  Only matches tokens with an attribute type that is a specialization of `std::basic_string_view`. ]]
-
-    [[ `str_l` ]
-     [ Matches exactly the given string `str`. ]
-     [ None. ]
-     [ This is a _udl_ that represents `_lit_np_(s)`, for example `"a string"_l`.  Only matches tokens with an attribute type that is a specialization of `std::basic_string_view`. ]]
-
-    [[ `_str_np_(r)`]
-     [ Matches exactly `r`, and generates the match as an attribute. ]
-     [ _std_str_ ]
-     [ _str_ does *not* take parse arguments.  _str_ matches the entire token or not at all.  Only matches tokens with an attribute type that is a specialization of `std::basic_string_view`. ]]
-
-    [[ `str_p`]
-     [ Matches exactly `str`, and generates the match as an attribute. ]
-     [ _std_str_ ]
-     [ This is a _udl_ that represents `_str_np_(s)`, for example `"a string"_p`.  Only matches tokens with an attribute type that is a specialization of `std::basic_string_view`. ]]
-
-    [[ `_rpt_np_(arg0)[p]` ]
-     [ Matches iff `p` matches exactly `_RES_np_(arg0)` times. ]
-     [ `std::string` if `_ATTR_np_(p)` is `char` or `char32_t`, otherwise `std::vector<_ATTR_np_(p)>` ]
-     [ The special value _inf_ may be used; it indicates unlimited repetition.  `decltype(_RES_np_(arg0))` must be implicitly convertible to `int64_t`.  Matching _e_ an unlimited number of times creates an infinite loop, which is undefined behavior in C++.  _Parser_ will assert in debug mode when it encounters `_rpt_np_(_inf_)[_e_]` (this applies to unconditional _e_ only). ]]
-
-    [[ `_rpt_np_(arg0, arg1)[p]` ]
-     [ Matches iff `p` matches between `_RES_np_(arg0)` and `_RES_np_(arg1)` times, inclusively. ]
-     [ `std::string` if `_ATTR_np_(p)` is `char` or `char32_t`, otherwise `std::vector<_ATTR_np_(p)>` ]
-     [ The special value _inf_ may be used for the upper bound; it indicates unlimited repetition.  `decltype(_RES_np_(arg0))` and `decltype(_RES_np_(arg1))` each must be implicitly convertible to `int64_t`.  Matching _e_ an unlimited number of times creates an infinite loop, which is undefined behavior in C++.  _Parser_ will assert in debug mode when it encounters `_rpt_np_(n, _inf_)[_e_]` (this applies to unconditional _e_ only). ]]
-
-    [[ `_if_np_(pred)[p]` ]
-     [ Equivalent to `_e_(pred) >> p`. ]
-     [ `std::optional<_ATTR_np_(p)>` ]
-     [ It is an error to write `_if_np_(pred)`.  That is, it is an error to omit the conditionally matched parser `p`. ]]
-
-    [[ `_sw_np_(arg0)(arg1, p1)(arg2, p2) ...` ]
-     [ Equivalent to `p1` when `_RES_np_(arg0) == _RES_np_(arg1)`, `p2` when `_RES_np_(arg0) == _RES_np_(arg2)`, etc.  If there is such no `argN`, the behavior of _sw_ is undefined. ]
-     [ `std::variant<_ATTR_np_(p1), _ATTR_np_(p2), ...>` ]
-     [ It is an error to write `_sw_np_(arg0)`.  That is, it is an error to omit the conditionally matched parsers `p1`, `p2`, .... ]]
-
-    [[ _symbols_t_ ]
-     [ _symbols_ is an associative container of key, value pairs.  Each key is a _std_str_ and each value has type `T`.  In the Unicode parsing path, the strings are considered to be UTF-8 encoded; in the non-Unicode path, no encoding is assumed.  _symbols_ Matches the longest prefix `pre` of the input that is equal to one of the keys `k`.  If the length `len` of `pre` is zero, and there is no zero-length key, it does not match the input.  If `len` is positive, the generated attribute is the value associated with `k`.]
-     [ `T` ]
-     [ Unlike the other entries in this table, _symbols_ is a type, not an object. ]]
-]
-]

example/callback_json.cpp

@@ -126,7 +126,7 @@ namespace json {
         }
     };
 
-    bp::parser_interface<bp::uint_parser<uint32_t, 16, 4, 4>> const hex_4_def;
+    auto const hex_4_def = boost::parser::uint_.base<16>().digits<4>();
 
     auto const escape_seq_def = "\\u" > hex_4;
 

example/json.cpp

@@ -5,7 +5,7 @@
 // http://www.boost.org/LICENSE_1_0.txt)
 //[ extended_json_example
 // This header includes a type called json::value that acts as a
-// Javascript-like polymorphic value type.
+// JavaScript-like polymorphic value type.
 #include "json.hpp"
 
 #include <boost/parser/parser.hpp>
@@ -151,12 +151,10 @@ namespace json {
         }
     };
 
-    // This is the verbose form of declaration for the integer and unsigned
-    // integer parsers int_parser and uint_parser.  In this case, we don't
-    // want to use boost::parser::hex directly, since it has a variable number
-    // of digits.  We want to match exactly 4 digits, and this is how we
-    // declare a hexadecimal parser that matches exactly 4.
-    bp::parser_interface<bp::uint_parser<uint32_t, 16, 4, 4>> const hex_4_def;
+    // We don't want to use boost::parser::hex directly, since it has a
+    // variable number of digits.  We want to match exactly 4 digits, and this
+    // is how we declare a hexadecimal parser that matches exactly 4.
+    auto const hex_4_def = boost::parser::uint_.base<16>().digits<4>();
 
     // We use > here instead of >>, because once we see \u, we know that
     // exactly four hex digits must follow -- no other production rule starts

example/user_error_handler.cpp

@@ -31,9 +31,9 @@ struct logging_error_handler
     // and rethrow.  Returning fail fails the top-level parse; returning
     // rethrow just re-throws the parse_error exception that got us here in
     // the first place.
-    template<typename Iter, typename Sentinel, template<class> class Exception>
+    template<typename Iter, typename Sentinel>
     bp::error_handler_result
-    operator()(Iter first, Sentinel last, Exception<Iter> const & e) const
+    operator()(Iter first, Sentinel last, bp::parse_error<Iter> const & e) const
     {
         bp::write_formatted_expectation_failure_error_message(
             ofs_, filename_, first, last, e);

include/boost/parser/concepts.hpp

@@ -12,8 +12,6 @@
 
 #if defined(BOOST_PARSER_DOXYGEN) || BOOST_PARSER_USE_CONCEPTS
 
-#include <boost/parser/lexer_fwd.hpp>
-
 #include <ranges>
 
 
@@ -28,19 +26,15 @@ namespace boost { namespace parser {
         std::same_as<std::remove_cv_t<T>, char16_t>||
         std::same_as<std::remove_cv_t<T>, char32_t>;
 
-    template<typename T>
-    concept token_iter = is_token_v<std::iter_value_t<T>>;
-
     template<typename T>
     concept parsable_iter =
-        (std::forward_iterator<T> && code_unit<std::iter_value_t<T>>) ||
-        token_iter<T>;
+        std::forward_iterator<T> && code_unit<std::iter_value_t<T>>;
 
+    //[ parsable_range_like_concept
     //[ parsable_range_concept
     template<typename T>
-    concept parsable_range = (std::ranges::forward_range<T> &&
-                              code_unit<std::ranges::range_value_t<T>>) ||
-                             detail::is_tokens_view_v<T>;
+    concept parsable_range = std::ranges::forward_range<T> &&
+        code_unit<std::ranges::range_value_t<T>>;
     //]
 
     template<typename T>
@@ -49,6 +43,7 @@ namespace boost { namespace parser {
 
     template<typename T>
     concept parsable_range_like = parsable_range<T> || parsable_pointer<T>;
+    //]
 
     template<typename T>
     concept range_like = std::ranges::range<T> || parsable_pointer<T>;
@@ -64,7 +59,7 @@ namespace boost { namespace parser {
         std::declval<bool &>(),
         std::declval<int &>(),
         std::declval<ErrorHandler const &>(),
-        std::declval<detail::nope &>(),
+        std::declval<detail::nope const &>(),
         std::declval<detail::symbol_table_tries_t &>(),
         std::declval<detail::pending_symbol_table_operations_t &>()));
 

include/boost/parser/config.hpp

@@ -16,19 +16,19 @@
 
 /** Boost.Parser uses assertions (`BOOST_ASSERT()`) in several places to
     indicate that your use of the library has an error in it.  All of those
-    places could heve instead been ill-formed code, caught at compile time.
+    places could have instead been ill-formed code, caught at compile time.
     It is far quicker and easier to determine exactly where in your code such
     an error is located if this is a runtime failure; you can just look at the
-    stack in your favorite debugger.  However, if you want to make thes kinds
+    stack in your favorite debugger.  However, if you want to make these kinds
     of errors always ill-formed code, define this macro. */
 #    define BOOST_PARSER_NO_RUNTIME_ASSERTIONS
 
 /** Asserts that the given condition is true.  If
     `BOOST_PARSER_NO_RUNTIME_ASSERTIONS` macro is defined by the user,
-    `BOOST_PARSER_ASSERT` expends to a compile-time `static_assert()`.
+    `BOOST_PARSER_ASSERT` expands to a compile-time `static_assert()`.
     Otherwise, it expands to a run-time `BOOST_ASSERT()`.  Note that defining
     `BOOST_DISABLE_ASSERTS` disables the use of C `assert`, even when
-    `BOOST_ASSERT` is unavailble. */
+    `BOOST_ASSERT` is unavailable. */
 #    define BOOST_PARSER_ASSERT(condition)
 
 /** Boost.Parser will automatically use concepts to constrain templates when
@@ -59,12 +59,6 @@
     also defined. */
 #    define BOOST_PARSER_TRACE_TO_VS_OUTPUT
 
-/** When lexing is enabled, each token contains its position within the
-    underlying range.  To save a bit of space, an `unsiged int` is used for
-    this.  If you parse input sequences longer than 2^32-1 characters, define
-    `BOOST_PARSER_TOKEN_POSITION_TYPE` to be a larger integral type. */
-#    define BOOST_PARSER_TOKEN_POSITION_TYPE unsigned int
-
 #else
 
 #    ifdef BOOST_PARSER_NO_RUNTIME_ASSERTIONS
@@ -79,6 +73,12 @@
 
 #endif
 
+// Follows logic in boost/config/detail/select_compiler_config.hpp.
+#if defined(__clang__) && !defined(__ibmxl__) && !defined(__CODEGEARC__)
+#elif defined(__GNUC__) && !defined(__ibmxl__)
+#define BOOST_PARSER_GCC
+#endif
+
 #if defined(__cpp_lib_constexpr_algorithms)
 #    define BOOST_PARSER_ALGO_CONSTEXPR constexpr
 #else
@@ -109,10 +109,6 @@
 #    define BOOST_PARSER_MAX_AGGREGATE_SIZE 25
 #endif
 
-#if !defined(BOOST_PARSER_TOKEN_POSITION_TYPE)
-#    define BOOST_PARSER_TOKEN_POSITION_TYPE unsigned int
-#endif
-
 // VS2019 and VS2017 need conditional constexpr in some places, even in C++17 mode.
 #if !defined(_MSC_VER) || 1930 <= _MSC_VER
 #    define BOOST_PARSER_CONSTEXPR constexpr
@@ -126,18 +122,4 @@
 #    define BOOST_PARSER_TRACE_OSTREAM std::cout
 #endif
 
-#if defined(_MSC_VER)
-#    define BOOST_PARSER_DIAGNOSTIC_PUSH __pragma(warning(push))
-#    define BOOST_PARSER_DIAGNOSTIC_POP __pragma(warning(pop))
-#elif defined(__clang_major__)
-#    define BOOST_PARSER_DIAGNOSTIC_PUSH _Pragma("clang diagnostic push")
-#    define BOOST_PARSER_DIAGNOSTIC_POP _Pragma("clang diagnostic pop")
-#elif defined(__GNUC__)
-#    define BOOST_PARSER_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push")
-#    define BOOST_PARSER_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop")
-#else
-#    define BOOST_PARSER_DIAGNOSTIC_PUSH
-#    define BOOST_PARSER_DIAGNOSTIC_POP
-#endif
-
 #endif

include/boost/parser/detail/case_fold.hpp

@@ -47,7 +47,10 @@ namespace boost::parser::detail {
         // One-byte fast path.
         if (cp < 0x100) {
             // ASCII letter fast path.
-            if (0x41 <= cp && cp < 0x5a) {
+            if (0x61 <= cp && cp <= 0x7a) {
+                *out++ = cp;
+                return out;
+            } else if (0x41 <= cp && cp <= 0x5a) {
                 *out++ = cp + 0x20;
                 return out;
             } else if (cp == 0x00DF) {

include/boost/parser/detail/hl.hpp

@@ -197,31 +197,6 @@ namespace boost { namespace parser { namespace detail::hl {
     }
 
 
-    // fold_n
-
-    template<std::size_t I, std::size_t N>
-    struct fold_n_dispatch
-    {
-        template<typename F, typename State>
-        constexpr static auto call(State && s, F const & f)
-        {
-            if constexpr (I + 1 == N) {
-                return f((State &&)s, llong<I>{});
-            } else {
-                return fold_n_dispatch<I + 1, N>::call(
-                    f((State &&)s, llong<I>{}), f);
-            }
-        }
-    };
-
-    template<std::size_t N, typename F, typename State>
-    constexpr auto fold_n(State && s, F const & f)
-    {
-        static_assert(0 < N, "fold_n must operate on sequences of length >= 1");
-        return hl::fold_n_dispatch<0, N>::call((State &&)s, (F &&)f);
-    }
-
-
     // size
 
     template<typename... Args>

include/boost/parser/detail/printing.hpp

@@ -73,10 +73,10 @@ namespace boost { namespace parser { namespace detail {
         std::ostream & os,
         int components = 0);
 
-    template<typename Context, typename ParserTuple>
+    template<typename Context, typename ParserTuple, typename DelimiterParser>
     void print_parser(
         Context const & context,
-        perm_parser<ParserTuple> const & parser,
+        perm_parser<ParserTuple, DelimiterParser> const & parser,
         std::ostream & os,
         int components = 0);
 
@@ -245,6 +245,13 @@ namespace boost { namespace parser { namespace detail {
         std::ostream & os,
         int components = 0);
 
+    template<typename Context>
+    void print_parser(
+        Context const & context,
+        char_set_parser<symb_chars> const & parser,
+        std::ostream & os,
+        int components = 0);
+
     template<typename Context>
     void print_parser(
         Context const & context,
@@ -280,10 +287,14 @@ namespace boost { namespace parser { namespace detail {
         std::ostream & os,
         int components = 0);
 
-    template<typename Context, typename Quotes, typename Escapes>
+    template<
+        typename Context,
+        typename Quotes,
+        typename Escapes,
+        typename CharParser>
     void print_parser(
         Context const & context,
-        quoted_string_parser<Quotes, Escapes> const & parser,
+        quoted_string_parser<Quotes, Escapes, CharParser> const & parser,
         std::ostream & os,
         int components = 0);
 
@@ -355,13 +366,6 @@ namespace boost { namespace parser { namespace detail {
         std::ostream & os,
         int components = 0);
 
-    template<typename Context, typename TokenSpec, typename Expected>
-    void print_parser(
-        Context const & context,
-        token_parser<TokenSpec, Expected> const & parser,
-        std::ostream & os,
-        int components = 0);
-
     enum { trace_indent_factor = 2 };
 
     inline void trace_indent(std::ostream & os, int indent)
@@ -609,19 +613,29 @@ namespace boost { namespace parser { namespace detail {
             Context const & context,
             flags f,
             Attribute const & attr,
-            std::string name);
-        ~scoped_trace_t();
-        // implemented in printing_impl.hpp
+            std::string name) :
+            os_(os),
+            initial_first_(first),
+            first_(first),
+            last_(last),
+            context_(context),
+            flags_(f),
+            attr_(attr),
+            name_(std::move(name))
+        {
+            if (!detail::do_trace(flags_))
+                return;
+            detail::trace_prefix(os, first_, last_, context_, name_);
+        }
 
-        template<typename I, typename S>
-        void impl(I initial_first, I first, S last)
+        ~scoped_trace_t()
         {
             if (!detail::do_trace(flags_))
                 return;
             detail::trace_indent(os_, detail::_indent(context_));
             if (*context_.pass_) {
                 os_ << "matched ";
-                detail::trace_input(os_, initial_first, first);
+                detail::trace_input(os_, initial_first_, first_);
                 os_ << "\n";
                 detail::print_attribute(
                     os_,
@@ -630,7 +644,7 @@ namespace boost { namespace parser { namespace detail {
             } else {
                 os_ << "no match\n";
             }
-            detail::trace_suffix(os_, first, last, context_, name_);
+            detail::trace_suffix(os_, first_, last_, context_, name_);
         }
 
         std::ostream & os_;

include/boost/parser/detail/printing_impl.hpp

@@ -63,8 +63,9 @@ namespace boost { namespace parser { namespace detail {
     struct n_aray_parser<or_parser<ParserTuple>> : std::true_type
     {};
 
-    template<typename ParserTuple>
-    struct n_aray_parser<perm_parser<ParserTuple>> : std::true_type
+    template<typename ParserTuple, typename DelimiterParser>
+    struct n_aray_parser<perm_parser<ParserTuple, DelimiterParser>>
+        : std::true_type
     {};
 
     template<
@@ -206,15 +207,23 @@ namespace boost { namespace parser { namespace detail {
             context, parser, os, components, " | ...", " | ");
     }
 
-    template<typename Context, typename ParserTuple>
+    template<typename Context, typename ParserTuple, typename DelimiterParser>
     void print_parser(
         Context const & context,
-        perm_parser<ParserTuple> const & parser,
+        perm_parser<ParserTuple, DelimiterParser> const & parser,
         std::ostream & os,
         int components)
     {
+        if constexpr (!is_nope_v<DelimiterParser>) {
+            os << "delimiter(";
+            detail::print_parser(
+                context, parser.delimiter_parser_, os, components);
+            os << ")[";
+        }
         detail::print_or_like_parser(
             context, parser, os, components, " || ...", " || ");
+        if constexpr (!is_nope_v<DelimiterParser>)
+            os << "]";
     }
 
     template<
@@ -627,6 +636,16 @@ namespace boost { namespace parser { namespace detail {
         os << "punct";
     }
 
+    template<typename Context>
+    void print_parser(
+        Context const & context,
+        char_set_parser<symb_chars> const & parser,
+        std::ostream & os,
+        int components)
+    {
+        os << "symb";
+    }
+
     template<typename Context>
     void print_parser(
         Context const & context,
@@ -695,10 +714,14 @@ namespace boost { namespace parser { namespace detail {
         os << "\"";
     }
 
-    template<typename Context, typename Quotes, typename Escapes>
+    template<
+        typename Context,
+        typename Quotes,
+        typename Escapes,
+        typename CharParser>
     void print_parser(
         Context const & context,
-        quoted_string_parser<Quotes, Escapes> const & parser,
+        quoted_string_parser<Quotes, Escapes, CharParser> const & parser,
         std::ostream & os,
         int components)
     {
@@ -942,100 +965,6 @@ namespace boost { namespace parser { namespace detail {
             context, parser.or_parser_, os, components);
     }
 
-#if defined(BOOST_PARSER_TOKEN_PARSER_HPP)
-
-    template<typename Context, typename TokenSpec, typename Expected>
-    void print_parser(
-        Context const & context,
-        token_parser<TokenSpec, Expected> const & parser,
-        std::ostream & os,
-        int components)
-    {
-        constexpr bool do_print_value = requires { parser.expected_.value_; };
-
-        auto print_value = [&] {
-            if constexpr (do_print_value) {
-                if constexpr (std::ranges::range<
-                                  decltype(parser.expected_.value_)>) {
-                    os << '"';
-                    for (auto c : parser.expected_.value_ | text::as_utf8) {
-                        detail::print_char(os, c);
-                    }
-                    os << '"';
-                } else {
-                    detail::print(os, parser.expected_.value_);
-                }
-            }
-        };
-
-        if constexpr (requires {
-                          os << TokenSpec::id;
-                      } && std::is_enum_v<typename TokenSpec::id_type>) {
-            if constexpr (do_print_value) {
-                print_value();
-            } else {
-                os << TokenSpec::id;
-            }
-        } else {
-            os << "tok<" << (int)TokenSpec::id << '>';
-            if constexpr (do_print_value) {
-                os << '(';
-                print_value();
-                os << ')';
-            }
-        }
-    }
-
-#endif
-
-    template<
-        bool DoTrace,
-        typename Iter,
-        typename Sentinel,
-        typename Context,
-        typename Attribute>
-    scoped_trace_t<DoTrace, Iter, Sentinel, Context, Attribute>::scoped_trace_t(
-        std::ostream & os,
-        Iter & first,
-        Sentinel last,
-        Context const & context,
-        flags f,
-        Attribute const & attr,
-        std::string name) :
-        os_(os),
-        initial_first_(first),
-        first_(first),
-        last_(last),
-        context_(context),
-        flags_(f),
-        attr_(attr),
-        name_(std::move(name))
-    {
-        if (!detail::do_trace(flags_))
-            return;
-        if constexpr (is_token_iter_v<Iter>) {
-            detail::trace_prefix(
-                os, first_.base(), first_.range_end(), context_, name_);
-        } else {
-            detail::trace_prefix(os, first_, last_, context_, name_);
-        }
-    }
-
-    template<
-        bool DoTrace,
-        typename Iter,
-        typename Sentinel,
-        typename Context,
-        typename Attribute>
-    scoped_trace_t<DoTrace, Iter, Sentinel, Context, Attribute>::
-        ~scoped_trace_t()
-    {
-        if constexpr (is_token_iter_v<Iter>)
-            impl(first_.range_begin(), first_.base(), first_.range_end());
-        else
-            impl(initial_first_, first_, last_);
-    }
-
 }}}
 
 #endif

include/boost/parser/detail/stl_interfaces/iterator_interface.hpp

@@ -47,11 +47,9 @@ namespace boost::parser::detail { namespace stl_interfaces {
         `T`. */
     template<typename T>
 #if defined(BOOST_STL_INTERFACES_DOXYGEN) || BOOST_PARSER_USE_CONCEPTS
-    // clang-format off
         requires std::is_object_v<T>
 #endif
     struct proxy_arrow_result
-    // clang-format on
     {
         constexpr proxy_arrow_result(T const & value) noexcept(
             noexcept(T(value))) :
@@ -619,33 +617,25 @@ namespace boost::parser::detail { namespace stl_interfaces { BOOST_PARSER_DETAIL
         using iter_concept_t = typename iter_concept<Iterator>::type;
 
         template<typename D, typename DifferenceType>
-        // clang-format off
-        concept plus_eq = requires (D d) { d += DifferenceType(1); };
-        // clang-format on
+        concept plus_eq = requires(D d) { d += DifferenceType(1); };
 
         template<typename D, typename D2 = D>
-        // clang-format off
         concept base_3way =
 #if defined(__cpp_impl_three_way_comparison)
-            requires (D d, D2 d2) { access::base(d) <=> access::base(d2); };
+            requires(D d, D2 d2) { access::base(d) <=> access::base(d2); };
 #else
             false;
 #endif
-        // clang-format on
 
         template<typename D1, typename D2 = D1>
-        // clang-format off
         concept base_eq =
-            requires (D1 d1, D2 d2) { access::base(d1) == access::base(d2); };
-        // clang-format on
+            requires(D1 d1, D2 d2) { access::base(d1) == access::base(d2); };
 
         template<typename D, typename D2 = D>
-        // clang-format off
-        concept iter_sub = requires (D d, D2 d2) {
+        concept iter_sub = requires(D d, D2 d2) {
             typename D::difference_type;
-            {d - d2} -> std::convertible_to<typename D::difference_type>;
+            { d - d2 } -> std::convertible_to<typename D::difference_type>;
         };
-        // clang-format on
 
         // This iterator concept -> category mapping scheme follows the one
         // from zip_transform_view; see

include/boost/parser/detail/stl_interfaces/view_adaptor.hpp

@@ -24,8 +24,12 @@
 #define BOOST_PARSER_USE_CPP23_STD_RANGE_ADAPTOR_CLOSURE 0
 #endif
 
-#if !BOOST_PARSER_USE_CPP23_STD_RANGE_ADAPTOR_CLOSURE &&               \
-    BOOST_PARSER_DETAIL_STL_INTERFACES_USE_CONCEPTS && defined(__GNUC__) && 12 <= __GNUC__
+#if !BOOST_STL_INTERFACES_USE_CPP23_STD_RANGE_ADAPTOR_CLOSURE &&               \
+    BOOST_STL_INTERFACES_USE_CONCEPTS && defined(BOOST_GCC) && 14 <= __GNUC__
+#define BOOST_PARSER_USE_LIBSTDCPP_GCC14_RANGE_ADAPTOR_CLOSURE 1
+#elif !BOOST_PARSER_USE_CPP23_STD_RANGE_ADAPTOR_CLOSURE &&                     \
+    BOOST_PARSER_DETAIL_STL_INTERFACES_USE_CONCEPTS &&                         \
+    defined(BOOST_PARSER_GCC) && 12 <= __GNUC__
 #define BOOST_PARSER_USE_LIBSTDCPP_GCC12_RANGE_ADAPTOR_CLOSURE 1
 #else
 #define BOOST_PARSER_USE_LIBSTDCPP_GCC12_RANGE_ADAPTOR_CLOSURE 0
@@ -197,6 +201,11 @@ namespace boost::parser::detail { namespace stl_interfaces {
     template<typename D>
     using range_adaptor_closure = std::ranges::range_adaptor_closure<D>;
 
+#elif BOOST_PARSER_USE_LIBSTDCPP_GCC14_RANGE_ADAPTOR_CLOSURE
+
+    template<typename D>
+    using range_adaptor_closure = std::views::__adaptor::_RangeAdaptorClosure<D>;
+
 #elif BOOST_PARSER_USE_LIBSTDCPP_GCC12_RANGE_ADAPTOR_CLOSURE
 
     template<typename D>
@@ -258,7 +267,7 @@ namespace boost::parser::detail { namespace stl_interfaces {
     template<typename F>
     struct closure : range_adaptor_closure<closure<F>>
     {
-        constexpr closure(F f) : f_(f) {}
+        constexpr closure(F f) : f_(std::move(f)) {}
 
 #if BOOST_PARSER_DETAIL_STL_INTERFACES_USE_CONCEPTS
         template<typename T>
@@ -326,25 +335,22 @@ namespace boost::parser::detail { namespace stl_interfaces {
     template<typename F>
     struct adaptor
     {
-        constexpr adaptor(F f) : f_(f) {}
+        constexpr adaptor(F f) : f_(std::move(f)) {}
 
-        // clang-format off
         template<typename... Args>
         constexpr auto operator()(Args &&... args) const
-        // clang-format on
         {
 #if BOOST_PARSER_DETAIL_STL_INTERFACES_USE_CONCEPTS
             if constexpr (std::is_invocable_v<F const &, Args...>) {
-                return f_((Args &&) args...);
+                return f_((Args &&)args...);
             } else {
-                return closure(
-                    stl_interfaces::bind_back(f_, (Args &&) args...));
+                return closure(stl_interfaces::bind_back(f_, (Args &&)args...));
             }
 #else
             return detail::adaptor_impl<
                 F const &,
                 detail::is_invocable_v<F const &, Args...>,
-                Args...>::call(f_, (Args &&) args...);
+                Args...>::call(f_, (Args &&)args...);
 #endif
         }
 

include/boost/parser/detail/text/concepts.hpp

@@ -97,17 +97,14 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
 
     template<typename T>
     concept grapheme_iter =
-        // clang-format off
-        std::input_iterator<T> &&
-        code_point_range<std::iter_reference_t<T>> &&
+        std::input_iterator<T> && code_point_range<std::iter_reference_t<T>> &&
         requires(T t) {
-        { t.base() } -> code_point_iter;
-        // clang-format on
-    };
+            { t.base() } -> code_point_iter;
+        };
 
     template<typename T>
     concept grapheme_range = std::ranges::input_range<T> &&
-        grapheme_iter<std::ranges::iterator_t<T>>;
+                             grapheme_iter<std::ranges::iterator_t<T>>;
 
     template<typename R>
     using code_point_iterator_t = decltype(std::declval<R>().begin().base());
@@ -116,75 +113,63 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     using code_point_sentinel_t = decltype(std::declval<R>().end().base());
 
     template<typename T, format F>
-    concept grapheme_iter_code_unit =
-        // clang-format off
-        grapheme_iter<T> &&
-        requires(T t) {
+    concept grapheme_iter_code_unit = grapheme_iter<T> && requires(T t) {
         { t.base().base() } -> code_unit_iter<F>;
-        // clang-format on
     };
 
     template<typename T, format F>
-    concept grapheme_range_code_unit = grapheme_range<T> &&
+    concept grapheme_range_code_unit =
+        grapheme_range<T> &&
         grapheme_iter_code_unit<std::ranges::iterator_t<T>, F>;
 
 
     namespace dtl {
         template<typename T, class CodeUnit>
         concept eraseable_insertable_sized_bidi_range =
-            // clang-format off
-            std::ranges::sized_range<T> &&
-            std::ranges::input_range<T> &&
+            std::ranges::sized_range<T> && std::ranges::input_range<T> &&
             requires(T t, CodeUnit const * it) {
-                { t.erase(t.begin(), t.end()) } ->
-                    std::same_as<std::ranges::iterator_t<T>>;
-                { t.insert(t.end(), it, it) } ->
-                    std::same_as<std::ranges::iterator_t<T>>;
+                {
+                    t.erase(t.begin(), t.end())
+                } -> std::same_as<std::ranges::iterator_t<T>>;
+                {
+                    t.insert(t.end(), it, it)
+                } -> std::same_as<std::ranges::iterator_t<T>>;
             };
-        // clang-format on
     }
 
     template<typename T>
-    concept utf8_string =
-        // clang-format off
-        utf8_code_unit<std::ranges::range_value_t<T>> &&
-        dtl::eraseable_insertable_sized_bidi_range<
-            T, std::ranges::range_value_t<T>>;
-        // clang-format on
+    concept utf8_string = utf8_code_unit<std::ranges::range_value_t<T>> &&
+                          dtl::eraseable_insertable_sized_bidi_range<
+                              T,
+                              std::ranges::range_value_t<T>>;
 
     template<typename T>
-    concept utf16_string =
-        // clang-format off
-        utf16_code_unit<std::ranges::range_value_t<T>> &&
-        dtl::eraseable_insertable_sized_bidi_range<
-            T, std::ranges::range_value_t<T>>;
-        // clang-format on
+    concept utf16_string = utf16_code_unit<std::ranges::range_value_t<T>> &&
+                           dtl::eraseable_insertable_sized_bidi_range<
+                               T,
+                               std::ranges::range_value_t<T>>;
 
     template<typename T>
     concept utf_string = utf8_string<T> || utf16_string<T>;
 
     template<typename T>
-    // clang-format off
-        concept transcoding_error_handler = requires(T t, std::string_view msg) {
+    concept transcoding_error_handler = requires(T t, std::string_view msg) {
         { t(msg) } -> std::same_as<char32_t>;
-        // clang-format on
     };
     //]
 
     // Clang 13 defines __cpp_lib_concepts but not std::indirectly copyable.
 #if defined(__clang_major__) && __clang_major__ <= 13
     template<typename In, typename Out>
-    // clang-format off
     concept indirectly_copyable =
         std::indirectly_readable<In> &&
         std::indirectly_writable<Out, std::iter_reference_t<In>>;
-    // clang-format on
 #else
     template<typename In, typename Out>
     concept indirectly_copyable = std::indirectly_copyable<In, Out>;
 #endif
-
-}}}
+    }
+}}
 
 #endif
 

include/boost/parser/detail/text/detail/algorithm.hpp

@@ -51,13 +51,10 @@ namespace boost::parser::detail { namespace text { namespace detail {
     // that is comparable with T's interator type.
     template<typename T>
     concept cp_sentinel_gr_rng =
-        // clang-format off
-        grapheme_range<T> &&
-        !grapheme_iter<sentinel_t<T>> &&
+        grapheme_range<T> && !grapheme_iter<sentinel_t<T>> &&
         requires(iterator_t<T> first, sentinel_t<T> last) {
-        { first.base() == last } -> std::convertible_to<bool>;
-        // clang-format on
-    };
+            { first.base() == last } -> std::convertible_to<bool>;
+        };
 
     template<typename T>
     using gr_rng_cp_iter_t = decltype(std::declval<iterator_t<T>>().base());

include/boost/parser/detail/text/detail/all_t.hpp

@@ -42,7 +42,8 @@ namespace boost::parser::detail::text::detail {
     template<typename R>
     constexpr bool view =
 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS ||                                   \
-    (defined(__cpp_lib_concepts) && (!defined(__GNUC__) || 12 <= __GNUC__))
+    (defined(__cpp_lib_concepts) &&                                            \
+     (!defined(BOOST_PARSER_GCC) || 12 <= __GNUC__))
         std::ranges::view<R>
 #else
         range_<R> && !container_<R> &&
@@ -125,7 +126,7 @@ namespace boost::parser::detail::text::detail {
             else if constexpr (can_ref_view<R>)
                 return ref_view(r);
             else
-                return owning_view<T>(std::move(r));
+                return owning_view<T>((R &&)r);
         }
     };
 

include/boost/parser/detail/text/transcode_iterator.hpp

@@ -2350,11 +2350,9 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     }
 
     template<typename Cont>
-    // clang-format off
         requires requires { typename Cont::value_type; } &&
-        utf_code_unit<typename Cont::value_type>
+                 utf_code_unit<typename Cont::value_type>
     constexpr auto from_utf8_inserter(Cont & c, typename Cont::iterator it)
-    // clang-format on
     {
         if constexpr (sizeof(typename Cont::value_type) == 1) {
             return std::insert_iterator<Cont>(c, it);
@@ -2366,11 +2364,9 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     }
 
     template<typename Cont>
-    // clang-format off
         requires requires { typename Cont::value_type; } &&
-        utf_code_unit<typename Cont::value_type>
+                 utf_code_unit<typename Cont::value_type>
     constexpr auto from_utf16_inserter(Cont & c, typename Cont::iterator it)
-    // clang-format on
     {
         if constexpr (sizeof(typename Cont::value_type) == 1) {
             return utf_16_to_8_insert_iterator<Cont>(c, it);
@@ -2382,11 +2378,9 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     }
 
     template<typename Cont>
-    // clang-format off
         requires requires { typename Cont::value_type; } &&
-        utf_code_unit<typename Cont::value_type>
+                 utf_code_unit<typename Cont::value_type>
     constexpr auto from_utf32_inserter(Cont & c, typename Cont::iterator it)
-    // clang-format on
     {
         if constexpr (sizeof(typename Cont::value_type) == 1) {
             return utf_32_to_8_insert_iterator<Cont>(c, it);
@@ -2398,11 +2392,9 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     }
 
     template<typename Cont>
-    // clang-format off
         requires requires { typename Cont::value_type; } &&
-        utf_code_unit<typename Cont::value_type>
+                 utf_code_unit<typename Cont::value_type>
     constexpr auto from_utf8_back_inserter(Cont & c)
-    // clang-format on
     {
         if constexpr (sizeof(typename Cont::value_type) == 1) {
             return std::back_insert_iterator<Cont>(c);
@@ -2414,11 +2406,9 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     }
 
     template<typename Cont>
-    // clang-format off
         requires requires { typename Cont::value_type; } &&
-        utf_code_unit<typename Cont::value_type>
+                 utf_code_unit<typename Cont::value_type>
     constexpr auto from_utf16_back_inserter(Cont & c)
-    // clang-format on
     {
         if constexpr (sizeof(typename Cont::value_type) == 1) {
             return utf_16_to_8_back_insert_iterator<Cont>(c);
@@ -2430,11 +2420,9 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     }
 
     template<typename Cont>
-    // clang-format off
         requires requires { typename Cont::value_type; } &&
-        utf_code_unit<typename Cont::value_type>
+                 utf_code_unit<typename Cont::value_type>
     constexpr auto from_utf32_back_inserter(Cont & c)
-    // clang-format on
     {
         if constexpr (sizeof(typename Cont::value_type) == 1) {
             return utf_32_to_8_back_insert_iterator<Cont>(c);
@@ -2446,11 +2434,9 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     }
 
     template<typename Cont>
-    // clang-format off
         requires requires { typename Cont::value_type; } &&
-        utf_code_unit<typename Cont::value_type>
+                 utf_code_unit<typename Cont::value_type>
     constexpr auto from_utf8_front_inserter(Cont & c)
-    // clang-format on
     {
         if constexpr (sizeof(typename Cont::value_type) == 1) {
             return std::front_insert_iterator<Cont>(c);
@@ -2462,11 +2448,9 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     }
 
     template<typename Cont>
-    // clang-format off
         requires requires { typename Cont::value_type; } &&
-        utf_code_unit<typename Cont::value_type>
+                 utf_code_unit<typename Cont::value_type>
     constexpr auto from_utf16_front_inserter(Cont & c)
-    // clang-format on
     {
         if constexpr (sizeof(typename Cont::value_type) == 1) {
             return utf_16_to_8_front_insert_iterator<Cont>(c);
@@ -2478,11 +2462,9 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
     }
 
     template<typename Cont>
-    // clang-format off
         requires requires { typename Cont::value_type; } &&
-        utf_code_unit<typename Cont::value_type>
+                 utf_code_unit<typename Cont::value_type>
     constexpr auto from_utf32_front_inserter(Cont & c)
-    // clang-format on
     {
         if constexpr (sizeof(typename Cont::value_type) == 1) {
             return utf_32_to_8_front_insert_iterator<Cont>(c);
@@ -2492,7 +2474,6 @@ namespace boost::parser::detail { namespace text { BOOST_PARSER_DETAIL_TEXT_NAME
             return std::front_insert_iterator<Cont>(c);
         }
     }
-
 }}}
 
 #endif

include/boost/parser/detail/text/transcode_view.hpp

@@ -88,10 +88,12 @@ namespace boost::parser::detail { namespace text {
     {
         V base_ = V();
 
-        template<bool Const>
-        class iterator;
+        // HACK: SentType is here to work around irritating big-3
+        // implementation inconsistencies.
         template<bool Const>
         class sentinel;
+        template<bool Const, typename SentType = sentinel<Const>>
+        class iterator;
 
     public:
         constexpr project_view()
@@ -140,7 +142,7 @@ namespace boost::parser::detail { namespace text {
 #else
     template<typename V, typename F>
 #endif
-    template<bool Const>
+    template<bool Const, typename SentType>
     class project_view<V, F>::iterator
         : public boost::parser::detail::stl_interfaces::proxy_iterator_interface<
               iterator<Const>,
@@ -161,7 +163,7 @@ namespace boost::parser::detail { namespace text {
             decltype(detail::function_for_tag<F>(0))
 #endif
             ;
-        using sentinel = project_view<V, F>::sentinel<Const>;
+        using sentinel = SentType;
 
         friend boost::parser::detail::stl_interfaces::access;
         iterator_type & base_reference() noexcept { return it_; }
@@ -169,7 +171,7 @@ namespace boost::parser::detail { namespace text {
 
         iterator_type it_ = iterator_type();
 
-        friend project_view<V, F>::sentinel<Const>;
+        friend project_view<V, F>::template sentinel<Const>;
 
         template<bool OtherConst>
 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
@@ -423,49 +425,57 @@ namespace boost::parser::detail { namespace text {
 #if defined(__cpp_char8_t)
     inline constexpr detail::as_charn_impl<char8_view, format::utf8> as_char8_t;
 #endif
-    inline constexpr detail::as_charn_impl<char16_view, format::utf16> as_char16_t;
-    inline constexpr detail::as_charn_impl<char32_view, format::utf32> as_char32_t;
+    inline constexpr detail::as_charn_impl<char16_view, format::utf16>
+        as_char16_t;
+    inline constexpr detail::as_charn_impl<char32_view, format::utf32>
+        as_char32_t;
 
-    // clang-format off
 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
     template<utf_range V>
-    requires std::ranges::view<V> && std::ranges::forward_range<V>
+        requires std::ranges::view<V> && std::ranges::forward_range<V>
 #else
     template<typename V>
 #endif
-    class unpacking_view : public stl_interfaces::view_interface<unpacking_view<V>> {
-      V base_ = V();
+    class unpacking_view
+        : public stl_interfaces::view_interface<unpacking_view<V>>
+    {
+        V base_ = V();
 
     public:
-      constexpr unpacking_view()
+        constexpr unpacking_view()
 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
-          requires std::default_initializable<V>
+            requires std::default_initializable<V>
 #endif
-      = default;
-      constexpr unpacking_view(V base) : base_(std::move(base)) {}
+        = default;
+        constexpr unpacking_view(V base) : base_(std::move(base)) {}
 
-      constexpr V base() const &
+        constexpr V base() const &
 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
-          requires std::copy_constructible<V>
+            requires std::copy_constructible<V>
 #endif
-      { return base_; }
-      constexpr V base() && { return std::move(base_); }
+        {
+            return base_;
+        }
+        constexpr V base() && { return std::move(base_); }
 
-      constexpr auto code_units() const noexcept {
-        auto unpacked = boost::parser::detail::text::unpack_iterator_and_sentinel(detail::begin(base_), detail::end(base_));
-        return BOOST_PARSER_DETAIL_TEXT_SUBRANGE(unpacked.first, unpacked.last);
-      }
+        constexpr auto code_units() const noexcept
+        {
+            auto unpacked =
+                boost::parser::detail::text::unpack_iterator_and_sentinel(
+                    detail::begin(base_), detail::end(base_));
+            return BOOST_PARSER_DETAIL_TEXT_SUBRANGE(
+                unpacked.first, unpacked.last);
+        }
 
-      constexpr auto begin() { return code_units().begin(); }
-      constexpr auto begin() const { return code_units().begin(); }
+        constexpr auto begin() { return code_units().begin(); }
+        constexpr auto begin() const { return code_units().begin(); }
 
-      constexpr auto end() { return code_units().end(); }
-      constexpr auto end() const { return code_units().end(); }
+        constexpr auto end() { return code_units().end(); }
+        constexpr auto end() const { return code_units().end(); }
     };
 
     template<class R>
     unpacking_view(R &&) -> unpacking_view<detail::all_t<R>>;
-    // clang-format on
 
 #if BOOST_PARSER_DETAIL_TEXT_USE_CONCEPTS
     template<format Format, utf_range V>
@@ -693,14 +703,12 @@ namespace boost::parser::detail { namespace text {
             using T = detail::remove_cv_ref_t<R>;
             if constexpr (forward_range_v<T>) {
                 auto unpacked =
-                    boost::parser::detail::text::unpack_iterator_and_sentinel(
-                        detail::begin(r), detail::end(r));
+                    boost::parser::detail::text::unpack_iterator_and_sentinel(detail::begin(r), detail::end(r));
                 if constexpr (is_bounded_array_v<T>) {
                     constexpr auto n = std::extent_v<T>;
                     if (n && !r[n - 1])
                         --unpacked.last;
-                    return BOOST_PARSER_DETAIL_TEXT_SUBRANGE(
-                        unpacked.first, unpacked.last);
+                    return BOOST_PARSER_DETAIL_TEXT_SUBRANGE(unpacked.first, unpacked.last);
                 } else if constexpr (
                     !std::is_same_v<decltype(unpacked.first), iterator_t<R>> ||
                     !std::is_same_v<decltype(unpacked.last), sentinel_t<R>>) {

include/boost/parser/error_handling.hpp

@@ -31,7 +31,7 @@ namespace boost { namespace parser {
     }
 
     /** Returns the `line_position` for `it`, counting lines from the
-        beginning of the input `first`.  Requires non-token iterators. */
+        beginning of the input `first`. */
     template<typename Iter>
     line_position<Iter> find_line_position(Iter first, Iter it)
     {
@@ -57,7 +57,7 @@ namespace boost { namespace parser {
     }
 
     /** Returns the iterator to the end of the line in which `it` is
-        found.  Requires non-token iterators.  */
+        found.  */
     template<typename Iter, typename Sentinel>
     Iter find_line_end(Iter it, Sentinel last)
     {
@@ -73,16 +73,13 @@ namespace boost { namespace parser {
     std::ostream & write_formatted_message(
         std::ostream & os,
         std::string_view filename,
-        Iter first_,
-        Iter it_,
-        Sentinel last_,
+        Iter first,
+        Iter it,
+        Sentinel last,
         std::string_view message,
         int64_t preferred_max_line_length,
         int64_t max_after_caret)
     {
-        auto [first, it, last] =
-            parser::normalize_iterators(first_, it_, last_);
-
         if (!filename.empty())
             os << filename << ':';
         auto const position = parser::find_line_position(first, it);
@@ -93,6 +90,9 @@ namespace boost { namespace parser {
         os << ":\n";
 
         std::string underlining(std::distance(position.line_start, it), ' ');
+        std::transform(position.line_start, it,
+            underlining.begin(),
+            [](auto c) { return c == '\t' ? '\t' : ' ';});
         detail::trace_input(os, position.line_start, it, false, 1u << 31);
         if (it == last) {
             os << '\n' << underlining << "^\n";
@@ -121,15 +121,13 @@ namespace boost { namespace parser {
     std::ostream & write_formatted_message(
         std::ostream & os,
         std::wstring_view filename,
-        Iter first_,
-        Iter it_,
-        Sentinel last_,
+        Iter first,
+        Iter it,
+        Sentinel last,
         std::string_view message,
         int64_t preferred_max_line_length,
         int64_t max_after_caret)
     {
-        auto [first, it, last] =
-            parser::normalize_iterators(first_, it_, last_);
         auto const r = filename | parser::detail::text::as_utf8;
         std::string s(r.begin(), r.end());
         return parser::write_formatted_message(
@@ -144,24 +142,23 @@ namespace boost { namespace parser {
     }
 #endif
 
-    template<typename Iter, typename Sentinel, template<class> class Exception>
+    template<typename Iter, typename Sentinel>
     std::ostream & write_formatted_expectation_failure_error_message(
         std::ostream & os,
         std::string_view filename,
-        Iter first_,
-        Sentinel last_,
-        Exception<Iter> const & e,
+        Iter first,
+        Sentinel last,
+        parse_error<Iter> const & e,
         int64_t preferred_max_line_length,
         int64_t max_after_caret)
     {
         std::string message = "error: Expected ";
         message += e.what();
-        auto [first, it, last] = parser::normalize_iterators(first_, e, last_);
         return parser::write_formatted_message(
             os,
             filename,
             first,
-            it,
+            e.iter,
             last,
             message,
             preferred_max_line_length,
@@ -169,13 +166,13 @@ namespace boost { namespace parser {
     }
 
 #if defined(_MSC_VER)
-    template<typename Iter, typename Sentinel, template<class> class Exception>
+    template<typename Iter, typename Sentinel>
     std::ostream & write_formatted_expectation_failure_error_message(
         std::ostream & os,
         std::wstring_view filename,
         Iter first,
         Sentinel last,
-        Exception<Iter> const & e,
+        parse_error<Iter> const & e,
         int64_t preferred_max_line_length,
         int64_t max_after_caret)
     {
@@ -186,35 +183,6 @@ namespace boost { namespace parser {
     }
 #endif
 
-    namespace detail {
-        template<typename I, typename S>
-        auto normalize_iterators_impl(I first, I it, S last)
-        {
-            if constexpr (detail::is_token_iter_v<I>)
-                return std::tuple(it.range_begin(), it.base(), it.range_end());
-            else
-                return std::tuple(first, it, last);
-        }
-    }
-
-    template<typename I, typename S>
-    auto normalize_iterators(I first, I it, S last)
-    {
-        return detail::normalize_iterators_impl(first, it, last);
-    }
-
-    template<typename I, typename S>
-    auto normalize_iterators(I first, parse_error<I> e, S last)
-    {
-        return detail::normalize_iterators_impl(first, e.iter, last);
-    }
-
-    template<typename I, typename S>
-    auto normalize_iterators(I first, lex_error<I> e, S last)
-    {
-        return detail::normalize_iterators_impl(first, e.iter, last);
-    }
-
     /** An error handler that allows users to supply callbacks to handle the
         reporting of warnings and errors.  The reporting of errors and/or
         warnings can be suppressed by supplying one or both
@@ -246,13 +214,9 @@ namespace boost { namespace parser {
             filename_.assign(r.begin(), r.end());
         }
 #endif
-        template<
-            typename Iter,
-            typename Sentinel,
-            template<class>
-            class Exception>
+        template<typename Iter, typename Sentinel>
         error_handler_result
-        operator()(Iter first, Sentinel last, Exception<Iter> const & e) const
+        operator()(Iter first, Sentinel last, parse_error<Iter> const & e) const
         {
             if (error_) {
                 std::stringstream ss;
@@ -299,15 +263,13 @@ namespace boost { namespace parser {
         std::string filename_;
     };
 
+    /** An error handler that just re-throws any exception generated by the
+        parse. */
     struct rethrow_error_handler
     {
-        template<
-            typename Iter,
-            typename Sentinel,
-            template<class>
-            class Exception>
+        template<typename Iter, typename Sentinel>
         error_handler_result
-        operator()(Iter first, Sentinel last, Exception<Iter> const & e) const
+        operator()(Iter first, Sentinel last, parse_error<Iter> const & e) const
         {
             return error_handler_result::rethrow;
         }
@@ -329,6 +291,8 @@ namespace boost { namespace parser {
     };
 
 #if defined(_MSC_VER) || defined(BOOST_PARSER_DOXYGEN)
+    /** An error handler that prints to the Visual Studio debugger via calls
+        to `OutputDebugString()`. */
     struct vs_output_error_handler : stream_error_handler
     {
         vs_output_error_handler() :
@@ -348,9 +312,9 @@ namespace boost { namespace parser {
 
     // implementations
 
-    template<typename Iter, typename Sentinel, template<class> class Exception>
+    template<typename Iter, typename Sentinel>
     error_handler_result default_error_handler::operator()(
-        Iter first, Sentinel last, Exception<Iter> const & e) const
+        Iter first, Sentinel last, parse_error<Iter> const & e) const
     {
         parser::write_formatted_expectation_failure_error_message(
             std::cerr, "", first, last, e);
@@ -382,9 +346,9 @@ namespace boost { namespace parser {
         diagnose(kind, message, context, parser::_where(context).begin());
     }
 
-    template<typename Iter, typename Sentinel, template<class> class Exception>
+    template<typename Iter, typename Sentinel>
     error_handler_result stream_error_handler::operator()(
-        Iter first, Sentinel last, Exception<Iter> const & e) const
+        Iter first, Sentinel last, parse_error<Iter> const & e) const
     {
         std::ostream * os = err_os_;
         if (!os)

include/boost/parser/error_handling_fwd.hpp

@@ -24,29 +24,10 @@ namespace boost { namespace parser {
     template<typename Iter>
     struct parse_error : std::runtime_error
     {
-        parse_error(Iter it, std::string msg) :
-            runtime_error(""), message(msg), iter(it)
+        parse_error(Iter it, std::string const & msg) :
+            runtime_error(msg), iter(it)
         {}
 
-        char const * what() const noexcept override { return message.c_str(); }
-
-        std::string message;
-        Iter iter;
-    };
-
-    /** The exception thrown when a lexing error is encountered, consisting of
-        an iterator to the point of failure, and a description of the value
-        expected at the point of failure in `what()`. */
-    template<typename Iter>
-    struct lex_error : std::runtime_error
-    {
-        lex_error(Iter it, std::string msg) :
-            runtime_error(""), message(msg), iter(it)
-        {}
-
-        char const * what() const noexcept override { return message.c_str(); }
-
-        std::string message;
         Iter iter;
     };
 
@@ -61,7 +42,7 @@ namespace boost { namespace parser {
     };
 
     /** Writes a formatted message (meaning prefixed with the file name, line,
-        and column number) to `os`.  Normalizes token iterators as needed. */
+        and column number) to `os`. */
     template<typename Iter, typename Sentinel>
     std::ostream & write_formatted_message(
         std::ostream & os,
@@ -75,8 +56,7 @@ namespace boost { namespace parser {
 
 #if defined(_MSC_VER) || defined(BOOST_PARSER_DOXYGEN)
     /** Writes a formatted message (meaning prefixed with the file name, line,
-        and column number) to `os`.  Normalizes token iterators as needed.
-        This overload is Windows-only. */
+        and column number) to `os`.  This overload is Windows-only. */
     template<typename Iter, typename Sentinel>
     std::ostream & write_formatted_message(
         std::ostream & os,
@@ -90,59 +70,32 @@ namespace boost { namespace parser {
 #endif
 
     /** Writes a formatted parse-expectation failure (meaning prefixed with
-        the file name, line, and column number) to `os`.  Normalizes token
-        iterators as needed. */
-    template<typename Iter, typename Sentinel, template<class> class Exception>
+        the file name, line, and column number) to `os`. */
+    template<typename Iter, typename Sentinel>
     std::ostream & write_formatted_expectation_failure_error_message(
         std::ostream & os,
         std::string_view filename,
         Iter first,
         Sentinel last,
-        Exception<Iter> const & e,
+        parse_error<Iter> const & e,
         int64_t preferred_max_line_length = 80,
         int64_t max_after_caret = 40);
 
 #if defined(_MSC_VER) || defined(BOOST_PARSER_DOXYGEN)
     /** Writes a formatted parse-expectation failure (meaning prefixed with
-        the file name, line, and column number) to `os`.  Normalizes token
-        iterators as needed.  This overload is Windows-only. */
-    template<typename Iter, typename Sentinel, template<class> class Exception>
+        the file name, line, and column number) to `os`.  This overload is
+        Windows-only. */
+    template<typename Iter, typename Sentinel>
     std::ostream & write_formatted_expectation_failure_error_message(
         std::ostream & os,
         std::wstring_view filename,
         Iter first,
         Sentinel last,
-        Exception<Iter> const & e,
+        parse_error<Iter> const & e,
         int64_t preferred_max_line_length = 80,
         int64_t max_after_caret = 40);
 #endif
 
-    /** Returns a tuple of three iterators (corresponding to `first`, `curr`,
-        and `last`) that are suitable for use in the other error handling
-        functions, many of which require iterators into the undelying sequence
-        being parsed.  For non-token parsing cases, this is effectively a
-        no-op; the given iterators are simply returned as-is. */
-    template<typename I, typename S>
-    auto normalize_iterators(I first, I curr, S last);
-
-    /** Returns a tuple of three iterators (corresponding to `first`, the
-        iterator captured in `e`, and `last`) that are suitable for use in the
-        other error handling functions, many of which require iterators into
-        the undelying sequence being parsed.  For non-token parsing cases,
-        this is effectively a no-op; the given iterators are simply returned
-        as-is. */
-    template<typename I, typename S>
-    auto normalize_iterators(I first, parse_error<I> e, S last);
-
-    /** Returns a tuple of three iterators (corresponding to `first`, the
-        iterator captured in `e`, and `last`) that are suitable for use in the
-        other error handling functions, many of which require iterators into
-        the undelying sequence being parsed.  For non-token parsing cases,
-        this is effectively a no-op; the given iterators are simply returned
-        as-is. */
-    template<typename I, typename S>
-    auto normalize_iterators(I first, lex_error<I> e, S last);
-
     /** The kinds of diagnostics that can be handled by an error handler. */
     enum class diagnostic_kind {
         error,  /// An error diagnostic.
@@ -151,21 +104,17 @@ namespace boost { namespace parser {
 
     /** The error handler used when the user does not specify a custom one.
         This error handler prints warnings and errors to `std::cerr`, and does
-        not have an associcated filename. */
+        not have an associated filename. */
     struct default_error_handler
     {
         constexpr default_error_handler() = default;
 
-        /** Handles a `parse_error` or `lex_error` exception thrown during
-            parsing/lexing.  A formatted parse-expectation failure is printed
-            to `std::cerr`.  Always returns `error_handler_result::fail`. */
-        template<
-            typename Iter,
-            typename Sentinel,
-            template<class>
-            class Exception>
-        error_handler_result
-        operator()(Iter first, Sentinel last, Exception<Iter> const & e) const;
+        /** Handles a `parse_error` exception thrown during parsing.  A
+            formatted parse-expectation failure is printed to `std::cerr`.
+            Always returns `error_handler_result::fail`. */
+        template<typename Iter, typename Sentinel>
+        error_handler_result operator()(
+            Iter first, Sentinel last, parse_error<Iter> const & e) const;
 
         /** Prints `message` to `std::cerr`.  The diagnostic is printed with
             the given `kind`, indicating the location as being at `it`.  This
@@ -242,13 +191,9 @@ namespace boost { namespace parser {
             formatted parse-expectation failure is printed to `*err_os_` when
             `err_os_` is non-null, or `std::cerr` otherwise.  Always returns
             `error_handler_result::fail`. */
-        template<
-            typename Iter,
-            typename Sentinel,
-            template<class>
-            class Exception>
+        template<typename Iter, typename Sentinel>
         error_handler_result
-        operator()(Iter first, Sentinel last, Exception<Iter> const & e) const;
+        operator()(Iter first, Sentinel last, parse_error<Iter> const & e) const;
 
         /** Let `std::ostream * s = kind == diagnostic_kind::error : err_os_ :
             warn_os_`; prints `message` to `*s` when `s` is non-null, or
@@ -280,16 +225,6 @@ namespace boost { namespace parser {
         std::ostream * warn_os_;
     };
 
-    /** An error handler that just re-throws any exception generated by the
-        parse. */
-    struct rethrow_error_handler;
-
-#if defined(_MSC_VER) || defined(BOOST_PARSER_DOXYGEN)
-    /** An error handler that prints to the Visual Studio debugger via calls
-        to `OutputDebugString()`. */
-    struct vs_output_error_handler;
-#endif
-
 }}
 
 #endif

include/boost/parser/lexer_fwd.hpp

@@ -1,36 +0,0 @@
-// Copyright (C) 2024 T. Zachary Laine
-//
-// Distributed under the Boost Software License, Version 1.0. (See
-// accompanying file LICENSE_1_0.txt or copy at
-// http://www.boost.org/LICENSE_1_0.txt)
-#ifndef BOOST_PARSER_LEXER_FWD_HPP
-#define BOOST_PARSER_LEXER_FWD_HPP
-
-#include <ranges>
-#include <vector>
-
-namespace boost { namespace parser {
-
-    /** A `std::views`-compatible view that provides the tokens from the given
-        contiguous range, using the given lexer and optional token cache.  You
-        should typically not need to use this type directly; use
-        `boost::parser::to_tokens` instead. */
-    template<
-        std::ranges::contiguous_range V,
-        typename Lexer,
-        typename TokenCache = std::vector<typename Lexer::token_type>>
-        requires std::ranges::view<V>
-    struct tokens_view;
-
-    namespace detail {
-        template<typename T>
-        constexpr bool is_tokens_view_v = false;
-
-        template<typename V, typename Lexer, typename TokenCache>
-        constexpr bool is_tokens_view_v<tokens_view<V, Lexer, TokenCache>> =
-            true;
-    }
-
-}}
-
-#endif

include/boost/parser/parser_fwd.hpp

@@ -68,32 +68,6 @@ namespace boost { namespace parser {
         return BOOST_PARSER_SUBRANGE(ptr, detail::text::null_sentinel);
     }
 
-    /** The token ID used for whitespace tokens. */
-    inline constexpr int ws_id = -1000000;
-
-    /** The token ID used for single-character tokens. */
-    inline constexpr int character_id = -2000000;
-
-#ifdef BOOST_PARSER_DOXYGEN
-
-    /** A type trait that evaluates to `true` iff `T` is a specialization of
-        `boost::parser::token`. */
-    template<typename T>
-    constexpr bool is_token_v = detail::foo;
-
-#else
-
-    template<typename CharType>
-    struct token;
-
-    template<typename T>
-    constexpr bool is_token_v = false;
-
-    template<typename CharType>
-    constexpr bool is_token_v<token<CharType>> = true;
-
-#endif
-
     namespace detail {
         template<typename T>
         constexpr bool is_optional_v = enable_optional<T>;
@@ -141,7 +115,7 @@ namespace boost { namespace parser {
             bool & success,
             int & indent,
             ErrorHandler const & error_handler,
-            nope &,
+            nope const &,
             symbol_table_tries_t & symbol_table_tries,
             pending_symbol_table_operations_t &
                 pending_symbol_table_operations) noexcept;
@@ -169,22 +143,12 @@ namespace boost { namespace parser {
 
         struct punct_chars
         {};
+        struct symb_chars
+        {};
         struct lower_case_chars
         {};
         struct upper_case_chars
         {};
-
-        struct any_token_value
-        {
-            template<typename T>
-            bool matches_value(T) const
-            {
-                return true;
-            }
-        };
-
-        template<typename I, typename Context>
-        struct scoped_lexeme;
     }
 
     /** Repeats the application of another parser `p` of type `Parser`,
@@ -245,7 +209,7 @@ namespace boost { namespace parser {
         `ParserTuple`, not the order of the parsers' matches.  It is an error
         to specialize `perm_parser` with a `ParserTuple` template parameter
         that includes an `eps_parser`. */
-    template<typename ParserTuple>
+    template<typename ParserTuple, typename DelimiterParser>
     struct perm_parser;
 
     /** Applies each parser in `ParserTuple`, in order.  The parse succeeds
@@ -418,7 +382,10 @@ namespace boost { namespace parser {
 
     /** Matches a string delimited by quotation marks; produces a
         `std::string` attribute. */
-    template<typename Quotes = detail::nope, typename Escapes = detail::nope>
+    template<
+        typename Quotes = detail::nope,
+        typename Escapes = detail::nope,
+        typename CharParser = char_parser<detail::nope>>
     struct quoted_string_parser;
 
     /** Matches an end-of-line (`NewlinesOnly == true`), whitespace
@@ -437,8 +404,8 @@ namespace boost { namespace parser {
         and at most `MaxDigits`, producing an attribute of type `T`.  Fails on
         any other input.  The parse will also fail if `Expected` is anything
         but `detail::nope` (which it is by default), and the produced
-        attribute is not equal to `expected_`.  `Radix` must be in `[2,
-        36]`. */
+        attribute is not equal to `expected_`.  `Radix` must be one of `2`,
+        `8`, `10`, or `16`. */
     template<
         typename T,
         int Radix = 10,
@@ -466,20 +433,6 @@ namespace boost { namespace parser {
     template<typename T>
     struct float_parser;
 
-    /** A tag type used to represent a value type that is any specialization
-        of `std::basic_string_view`.  Which specialization is used depends on
-        the input. */
-    struct string_view_tag
-    {};
-
-    /** Matches a token from the input with ID `TokenSpec::id`.  Fails on any
-        other input.  The parse will also fail if `Expected` is anything but
-        `detail::nope` (which it is by default), and `expected_.matches(attr)`
-        is not `true` for the produced attribute `attr`.  Used in token
-        parsing only.  */
-    template<typename TokenSpec, typename Expected>
-    struct token_parser;
-
     /** Applies at most one of the parsers in `OrParser`.  If `switch_value_`
         matches one or more of the values in the parsers in `OrParser`, the
         first such parser is applied, and the success or failure and attribute

include/boost/parser/replace.hpp

@@ -77,22 +77,16 @@ namespace boost::parser {
             range_rvalue_reference_t<V2>>;
 
 #if BOOST_PARSER_USE_CONCEPTS
-        // clang-format off
         template<typename ReplacementV, typename V>
         concept concatable = requires {
             typename detail::concat_reference_t<ReplacementV, V>;
             typename detail::concat_value_t<ReplacementV, V>;
             typename detail::concat_rvalue_reference_t<ReplacementV, V>;
         };
-        // clang-format on
 #else
         template<typename ReplacementV, typename V>
-        // clang-format off
-        using concatable_expr = decltype(
-            std::declval<concat_reference_t<ReplacementV, V>>(),
-            std::declval<concat_value_t<ReplacementV, V>>(),
-            std::declval<concat_rvalue_reference_t<ReplacementV, V>>());
-        // clang-format on
+        using concatable_expr =
+            decltype(std::declval<concat_reference_t<ReplacementV, V>>(), std::declval<concat_value_t<ReplacementV, V>>(), std::declval<concat_rvalue_reference_t<ReplacementV, V>>());
         template<typename ReplacementV, typename V>
         constexpr bool concatable =
             is_detected_v<concatable_expr, ReplacementV, V>;
@@ -107,7 +101,7 @@ namespace boost::parser {
 #endif
             >
 #if BOOST_PARSER_USE_CONCEPTS
-        requires concatable<V1, V2>
+            requires concatable<V1, V2>
 #endif
         struct either_iterator_impl
             : detail::stl_interfaces::iterator_interface<
@@ -169,14 +163,12 @@ namespace boost::parser {
             either_iterator_impl<V1, V2>>;
 
 #if BOOST_PARSER_USE_CONCEPTS
-        // clang-format off
         template<typename ReplacementV, typename V>
-        concept replacement_for = requires (ReplacementV replacement, V base) {
+        concept replacement_for = requires(ReplacementV replacement, V base) {
             { either_iterator<V, ReplacementV>(replacement.begin()) };
             { either_iterator<V, ReplacementV>(replacement.end()) };
             { either_iterator<V, ReplacementV>(base.begin()) };
         };
-        // clang-format on
 #else
         template<typename ReplacementV, typename V>
         using replacement_for_expr = decltype(
@@ -528,23 +520,19 @@ namespace boost::parser {
                 typename GlobalState,
                 typename ErrorHandler,
                 typename SkipParser>
-            requires
-                // clang-format off
-                std::ranges::viewable_range<R> &&
-                std::ranges::viewable_range<ReplacementR> &&
-                // clang-format on
-                can_replace_view<
-                    to_range_t<R>,
-                    decltype(to_range<
-                             ReplacementR,
-                             true,
-                             detail::range_utf_format_v<R>>::
-                                 call(std::declval<ReplacementR>())),
-                    Parser,
-                    GlobalState,
-                    ErrorHandler,
-                    SkipParser>
-                // clang-format off
+                requires std::ranges::viewable_range<R> &&
+                         std::ranges::viewable_range<ReplacementR> &&
+                         can_replace_view<
+                             to_range_t<R>,
+                             decltype(to_range<
+                                      ReplacementR,
+                                      true,
+                                      detail::range_utf_format_v<R>>::
+                                          call(std::declval<ReplacementR>())),
+                             Parser,
+                             GlobalState,
+                             ErrorHandler,
+                             SkipParser>
             [[nodiscard]] constexpr auto operator()(
                 R && r,
                 parser_interface<Parser, GlobalState, ErrorHandler> const &
@@ -552,10 +540,9 @@ namespace boost::parser {
                 parser_interface<SkipParser> const & skip,
                 ReplacementR && replacement,
                 trace trace_mode = trace::off) const
-            // clang-format on
             {
                 return replace_view(
-                    to_range<R>::call((R &&) r),
+                    to_range<R>::call((R &&)r),
                     parser,
                     skip,
                     to_range<
@@ -572,36 +559,31 @@ namespace boost::parser {
                 typename Parser,
                 typename GlobalState,
                 typename ErrorHandler>
-            requires
-                // clang-format off
-                std::ranges::viewable_range<R> &&
-                std::ranges::viewable_range<ReplacementR> &&
-                // clang-format on
-                can_replace_view<
-                    to_range_t<R>,
-                    decltype(to_range<
-                             ReplacementR,
-                             true,
-                             detail::range_utf_format_v<R>>::
-                                 call(std::declval<ReplacementR>())),
-                    Parser,
-                    GlobalState,
-                    ErrorHandler,
-                    parser_interface<eps_parser<detail::phony>>>
-                // clang-format off
+                requires std::ranges::viewable_range<R> &&
+                         std::ranges::viewable_range<ReplacementR> &&
+                         can_replace_view<
+                             to_range_t<R>,
+                             decltype(to_range<
+                                      ReplacementR,
+                                      true,
+                                      detail::range_utf_format_v<R>>::
+                                          call(std::declval<ReplacementR>())),
+                             Parser,
+                             GlobalState,
+                             ErrorHandler,
+                             parser_interface<eps_parser<detail::phony>>>
             [[nodiscard]] constexpr auto operator()(
                 R && r,
                 parser_interface<Parser, GlobalState, ErrorHandler> const &
                     parser,
                 ReplacementR && replacement,
                 trace trace_mode = trace::off) const
-            // clang-format on
             {
                 return (*this)(
-                    (R &&) r,
+                    (R &&)r,
                     parser,
                     parser_interface<eps_parser<detail::phony>>{},
-                    (ReplacementR &&) replacement,
+                    (ReplacementR &&)replacement,
                     trace_mode);
             }
 

include/boost/parser/search.hpp

@@ -541,25 +541,21 @@ namespace boost::parser {
                 typename GlobalState,
                 typename ErrorHandler,
                 typename SkipParser>
-            requires(
-                std::ranges::viewable_range<R>) &&
-                can_search_all_view<
-                    to_range_t<R>,
-                    Parser,
-                    GlobalState,
-                    ErrorHandler,
-                    SkipParser>
-                // clang-format off
+                requires(std::ranges::viewable_range<R>) && can_search_all_view<
+                                                                to_range_t<R>,
+                                                                Parser,
+                                                                GlobalState,
+                                                                ErrorHandler,
+                                                                SkipParser>
             [[nodiscard]] constexpr auto operator()(
                 R && r,
                 parser_interface<Parser, GlobalState, ErrorHandler> const &
                     parser,
                 parser_interface<SkipParser> const & skip,
                 trace trace_mode = trace::off) const
-            // clang-format on
             {
                 return search_all_view(
-                    to_range<R>::call((R &&) r), parser, skip, trace_mode);
+                    to_range<R>::call((R &&)r), parser, skip, trace_mode);
             }
 
             template<
@@ -567,24 +563,21 @@ namespace boost::parser {
                 typename Parser,
                 typename GlobalState,
                 typename ErrorHandler>
-            requires(
-                std::ranges::viewable_range<R>) &&
-                can_search_all_view<
-                    to_range_t<R>,
-                    Parser,
-                    GlobalState,
-                    ErrorHandler,
-                    parser_interface<eps_parser<detail::phony>>>
-                // clang-format off
+                requires(std::ranges::viewable_range<R>) &&
+                        can_search_all_view<
+                            to_range_t<R>,
+                            Parser,
+                            GlobalState,
+                            ErrorHandler,
+                            parser_interface<eps_parser<detail::phony>>>
             [[nodiscard]] constexpr auto operator()(
                 R && r,
                 parser_interface<Parser, GlobalState, ErrorHandler> const &
                     parser,
                 trace trace_mode = trace::off) const
-            // clang-format on
             {
                 return (*this)(
-                    (R &&) r,
+                    (R &&)r,
                     parser,
                     parser_interface<eps_parser<detail::phony>>{},
                     trace_mode);

include/boost/parser/split.hpp

@@ -258,25 +258,21 @@ namespace boost::parser {
                 typename GlobalState,
                 typename ErrorHandler,
                 typename SkipParser>
-            requires(
-                std::ranges::viewable_range<R>) &&
-                can_split_view<
-                    to_range_t<R>,
-                    Parser,
-                    GlobalState,
-                    ErrorHandler,
-                    SkipParser>
-                // clang-format off
+                requires(std::ranges::viewable_range<R>) && can_split_view<
+                                                                to_range_t<R>,
+                                                                Parser,
+                                                                GlobalState,
+                                                                ErrorHandler,
+                                                                SkipParser>
             [[nodiscard]] constexpr auto operator()(
                 R && r,
                 parser_interface<Parser, GlobalState, ErrorHandler> const &
                     parser,
                 parser_interface<SkipParser> const & skip,
                 trace trace_mode = trace::off) const
-            // clang-format on
             {
                 return split_view(
-                    to_range<R>::call((R &&) r), parser, skip, trace_mode);
+                    to_range<R>::call((R &&)r), parser, skip, trace_mode);
             }
 
             template<
@@ -284,24 +280,21 @@ namespace boost::parser {
                 typename Parser,
                 typename GlobalState,
                 typename ErrorHandler>
-            requires(
-                std::ranges::viewable_range<R>) &&
-                can_split_view<
-                    to_range_t<R>,
-                    Parser,
-                    GlobalState,
-                    ErrorHandler,
-                    parser_interface<eps_parser<detail::phony>>>
-                // clang-format off
+                requires(std::ranges::viewable_range<R>) &&
+                        can_split_view<
+                            to_range_t<R>,
+                            Parser,
+                            GlobalState,
+                            ErrorHandler,
+                            parser_interface<eps_parser<detail::phony>>>
             [[nodiscard]] constexpr auto operator()(
                 R && r,
                 parser_interface<Parser, GlobalState, ErrorHandler> const &
                     parser,
                 trace trace_mode = trace::off) const
-            // clang-format on
             {
                 return (*this)(
-                    (R &&) r,
+                    (R &&)r,
                     parser,
                     parser_interface<eps_parser<detail::phony>>{},
                     trace_mode);

include/boost/parser/token_parser.hpp

@@ -1,305 +0,0 @@
-// Copyright (C) 2024 T. Zachary Laine
-//
-// Distributed under the Boost Software License, Version 1.0. (See
-// accompanying file LICENSE_1_0.txt or copy at
-// http://www.boost.org/LICENSE_1_0.txt)
-#ifndef BOOST_PARSER_TOKEN_PARSER_HPP
-#define BOOST_PARSER_TOKEN_PARSER_HPP
-
-#if !defined(BOOST_PARSER_PARSER_HPP) || !defined(BOOST_PARSER_LEXER_HPP)
-#error "token_parser.hpp must be included after lexer.hpp and parser.hpp."
-#endif
-
-#include <boost/parser/parser_fwd.hpp>
-#include <boost/parser/concepts.hpp>
-#include <boost/parser/error_handling.hpp>
-
-#include <algorithm>
-
-
-namespace boost { namespace parser {
-
-    namespace detail {
-        template<typename AttributeType, typename CharType>
-        std::optional<AttributeType> token_as(token<CharType> tok)
-        {
-            if constexpr (std::is_floating_point_v<AttributeType>) {
-                if (tok.has_long_double())
-                    return tok.get_long_double();
-                return std::nullopt;
-            } else if constexpr (std::is_integral_v<AttributeType>) {
-                if (tok.has_long_long())
-                    return AttributeType(tok.get_long_long());
-                return std::nullopt;
-            } else {
-                if (tok.has_string_view())
-                    return tok.get_string_view();
-                return std::nullopt;
-            }
-        }
-
-        template<typename Expected>
-        struct token_with_value
-        {
-            explicit constexpr token_with_value(Expected value) :
-                expected_(value)
-            {}
-
-            template<typename T, typename Context>
-            bool matches(T value, Context const & context) const
-            {
-                return value == detail::resolve(context, expected_);
-            }
-
-            Expected expected_;
-        };
-
-        template<typename Subrange>
-        struct token_with_string_view
-        {
-            explicit constexpr token_with_string_view(Subrange subrange) :
-                subrange_(subrange)
-            {}
-
-            template<typename CharType, typename Context>
-            bool matches(
-                std::basic_string_view<CharType> value, Context const &) const
-            {
-                auto const value_cps =
-                    make_subrange<CharType>(value.begin(), value.end());
-                auto const subrange_cps =
-                    make_subrange<CharType>(subrange_.begin(), subrange_.end());
-                return std::ranges::equal(
-                    value_cps, subrange_cps, [](auto a, auto b) {
-                        return cast_char(a) == cast_char(b);
-                    });
-            }
-
-            template<typename T>
-            static auto cast_char(T c)
-            {
-                if constexpr (std::same_as<T, char>) {
-                    return (unsigned char)c;
-                } else {
-                    return c;
-                }
-            }
-
-            template<typename CharType, typename I, typename S>
-            static auto make_subrange(I f, S l)
-            {
-                auto subrange = BOOST_PARSER_SUBRANGE(f, l);
-                if constexpr (std::is_same_v<CharType, char>) {
-                    return subrange;
-                } else {
-                    return subrange | detail::text::as_utf32;
-                }
-            }
-
-            Subrange subrange_;
-        };
-    }
-
-#ifndef BOOST_PARSER_DOXYGEN
-
-    template<typename TokenSpec, typename Expected>
-    struct token_parser
-    {
-        using token_spec = TokenSpec;
-
-        template<typename Iter>
-        using attribute_type = std::conditional_t<
-            std::same_as<typename token_spec::value_type, string_view_tag>,
-            std::basic_string_view<
-                typename detail::iter_value_t<Iter>::char_type>,
-            typename token_spec::value_type>;
-
-        constexpr token_parser() = default;
-        constexpr token_parser(Expected expected) : expected_(expected) {}
-
-        template<
-            typename Iter,
-            typename Sentinel,
-            typename Context,
-            typename SkipParser>
-        auto call(
-            Iter & first,
-            Sentinel last,
-            Context const & context,
-            SkipParser const & skip,
-            detail::flags flags,
-            bool & success) const -> attribute_type<Iter>
-        {
-            attribute_type<Iter> retval;
-            call(first, last, context, skip, flags, success, retval);
-            return retval;
-        }
-
-        template<
-            typename Iter,
-            typename Sentinel,
-            typename Context,
-            typename SkipParser,
-            typename Attribute>
-        void call(
-            Iter & first,
-            Sentinel last,
-            Context const & context,
-            SkipParser const & skip,
-            detail::flags flags,
-            bool & success,
-            Attribute & retval) const
-        {
-            using value_type = std::remove_cvref_t<decltype(*first)>;
-            static_assert(
-                is_token_v<value_type>,
-                "token_parser can only be used when parsing sequences of "
-                "tokens.");
-
-            [[maybe_unused]] auto _ = detail::scoped_trace(
-                *this, first, last, context, flags, retval);
-
-            if (first == last) {
-                success = false;
-                return;
-            }
-
-            value_type const x = *first;
-            if (x.id() != (int)token_spec::id) {
-                success = false;
-                return;
-            }
-
-            constexpr bool use_expected = !std::same_as<Expected, detail::nope>;
-            if (use_expected || detail::gen_attrs(flags)) {
-                auto opt_attr = detail::token_as<attribute_type<Iter>>(x);
-                if constexpr (use_expected) {
-                    if (!opt_attr || !expected_.matches(*opt_attr, context)) {
-                        success = false;
-                        return;
-                    }
-                }
-                if (detail::gen_attrs(flags))
-                    detail::assign(retval, *opt_attr);
-            }
-
-            ++first;
-        }
-
-        /** Returns a `parser_interface` containing a `token_parser` that
-            matches `value`. */
-        template<typename T>
-            requires(!parsable_range_like<T>)
-        constexpr auto operator()(T value) const noexcept
-        {
-            BOOST_PARSER_ASSERT(
-                (detail::is_nope_v<Expected> &&
-                 "If you're seeing this, you tried to chain calls on one of "
-                 "your token_spec's, like 'my_token_spec(id1)(id2)'.  Quit "
-                 "it!'"));
-            return parser_interface(
-                token_parser<TokenSpec, detail::token_with_value<T>>(
-                    detail::token_with_value(std::move(value))));
-        }
-
-        /** Returns a `parser_interface` containing a `token_parser` that
-            matches the range `r`.  If the token being matched during the
-            parse has a `char_type` of `char8_t`, `char16_t`, or `char32_t`,
-            the elements of `r` are transcoded from their presumed encoding to
-            UTF-32 during the comparison.  Otherwise, the character being
-            matched is directly compared to the elements of `r`. */
-        template<parsable_range_like R>
-        constexpr auto operator()(R && r) const noexcept
-        {
-            BOOST_PARSER_ASSERT(
-                ((!std::is_rvalue_reference_v<R &&> ||
-                  !detail::is_range<detail::remove_cv_ref_t<R>>) &&
-                 "It looks like you tried to pass an rvalue range to "
-                 "token_spec().  Don't do that, or you'll end up with dangling "
-                 "references."));
-            BOOST_PARSER_ASSERT(
-                (detail::is_nope_v<Expected> &&
-                 "If you're seeing this, you tried to chain calls on "
-                 "token_spec, like 'token_spec(char-set)(char-set)'.  Quit "
-                 "it!'"));
-            auto expected =
-                detail::token_with_string_view{make_expected_range((R &&)r)};
-            return parser_interface(
-                token_parser<token_spec, decltype(expected)>(expected));
-        }
-
-        template<typename R>
-        static constexpr auto make_expected_range(R && r)
-        {
-            using T = detail::remove_cv_ref_t<R>;
-            if constexpr (std::is_bounded_array_v<T>) {
-                constexpr auto n = std::extent_v<T>;
-                auto const offset = n && !r[n - 1] ? 1 : 0;
-                return BOOST_PARSER_SUBRANGE(
-                    std::ranges::begin(r), std::ranges::end(r) - offset);
-            } else {
-                return BOOST_PARSER_SUBRANGE(
-                    std::ranges::begin(r), std::ranges::end(r));
-            }
-        }
-
-        // TODO: Consider adding a special string_view-like type that can be
-        // passed to the range overload above.  It would be based on
-        // adobe::name_t.  When comparing it to a tokens' string_view, if it
-        // matches, it would replace the token's string_view, so that
-        // subsequent comparisons are O(1) in the length of the string.
-
-        Expected expected_;
-    };
-
-#endif
-
-    /** A variable template that defines a token parser associated with
-        `boost::parser::token_spec_t<Regex, ID, ValueType, Base>`.  This token
-        parser can be used to specify a lexer, and may also be used in
-        parsers. */
-    template<
-        ctll::fixed_string Regex,
-        auto ID,
-        typename ValueType = string_view_tag,
-        int Base = 10>
-    constexpr parser_interface token_spec{
-        token_parser<token_spec_t<Regex, ID, ValueType, Base>, detail::nope>()};
-
-#ifndef BOOST_PARSER_DOXYGEN
-
-    template<
-        typename CharType,
-        typename ID,
-        ctll::fixed_string WsStr,
-        ctll::fixed_string RegexStr,
-        detail::nttp_array IDs,
-        detail::nttp_array Specs>
-    template<
-        ctll::fixed_string RegexStr2,
-        auto ID2,
-        typename ValueType,
-        int Base>
-    constexpr auto
-    lexer_t<CharType, ID, WsStr, RegexStr, IDs, Specs>::operator|(
-        parser_interface<token_parser<
-            token_spec_t<RegexStr2, ID2, ValueType, Base>,
-            detail::nope>> const &) const
-    {
-        static_assert(
-            std::same_as<ID, decltype(ID2)>,
-            "All id_types must be the same for all token_specs.");
-        constexpr auto new_regex =
-            detail::wrap_escape_concat<regex_str, RegexStr2>();
-        constexpr auto new_ids = IDs.template append<(int)ID2>();
-        constexpr auto new_specs = Specs.template append<detail::parse_spec_for<
-            token_spec_t<RegexStr2, ID2, ValueType, Base>>()>();
-        return lexer_t<CharType, ID, WsStr, new_regex, new_ids, new_specs>{};
-    }
-
-
-#endif
-
-}}
-
-#endif

include/boost/parser/transform_replace.hpp

@@ -4,7 +4,8 @@
 #include <boost/parser/replace.hpp>
 
 #if (!defined(_MSC_VER) || BOOST_PARSER_USE_CONCEPTS) &&                       \
-    (!defined(__GNUC__) || 12 <= __GNUC__ || !BOOST_PARSER_USE_CONCEPTS)
+    (!defined(BOOST_PARSER_GCC) || 12 <= __GNUC__ ||                           \
+     !BOOST_PARSER_USE_CONCEPTS)
 
 
 namespace boost::parser {
@@ -22,22 +23,21 @@ namespace boost::parser {
             std::declval<
                 parse_context<false, false, I, S, default_error_handler>>(),
             ws,
-            flags(uint32_t(flags::gen_attrs) | uint32_t(flags::use_skip)),
+            detail::default_flags(),
             std::declval<bool &>()));
         template<typename R, typename Parser>
         using range_attr_t = attr_type<iterator_t<R>, sentinel_t<R>, Parser>;
 
 #if BOOST_PARSER_USE_CONCEPTS
-        // clang-format off
         template<typename F, typename V, typename Parser>
         concept transform_replacement_for =
             std::regular_invocable<F &, range_attr_t<V, Parser>> &&
             detail::replacement_for<
-                std::invoke_result_t<F &, range_attr_t<V, Parser>>, V> &&
+                std::invoke_result_t<F &, range_attr_t<V, Parser>>,
+                V> &&
             (detail::range_utf_format_v<V> ==
              detail::range_utf_format_v<
                  std::invoke_result_t<F &, range_attr_t<V, Parser>>>);
-        // clang-format on
 #else
         template<typename F, typename V, typename Parser>
         using transform_replacement_for_expr = decltype(std::declval<F &>()(
@@ -646,24 +646,20 @@ namespace boost::parser {
                 typename GlobalState,
                 typename ErrorHandler,
                 typename SkipParser>
-            requires
-                // clang-format off
-                std::ranges::viewable_range<R> &&
-                std::regular_invocable<
-                    F &,
-                    range_attr_t<to_range_t<R>, Parser>> &&
-                // clang-format on
-                can_transform_replace_view<
-                    to_range_t<R>,
-                    utf_rvalue_shim<
-                        to_range_t<R>,
-                        std::remove_cvref_t<F>,
-                        range_attr_t<to_range_t<R>, Parser>>,
-                    Parser,
-                    GlobalState,
-                    ErrorHandler,
-                    SkipParser>
-                // clang-format off
+                requires std::ranges::viewable_range<R> &&
+                         std::regular_invocable<
+                             F &,
+                             range_attr_t<to_range_t<R>, Parser>> &&
+                         can_transform_replace_view<
+                             to_range_t<R>,
+                             utf_rvalue_shim<
+                                 to_range_t<R>,
+                                 std::remove_cvref_t<F>,
+                                 range_attr_t<to_range_t<R>, Parser>>,
+                             Parser,
+                             GlobalState,
+                             ErrorHandler,
+                             SkipParser>
             [[nodiscard]] constexpr auto operator()(
                 R && r,
                 parser_interface<Parser, GlobalState, ErrorHandler> const &
@@ -671,16 +667,15 @@ namespace boost::parser {
                 parser_interface<SkipParser> const & skip,
                 F && f,
                 trace trace_mode = trace::off) const
-            // clang-format on
             {
                 return transform_replace_view(
-                    to_range<R>::call((R &&) r),
+                    to_range<R>::call((R &&)r),
                     parser,
                     skip,
                     utf_rvalue_shim<
                         to_range_t<R>,
                         std::remove_cvref_t<F>,
-                        range_attr_t<to_range_t<R>, Parser>>((F &&) f),
+                        range_attr_t<to_range_t<R>, Parser>>((F &&)f),
                     trace_mode);
             }
 
@@ -690,37 +685,32 @@ namespace boost::parser {
                 typename Parser,
                 typename GlobalState,
                 typename ErrorHandler>
-            requires
-                // clang-format off
-                std::ranges::viewable_range<R> &&
-                std::regular_invocable<
-                    F &,
-                    range_attr_t<to_range_t<R>, Parser>> &&
-                // clang-format on
-                can_transform_replace_view<
-                    to_range_t<R>,
-                    utf_rvalue_shim<
-                        to_range_t<R>,
-                        std::remove_cvref_t<F>,
-                        range_attr_t<to_range_t<R>, Parser>>,
-                    Parser,
-                    GlobalState,
-                    ErrorHandler,
-                    parser_interface<eps_parser<detail::phony>>>
-                // clang-format off
+                requires std::ranges::viewable_range<R> &&
+                         std::regular_invocable<
+                             F &,
+                             range_attr_t<to_range_t<R>, Parser>> &&
+                         can_transform_replace_view<
+                             to_range_t<R>,
+                             utf_rvalue_shim<
+                                 to_range_t<R>,
+                                 std::remove_cvref_t<F>,
+                                 range_attr_t<to_range_t<R>, Parser>>,
+                             Parser,
+                             GlobalState,
+                             ErrorHandler,
+                             parser_interface<eps_parser<detail::phony>>>
             [[nodiscard]] constexpr auto operator()(
                 R && r,
                 parser_interface<Parser, GlobalState, ErrorHandler> const &
                     parser,
                 F && f,
                 trace trace_mode = trace::off) const
-            // clang-format on
             {
                 return (*this)(
-                    (R &&) r,
+                    (R &&)r,
                     parser,
                     parser_interface<eps_parser<detail::phony>>{},
-                    (F &&) f,
+                    (F &&)f,
                     trace_mode);
             }
 

include/boost/parser/tuple.hpp

@@ -128,7 +128,7 @@ namespace boost { namespace parser {
         /** A literal that can be used to concisely name `parser::llong`
             integral constants. */
         template<char... chars>
-        constexpr auto operator"" _c()
+        constexpr auto operator""_c()
         {
             constexpr long long n =
                 detail::parse_llong<sizeof...(chars)>({chars...});
@@ -185,14 +185,10 @@ namespace boost { namespace parser {
             template<typename T>
             operator T() const && noexcept
             {
-#if defined(__GNUC__) && __GNUC__ < 13
                 // Yuck.
                 std::remove_reference_t<T> * ptr = nullptr;
                 ptr += 1; // warning mitigation
                 return *ptr;
-#else
-                return std::declval<T>();
-#endif
             }
         };
 

meta/libraries.json

@@ -4,5 +4,6 @@
     "authors": [ "T. Zachary Laine" ],
     "maintainers": [ "Zach Laine <whatwasthataddress -at- gmail.com>" ],
     "description": "A parser combinator library.",
-    "category": [ "Parsing" ]
+    "category": [ "Parsing" ],
+    "cxxstd": "17"
 }

test/CMakeLists.txt

@@ -6,23 +6,6 @@ enable_testing()
 
 add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -j4 -C ${CMAKE_CFG_INTDIR})
 
-if (CXX_STD GREATER_EQUAL 20)
-    include(FetchContent)
-    FetchContent_Declare(
-      ctre
-      URL https://raw.githubusercontent.com/hanickadot/compile-time-regular-expressions/refs/heads/main/single-header/ctre-unicode.hpp
-      DOWNLOAD_NO_EXTRACT true
-    )
-
-    FetchContent_MakeAvailable(ctre)
-
-    set(ctre_include_dir ${CMAKE_BINARY_DIR}/_deps/ctre-src)
-    add_library(ctre_single_header INTERFACE)
-    target_include_directories(ctre_single_header INTERFACE ${ctre_include_dir})
-else()
-    add_library(ctre_single_header INTERFACE)
-endif()
-
 ##################################################
 # Parser tests
 ##################################################
@@ -48,7 +31,6 @@ add_test(NAME parser_api COMMAND parser_api)
 
 add_executable(
     compile_tests
-    compile_include_lexer_parser.cpp
     compile_tests_main.cpp
     compile_attribute.cpp
     compile_seq_attribute.cpp
@@ -57,12 +39,12 @@ add_executable(
     compile_all_t.cpp
 )
 set_property(TARGET compile_tests PROPERTY CXX_STANDARD ${CXX_STD})
-target_link_libraries(compile_tests parser boost ctre_single_header)
+target_link_libraries(compile_tests parser boost)
 
 macro(add_test_executable name)
     add_executable(${name} ${name}.cpp)
     set_property(TARGET ${name} PROPERTY CXX_STANDARD ${CXX_STD})
-    target_link_libraries(${name} parser boost ctre_single_header ${link_flags})
+    target_link_libraries(${name} parser boost ${link_flags})
     if (MSVC)
         target_compile_options(${name} PRIVATE /source-charset:utf-8 /bigobj)
     elseif (USE_ASAN OR USE_UBSAN)
@@ -100,14 +82,6 @@ add_test_executable(parser_seq_permutations_1)
 add_test_executable(parser_seq_permutations_2)
 add_test_executable(parser_or_permutations_1)
 add_test_executable(parser_or_permutations_2)
-if (CXX_STD GREATER_EQUAL 20)
-    add_test_executable(lexer)
-    add_test_executable(lexer_adobe_files)
-    add_test_executable(lexer_and_parser)
-    add_test_executable(lexer_and_parser_api)
-    add_test_executable(lexer_and_parser_terminals)
-    add_test_executable(lexer_and_parser_symbol_table)
-endif()
 
 if (MSVC)
     add_executable(vs_output_tracing tracing.cpp)

test/Jamfile.v2

@@ -8,6 +8,7 @@ import testing ;
 
 project
   : requirements <library>/boost/charconv//boost_charconv
+    <library>/boost/parser//boost_parser
   ;
 
 compile compile_all_t.cpp ;

test/adobe_lexer.hpp

@@ -1,76 +0,0 @@
-/**
- *   Copyright (C) 2024 T. Zachary Laine
- *
- *   Distributed under the Boost Software License, Version 1.0. (See
- *   accompanying file LICENSE_1_0.txt or copy at
- *   http://www.boost.org/LICENSE_1_0.txt)
- */
-#ifndef BOOST_PARSER_TEST_ADOBE_LEXER
-#define BOOST_PARSER_TEST_ADOBE_LEXER
-
-#include <boost/parser/parser.hpp>
-#include <boost/parser/lexer.hpp>
-
-
-namespace bp = boost::parser;
-
-enum class adobe_tokens {
-    keyword_true_false,
-    keyword_empty,
-    identifier,
-    lead_comment,
-    trail_comment,
-    quoted_string,
-    number,
-    eq_op,
-    rel_op,
-    mul_op,
-    define,
-    or_,
-    and_
-};
-
-constexpr auto true_false =
-    bp::token_spec<"true|false", adobe_tokens::keyword_true_false, bool>;
-constexpr auto empty = bp::token_spec<"empty", adobe_tokens::keyword_empty>;
-constexpr auto identifier =
-    bp::token_spec<"[a-zA-Z]\\w*", adobe_tokens::identifier>;
-constexpr auto lead_comment = bp::token_spec<
-    "\\/\\*[^*]*\\*+(?:[^/*][^*]*\\*+)*\\/",
-    adobe_tokens::lead_comment>;
-constexpr auto trail_comment =
-    bp::token_spec<"\\/\\/.*$", adobe_tokens::trail_comment>;
-constexpr auto quoted_string =
-    bp::token_spec<"\\\"[^\\\"]*\\\"|'[^']*'", adobe_tokens::quoted_string>;
-constexpr auto number =
-    bp::token_spec<"\\d+(?:\\.\\d*)?", adobe_tokens::number, double>;
-constexpr auto eq_op = bp::token_spec<"==|!=", adobe_tokens::eq_op>;
-constexpr auto define = bp::token_spec<"<==", adobe_tokens::define>;
-constexpr auto rel_op = bp::token_spec<"<|>|<=|>=", adobe_tokens::rel_op>;
-constexpr auto mul_op = bp::token_spec<"\\*|\\/|%", adobe_tokens::mul_op>;
-constexpr auto or_ = bp::token_spec<"\\|\\|", adobe_tokens::or_>;
-constexpr auto and_ = bp::token_spec<"&&", adobe_tokens::and_>;
-
-constexpr auto adobe_lexer = bp::lexer<char, adobe_tokens> | true_false |
-                             empty | identifier | lead_comment | trail_comment |
-                             quoted_string | number | eq_op | define | rel_op |
-                             mul_op | or_ | and_ |
-                             bp::token_chars<
-                                 '=',
-                                 '+',
-                                 '-',
-                                 '!',
-                                 '?',
-                                 ':',
-                                 '.',
-                                 ',',
-                                 '(',
-                                 ')',
-                                 '[',
-                                 ']',
-                                 '{',
-                                 '}',
-                                 '@',
-                                 ';'>;
-
-#endif

test/compile_attribute.cpp

@@ -65,7 +65,7 @@ void compile_attribute_non_unicode()
             using attr_t = decltype(parse(null_term(r), parser));
             static_assert(std::is_same_v<attr_t, std::optional<char>>);
             static_assert(std::is_same_v<
-                          attribute_t<decltype(null_term(r)), decltype(parser)>,
+                          attribute_t<decltype(r), decltype(parser)>,
                           char>);
         }
         {
@@ -73,7 +73,7 @@ void compile_attribute_non_unicode()
             using attr_t = decltype(parse(null_term(r), parser));
             static_assert(std::is_same_v<attr_t, std::optional<std::string>>);
             static_assert(std::is_same_v<
-                          attribute_t<decltype(null_term(r)), decltype(parser)>,
+                          attribute_t<decltype(r), decltype(parser)>,
                           std::string>);
         }
         {
@@ -81,7 +81,7 @@ void compile_attribute_non_unicode()
             using attr_t = decltype(parse(null_term(r), parser));
             static_assert(std::is_same_v<attr_t, std::optional<std::string>>);
             static_assert(std::is_same_v<
-                          attribute_t<decltype(null_term(r)), decltype(parser)>,
+                          attribute_t<decltype(r), decltype(parser)>,
                           std::string>);
         }
         {
@@ -89,7 +89,7 @@ void compile_attribute_non_unicode()
             using attr_t = decltype(parse(null_term(r), parser));
             static_assert(std::is_same_v<attr_t, std::optional<std::string>>);
             static_assert(std::is_same_v<
-                          attribute_t<decltype(null_term(r)), decltype(parser)>,
+                          attribute_t<decltype(r), decltype(parser)>,
                           std::string>);
         }
         {
@@ -97,7 +97,7 @@ void compile_attribute_non_unicode()
             using attr_t = decltype(parse(null_term(r), parser));
             static_assert(std::is_same_v<attr_t, std::optional<std::string>>);
             static_assert(std::is_same_v<
-                          attribute_t<decltype(null_term(r)), decltype(parser)>,
+                          attribute_t<decltype(r), decltype(parser)>,
                           std::string>);
         }
     }

test/compile_include_lexer_parser.cpp

@@ -1,10 +0,0 @@
-// Copyright (C) 2024 T. Zachary Laine
-//
-// Distributed under the Boost Software License, Version 1.0. (See
-// accompanying file LICENSE_1_0.txt or copy at
-// http://www.boost.org/LICENSE_1_0.txt)
-#include <boost/parser/config.hpp>
-#if BOOST_PARSER_USE_CONCEPTS
-#include <boost/parser/lexer.hpp>
-#endif
-#include <boost/parser/parser.hpp>

test/github_issues.cpp

@@ -237,6 +237,366 @@ void github_issue_125()
     }
 }
 
+void github_issue_209()
+{
+    namespace bp = boost::parser;
+
+    BOOST_TEST(std::is_sorted(
+        std::begin(bp::detail::char_set<detail::punct_chars>::chars),
+        std::end(bp::detail::char_set<detail::punct_chars>::chars)));
+
+    BOOST_TEST(std::is_sorted(
+        std::begin(bp::detail::char_set<detail::symb_chars>::chars),
+        std::end(bp::detail::char_set<detail::symb_chars>::chars)));
+
+    BOOST_TEST(std::is_sorted(
+        std::begin(bp::detail::char_set<detail::lower_case_chars>::chars),
+        std::end(bp::detail::char_set<detail::lower_case_chars>::chars)));
+
+    BOOST_TEST(std::is_sorted(
+        std::begin(bp::detail::char_set<detail::upper_case_chars>::chars),
+        std::end(bp::detail::char_set<detail::upper_case_chars>::chars)));
+}
+
+void github_issue_223()
+{
+    namespace bp = boost::parser;
+
+    // failing case
+    {
+        std::vector<char> v;
+        const auto parser = *('x' | bp::char_('y'));
+        bp::parse("xy", parser, bp::ws, v);
+
+        BOOST_TEST(v.size() == 1);
+        BOOST_TEST(v == std::vector<char>({'y'}));
+
+        // the assert fails since there are two elements in the vector: '\0'
+        // and 'y'. Seems pretty surprising to me
+    }
+
+    // working case
+    {
+        const auto parser = *('x' | bp::char_('y'));
+        const auto result = bp::parse("xy", parser, bp::ws);
+
+        BOOST_TEST(result->size() == 1);
+        BOOST_TEST(*(*result)[0] == 'y');
+
+        // success, the vector has only one 'y' element
+    }
+}
+
+namespace github_issue_248_ {
+    namespace bp = boost::parser;
+
+    static constexpr bp::rule<struct symbol, int> symbol = "//";
+    static constexpr bp::rule<struct vector, std::vector<int>> list =
+        "<int>(,<int>)*";
+    static constexpr bp::rule<struct working, std::vector<int>> working =
+        "working";
+    static constexpr bp::rule<struct failing, std::vector<int>> failing =
+        "failing";
+
+    static auto const symbol_def = bp::symbols<int>{{"//", 0}};
+    static constexpr auto list_def = bp::int_ % ',';
+    static constexpr auto working_def = -symbol >> (bp::int_ % ',');
+    static constexpr auto failing_def = -symbol >> list;
+
+    BOOST_PARSER_DEFINE_RULES(symbol, list, working, failing);
+}
+
+void github_issue_248()
+{
+    namespace bp = boost::parser;
+
+    using namespace github_issue_248_;
+
+    {
+        auto const result = bp::parse("//1,2,3", working, bp::ws);
+        auto const expected = std::vector<int>{0, 1, 2, 3};
+        BOOST_TEST(result.has_value());
+        bool const equal = std::equal(
+            result->begin(), result->end(), expected.begin(), expected.end());
+        BOOST_TEST(equal);
+        if (!equal) {
+            std::cout << "contents of *result:\n";
+            for (auto x : *result) {
+                std::cout << x << '\n';
+            }
+            std::cout << '\n';
+        }
+    }
+    {
+        auto const result = bp::parse("//1,2,3", failing, bp::ws);
+        auto const expected = std::vector<int>{0, 1, 2, 3};
+        BOOST_TEST(result.has_value());
+        bool const equal = std::equal(
+            result->begin(), result->end(), expected.begin(), expected.end());
+        BOOST_TEST(equal);
+        if (!equal) {
+            std::cout << "contents of *result:\n";
+            for (auto x : *result) {
+                std::cout << x << '\n';
+            }
+            std::cout << '\n';
+        }
+    }
+}
+
+#if BOOST_PARSER_USE_CONCEPTS
+namespace github_issue_268_ {
+    namespace bp = boost::parser;
+    constexpr bp::rule<struct name, std::string_view> name = "name";
+    auto name_def = bp::string_view[bp::lexeme[+(
+        bp::lower | bp::upper | bp::digit | bp::char_("_"))]];
+    BOOST_PARSER_DEFINE_RULES(name)
+    constexpr bp::rule<struct qd_vec, std::vector<double>> qd_vec = "qd_vec";
+    auto qd_vec_def = bp::lit("\"") >>
+                      bp::double_ %
+                          (bp::lit(",") |
+                           (bp::lit("\"") >> bp::lit(",") >> bp::lit("\""))) >>
+                      bp::lit('\"');
+    BOOST_PARSER_DEFINE_RULES(qd_vec)
+    struct lu_table_template_1
+    {
+        std::vector<double> index_1;
+        std::string_view variable_1;
+    };
+    constexpr boost::parser::
+        rule<struct lu_table_template_1_tag, lu_table_template_1>
+            lu_table_template_1_rule = "lu_table_template_1";
+    auto lu_table_template_1_rule_def = (bp::lit("index_1") >> '(' >> qd_vec >>
+                                         ')' >> ';') >>
+                                        (bp::lit("variable_1") >> ':' >> name >>
+                                         ';');
+    BOOST_PARSER_DEFINE_RULES(lu_table_template_1_rule)
+
+    constexpr boost::parser::
+        rule<struct lu_table_template_1_permut_tag, lu_table_template_1>
+            lu_table_template_1_permut_rule = "lu_table_template_1";
+    auto lu_table_template_1_permut_rule_def =
+        (bp::lit("index_1") >> '(' >> qd_vec >> ')' >> ';') ||
+        (bp::lit("variable_1") >> ':' >> name >> ';');
+    BOOST_PARSER_DEFINE_RULES(lu_table_template_1_permut_rule)
+}
+#endif
+
+void github_issue_268()
+{
+#if BOOST_PARSER_USE_CONCEPTS
+    namespace bp = boost::parser;
+    using namespace github_issue_268_;
+    std::string inputstring = "index_1 ( \"1\" ) ; variable_1 : bier;";
+
+    auto const def_result = bp::parse(
+        inputstring, lu_table_template_1_rule_def, bp::blank, bp::trace::off);
+    std::cout << "seq_parser generates this type:\n"
+              << typeid(def_result.value()).name() << std::endl;
+    BOOST_TEST(def_result);
+
+    auto const permut_def_result = bp::parse(
+        inputstring,
+        lu_table_template_1_permut_rule_def,
+        bp::blank,
+        bp::trace::off);
+    std::cout << "permut_parser generates this type:\n"
+              << typeid(permut_def_result.value()).name() << std::endl;
+    BOOST_TEST(permut_def_result);
+
+    auto const result = bp::parse(
+        inputstring, lu_table_template_1_rule, bp::blank, bp::trace::off);
+    std::cout << "seq_parser in rule generates this type:\n"
+              << typeid(result.value()).name() << std::endl;
+    BOOST_TEST(result);
+
+    auto const permut_result = bp::parse(
+        inputstring,
+        lu_table_template_1_permut_rule,
+        bp::blank,
+        bp::trace::off);
+    std::cout << "permut_parser generates this type:\n"
+              << typeid(permut_result.value()).name() << std::endl;
+    BOOST_TEST(permut_result);
+#endif
+}
+
+void github_issue_279()
+{
+    namespace bp = boost::parser;
+
+    {
+        constexpr auto condition_clause =
+            bp::lit(U"while") > bp::lit(U"someexpression") >> bp::attr(true);
+
+        constexpr auto do_statement =
+            bp::lexeme[bp::lit(U"do") >> &bp::ws] > -condition_clause > bp::eol;
+
+        auto const result =
+            bp::parse(U"do\n", do_statement, bp::blank, bp::trace::off);
+        BOOST_TEST(result);
+        std::optional<bool> const & condition = result.value();
+        BOOST_TEST(!condition.has_value());
+    }
+
+    {
+        constexpr auto condition_clause =
+            bp::lit(U"while") > bp::lit(U"someexpression") >> bp::attr(true);
+
+        constexpr auto do_statement_reverse =
+            -condition_clause > bp::lexeme[bp::lit(U"do") >> &bp::ws] > bp::eol;
+
+        auto const result =
+            bp::parse(U"do\n", do_statement_reverse, bp::blank, bp::trace::off);
+        BOOST_TEST(result);
+        std::optional<bool> const & condition = result.value();
+        BOOST_TEST(!condition.has_value());
+    }
+}
+
+namespace github_issue_285_ {
+    namespace bp = boost::parser;
+
+    struct Content
+    {
+        ~Content()
+        {
+            int setbreakpointhere = 0;
+            (void)setbreakpointhere;
+        }
+    };
+    constexpr bp::rule<struct content_tag, std::shared_ptr<Content>> content =
+        "content";
+    constexpr auto content_action = [](auto & ctx) {
+        std::shared_ptr<Content> & result = _val(ctx);
+        result = std::make_shared<Content>();
+    };
+    constexpr auto content_def =
+        (bp::lit(U"content") >> bp::eol)[content_action];
+    BOOST_PARSER_DEFINE_RULES(content);
+}
+
+void github_issue_285()
+{
+    using namespace github_issue_285_;
+    namespace bp = boost::parser;
+
+    constexpr auto prolog = bp::lit(U"prolog") >> bp::eol;
+
+    constexpr auto epilog =
+        bp::no_case[bp::lexeme[bp::lit(U"epi") >> bp::lit(U"log")]] >> bp::eol;
+
+    constexpr auto full_parser = prolog >> content >> epilog;
+
+    std::string teststring =
+        "prolog\n"
+        "content\n"
+        "epilog\n";
+
+    // "content" produces a shared_ptr with the result.
+    // The "epilog" parser must not delete the result.
+
+    auto const result = bp::parse(teststring, full_parser, bp::blank);
+    BOOST_TEST(result);
+    BOOST_TEST(result.value().get() != nullptr);
+}
+
+void github_pr_290()
+{
+    namespace bp = boost::parser;
+
+    auto const pTest = bp::lit("TEST:") > -bp::quoted_string;
+
+    auto result = bp::parse("TEST: \"foo\"", pTest, bp::blank);
+    BOOST_TEST(result);
+    BOOST_TEST(*result == "foo");
+}
+
+namespace github_issue_294_ {
+    namespace bp = boost::parser;
+    struct Foo
+    {};
+    constexpr bp::rule<struct foo_parser_tag, std::shared_ptr<Foo>> foo_parser =
+        "foo_parser";
+    constexpr auto foo_parser_action = [](auto & ctx) {
+        std::shared_ptr<Foo> & val = _val(ctx);
+        val = std::shared_ptr<Foo>(new Foo{});
+    };
+    constexpr auto foo_parser_def = bp::eps[foo_parser_action];
+    struct Bar
+    {
+        std::shared_ptr<Foo> foo;
+    };
+    constexpr bp::rule<struct bar_parser_tag, std::shared_ptr<Bar>> bar_parser =
+        "bar_parser";
+    constexpr auto bar_parser_action = [](auto & ctx) {
+        std::shared_ptr<Bar> & val = _val(ctx);
+        val = std::shared_ptr<Bar>(new Bar{});
+        std::optional<std::shared_ptr<Foo>> & attr = _attr(ctx);
+        if (attr) {
+            val->foo = attr.value();
+        }
+    };
+    constexpr auto bar_parser_def =
+        (bp::lit("(") > -foo_parser > bp::lit(")"))[bar_parser_action];
+
+    BOOST_PARSER_DEFINE_RULES(bar_parser, foo_parser);
+}
+
+void github_issue_294()
+{
+    namespace bp = boost::parser;
+    using namespace github_issue_294_;
+
+    bp::parse("()", bar_parser, bp::blank);
+}
+
+namespace github_pr_297_ {
+    namespace bp = boost::parser;
+    constexpr auto bar_required_f = [](auto& ctx) -> bool {
+        const bool& argument = bp::_p<0>(ctx);
+        return argument;
+    };
+    constexpr bp::rule<struct foobar_parser_tag> foobar_parser = "foobar_parser";
+    constexpr auto foobar_parser_def =
+        bp::lit("foo")
+        >> bp::switch_(bar_required_f)
+            (true, bp::lit("bar"))
+            (false, -bp::lit("bar"));
+    BOOST_PARSER_DEFINE_RULES(foobar_parser);
+}
+
+void github_pr_297()
+{
+    namespace bp = boost::parser;
+    using namespace github_pr_297_;
+
+    {
+        const bool bar_required = true;
+        const bool result =
+            bp::parse("foo bar", foobar_parser.with(bar_required), bp::blank);
+        BOOST_TEST(result);
+    }
+    {
+        const bool bar_required = true;
+        const bool result =
+            bp::parse("foo", foobar_parser.with(bar_required), bp::blank);
+        BOOST_TEST(!result);
+    }
+    {
+        const bool bar_required = false;
+        const bool result =
+            bp::parse("foo bar", foobar_parser.with(bar_required), bp::blank);
+        BOOST_TEST(result);
+    }
+    {
+        const bool bar_required = false;
+        const bool result =
+            bp::parse("foo", foobar_parser.with(bar_required), bp::blank);
+        BOOST_TEST(result);
+    }
+}
+
 
 int main()
 {
@@ -246,5 +606,14 @@ int main()
     github_issue_78();
     github_issue_90();
     github_issue_125();
+    github_issue_209();
+    github_issue_223();
+    github_issue_248();
+    github_issue_268();
+    github_issue_279();
+    github_issue_285();
+    github_pr_290();
+    github_issue_294();
+    github_pr_297();
     return boost::report_errors();
 }

test/lexer.cpp

@@ -1,569 +0,0 @@
-/**
- *   Copyright (C) 2024 T. Zachary Laine
- *
- *   Distributed under the Boost Software License, Version 1.0. (See
- *   accompanying file LICENSE_1_0.txt or copy at
- *   http://www.boost.org/LICENSE_1_0.txt)
- */
-#define BOOST_PARSER_TESTING
-#include <boost/parser/lexer.hpp>
-#include <boost/parser/parser.hpp>
-
-#include <boost/parser/transcode_view.hpp>
-
-#include "ill_formed.hpp"
-
-#include <boost/core/lightweight_test.hpp>
-#include <boost/container/small_vector.hpp>
-
-#include <deque>
-
-
-namespace bp = boost::parser;
-
-enum class my_tokens { ws, foo, bar, baz };
-
-int main()
-{
-    // formation of token_specs
-    {
-        auto const token_spec = bp::token_spec<"foo", 12>;
-
-        bp::token_spec_t<"foo", 12, bp::string_view_tag, 10>
-            token_spec_explicit;
-        static_assert(std::same_as<
-                      decltype(token_spec.parser_)::token_spec,
-                      decltype(token_spec_explicit)>);
-    }
-    {
-        auto const token_spec = bp::token_spec<"foo", my_tokens::foo>;
-
-        bp::token_spec_t<"foo", my_tokens::foo, bp::string_view_tag, 10>
-            token_spec_explicit;
-        static_assert(std::same_as<
-                      decltype(token_spec.parser_)::token_spec,
-                      decltype(token_spec_explicit)>);
-    }
-    {
-        auto const token_spec = bp::token_spec<"bar", my_tokens::bar>;
-
-        bp::token_spec_t<"bar", my_tokens::bar, bp::string_view_tag, 10>
-            token_spec_explicit;
-        static_assert(std::same_as<
-                      decltype(token_spec.parser_)::token_spec,
-                      decltype(token_spec_explicit)>);
-    }
-    {
-        auto const token_spec = bp::token_spec<"foo", 12, int, 2>;
-
-        bp::token_spec_t<"foo", 12, int, 2> token_spec_explicit;
-        static_assert(std::same_as<
-                      decltype(token_spec.parser_)::token_spec,
-                      decltype(token_spec_explicit)>);
-    }
-    {
-        auto const token_spec = bp::token_spec<"foo", 12>;
-
-        bp::token_spec_t<"foo", 12, bp::string_view_tag, 10>
-            token_spec_explicit;
-        static_assert(std::same_as<
-                      decltype(token_spec.parser_)::token_spec,
-                      decltype(token_spec_explicit)>);
-    }
-    {
-        auto const token_spec = bp::token_spec<"foo", 12, unsigned int, 8>;
-
-        bp::token_spec_t<"foo", 12, unsigned int, 8> token_spec_explicit;
-        static_assert(std::same_as<
-                      decltype(token_spec.parser_)::token_spec,
-                      decltype(token_spec_explicit)>);
-    }
-    {
-        auto const token_spec = bp::token_spec<"foo", 12, short>;
-
-        bp::token_spec_t<"foo", 12, short, 10> token_spec_explicit;
-        static_assert(std::same_as<
-                      decltype(token_spec.parser_)::token_spec,
-                      decltype(token_spec_explicit)>);
-    }
-    {
-        auto const token_spec = bp::token_spec<"foo", 12, float>;
-
-        bp::token_spec_t<"foo", 12, float, 10> token_spec_explicit;
-        static_assert(std::same_as<
-                      decltype(token_spec.parser_)::token_spec,
-                      decltype(token_spec_explicit)>);
-    }
-    {
-        auto const token_spec = bp::token_spec<"foo", 12, double>;
-
-        bp::token_spec_t<"foo", 12, double, 10> token_spec_explicit;
-        static_assert(std::same_as<
-                      decltype(token_spec.parser_)::token_spec,
-                      decltype(token_spec_explicit)>);
-    }
-
-    // making lexers
-    {
-        auto const lexer = bp::lexer<char, my_tokens> |
-                           bp::token_spec<"foo", my_tokens::foo> |
-                           bp::token_spec<"bar", my_tokens::bar> |
-                           bp::token_spec<"baz", my_tokens::baz>;
-
-        // +1 because of the 0-group
-        static_assert(decltype(lexer)::size() == 3 + 1);
-        static_assert(std::same_as<decltype(lexer)::id_type, my_tokens>);
-    }
-    {
-        auto const lexer = bp::lexer<char, my_tokens> | bp::token_chars<'='>;
-
-        static_assert(decltype(lexer)::size() == 1 + 1);
-        static_assert(std::same_as<decltype(lexer)::id_type, my_tokens>);
-    }
-    {
-        auto const lexer = bp::lexer<char, my_tokens> | bp::token_chars<'='> |
-                           bp::token_spec<"foo", my_tokens::foo> |
-                           bp::token_spec<"bar", my_tokens::bar> |
-                           bp::token_spec<"baz", my_tokens::baz>;
-
-        static_assert(decltype(lexer)::size() == 4 + 1);
-        static_assert(std::same_as<decltype(lexer)::id_type, my_tokens>);
-    }
-    {
-        auto const lexer =
-            bp::lexer<char, my_tokens> | bp::token_spec<"foo", my_tokens::foo> |
-            bp::token_spec<"bar", my_tokens::bar> |
-            bp::token_spec<"baz", my_tokens::baz> | bp::token_chars<'='>;
-
-        static_assert(decltype(lexer)::size() == 4 + 1);
-        static_assert(std::same_as<decltype(lexer)::id_type, my_tokens>);
-    }
-    {
-        auto const lexer = bp::lexer<char, my_tokens> | bp::token_chars<
-                                                            '=',
-                                                            '+',
-                                                            '-',
-                                                            '!',
-                                                            '?',
-                                                            ':',
-                                                            '.',
-                                                            ',',
-                                                            '(',
-                                                            ')',
-                                                            '[',
-                                                            ']',
-                                                            '{',
-                                                            '}',
-                                                            '@',
-                                                            ';'>;
-
-        static_assert(decltype(lexer)::size() == 16 + 1);
-        static_assert(std::same_as<decltype(lexer)::id_type, my_tokens>);
-    }
-#if 0 // This is a test of whether the escapes work for every possible char
-      // value accepted by detail::token_chars_spec.  This takes a long time and
-      // really only needs to happen once.
-    {
-        auto const lexer = bp::lexer<char, my_tokens> | bp::token_chars<
-                                                              char(0),
-                                                              char(1),
-                                                              char(2),
-                                                              char(3),
-                                                              char(4),
-                                                              char(5),
-                                                              char(6),
-                                                              char(7),
-                                                              char(8),
-                                                              char(9),
-                                                              char(10),
-                                                              char(11),
-                                                              char(12),
-                                                              char(13),
-                                                              char(14),
-                                                              char(15),
-                                                              char(16),
-                                                              char(17),
-                                                              char(18),
-                                                              char(19),
-                                                              char(20),
-                                                              char(21),
-                                                              char(22),
-                                                              char(23),
-                                                              char(24),
-                                                              char(25),
-                                                              char(26),
-                                                              char(27),
-                                                              char(28),
-                                                              char(29),
-                                                              char(30),
-                                                              char(31),
-                                                              char(32),
-                                                              char(33),
-                                                              char(34),
-                                                              char(35),
-                                                              char(36),
-                                                              char(37),
-                                                              char(38),
-                                                              char(39),
-                                                              char(40),
-                                                              char(41),
-                                                              char(42),
-                                                              char(43),
-                                                              char(44),
-                                                              char(45),
-                                                              char(46),
-                                                              char(47),
-                                                              char(48),
-                                                              char(49),
-                                                              char(50),
-                                                              char(51),
-                                                              char(52),
-                                                              char(53),
-                                                              char(54),
-                                                              char(55),
-                                                              char(56),
-                                                              char(57),
-                                                              char(58),
-                                                              char(59),
-                                                              char(60),
-                                                              char(61),
-                                                              char(62),
-                                                              char(63),
-                                                              char(64),
-                                                              char(65),
-                                                              char(66),
-                                                              char(67),
-                                                              char(68),
-                                                              char(69),
-                                                              char(70),
-                                                              char(71),
-                                                              char(72),
-                                                              char(73),
-                                                              char(74),
-                                                              char(75),
-                                                              char(76),
-                                                              char(77),
-                                                              char(78),
-                                                              char(79),
-                                                              char(80),
-                                                              char(81),
-                                                              char(82),
-                                                              char(83),
-                                                              char(84),
-                                                              char(85),
-                                                              char(86),
-                                                              char(87),
-                                                              char(88),
-                                                              char(89),
-                                                              char(90),
-                                                              char(91),
-                                                              char(92),
-                                                              char(93),
-                                                              char(94),
-                                                              char(95),
-                                                              char(96),
-                                                              char(97),
-                                                              char(98),
-                                                              char(99),
-
-                                                              char(100),
-                                                              char(101),
-                                                              char(103),
-                                                              char(102),
-                                                              char(104),
-                                                              char(105),
-                                                              char(106),
-                                                              char(107),
-                                                              char(108),
-                                                              char(109),
-                                                              char(110),
-                                                              char(111),
-                                                              char(112),
-                                                              char(113),
-                                                              char(114),
-                                                              char(115),
-                                                              char(116),
-                                                              char(117),
-                                                              char(118),
-                                                              char(119),
-                                                              char(120),
-                                                              char(121),
-                                                              char(122),
-                                                              char(123),
-                                                              char(124),
-                                                              char(125),
-                                                              char(126),
-                                                              char(127)>;
-    }
-#endif
-
-    {
-        // Mixed UTFs.
-        auto const lexer =
-            bp::lexer<char, my_tokens> | bp::token_spec<"foo", my_tokens::foo> |
-            bp::token_spec<u"bar", my_tokens::bar> |
-            bp::token_spec<U"baz", my_tokens::baz> | bp::token_chars<'='>;
-
-        // mutable vs. const token_views + mutable vs. const input views
-        std::string input = "foo = bar";
-        auto mr_mi = input | bp::to_tokens(lexer);
-        auto const cr_mi = input | bp::to_tokens(lexer);
-
-        auto const const_input = input;
-        auto mr_ci = input | bp::to_tokens(lexer);
-        auto const cr_ci = input | bp::to_tokens(lexer);
-
-        using tok_t = bp::token<char>;
-        tok_t const expected[] = {
-            tok_t((int)my_tokens::foo, 0, "foo"),
-            tok_t(bp::character_id, 0, (long long)'='),
-            tok_t((int)my_tokens::bar, 0, "bar")};
-
-        int position = 0;
-
-        position = 0;
-        for (auto tok : mr_mi) {
-            BOOST_TEST(tok == expected[position]);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-
-        position = 0;
-        for (auto tok : cr_mi) {
-            BOOST_TEST(tok == expected[position]);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-
-        position = 0;
-        for (auto tok : mr_ci) {
-            BOOST_TEST(tok == expected[position]);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-
-        position = 0;
-        for (auto tok : cr_ci) {
-            BOOST_TEST(tok == expected[position]);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-    }
-
-    // Check basic plumbing of connecting UTF inputs to CTRE.
-    {
-        auto const lexer =
-            bp::lexer<char, my_tokens> | bp::token_spec<"foo", my_tokens::foo> |
-            bp::token_spec<"bar", my_tokens::bar> |
-            bp::token_spec<"baz", my_tokens::baz> | bp::token_chars<'='>;
-
-        std::string s = "foo = bar";
-        using tok_t = bp::token<char>;
-        tok_t const expected[] = {
-            tok_t((int)my_tokens::foo, 0, "foo"),
-            tok_t(bp::character_id, 0, (long long)'='),
-            tok_t((int)my_tokens::bar, 0, "bar")};
-
-        auto const lexer8 = bp::lexer<char8_t, my_tokens> |
-                            bp::token_spec<"foo", my_tokens::foo> |
-                            bp::token_spec<"bar", my_tokens::bar> |
-                            bp::token_spec<"baz", my_tokens::baz> |
-                            bp::token_chars<'='>;
-
-        std::u8string u8s = u8"foo = bar";
-        using tok8_t = bp::token<char8_t>;
-        tok8_t const expected8[] = {
-            tok8_t((int)my_tokens::foo, 0, u8"foo"),
-            tok8_t(bp::character_id, 0, (long long)'='),
-            tok8_t((int)my_tokens::bar, 0, u8"bar")};
-
-        auto const lexer16 = bp::lexer<char16_t, my_tokens> |
-                             bp::token_spec<"foo", my_tokens::foo> |
-                             bp::token_spec<"bar", my_tokens::bar> |
-                             bp::token_spec<"baz", my_tokens::baz> |
-                             bp::token_chars<'='>;
-
-        std::u16string u16s = u"foo = bar";
-        using tok16_t = bp::token<char16_t>;
-        tok16_t const expected16[] = {
-            tok16_t((int)my_tokens::foo, 0, u"foo"),
-            tok16_t(bp::character_id, 0, (long long)'='),
-            tok16_t((int)my_tokens::bar, 0, u"bar")};
-
-        auto const lexer32 = bp::lexer<char32_t, my_tokens> |
-                             bp::token_spec<"foo", my_tokens::foo> |
-                             bp::token_spec<"bar", my_tokens::bar> |
-                             bp::token_spec<"baz", my_tokens::baz> |
-                             bp::token_chars<'='>;
-
-        std::u32string u32s = U"foo = bar";
-        using tok32_t = bp::token<char32_t>;
-        tok32_t const expected32[] = {
-            tok32_t((int)my_tokens::foo, 0, U"foo"),
-            tok32_t(bp::character_id, 0, (long long)'='),
-            tok32_t((int)my_tokens::bar, 0, U"bar")};
-
-
-        int position = 0;
-
-        position = 0;
-        for (auto tok : s | bp::to_tokens(lexer)) {
-            BOOST_TEST(tok == expected[position]);
-            static_assert(
-                std::
-                    same_as<decltype(tok.get_string_view()), std::string_view>);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-
-        position = 0;
-        for (auto tok : u8s | bp::to_tokens(lexer8)) {
-            BOOST_TEST(tok == expected8[position]);
-            static_assert(std::same_as<
-                          decltype(tok.get_string_view()),
-                          std::u8string_view>);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-
-        position = 0;
-        for (auto tok : u16s | bp::to_tokens(lexer16)) {
-            BOOST_TEST(tok == expected16[position]);
-            static_assert(std::same_as<
-                          decltype(tok.get_string_view()),
-                          std::u16string_view>);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-
-        position = 0;
-        for (auto tok : u32s | bp::to_tokens(lexer32)) {
-            BOOST_TEST(tok == expected32[position]);
-            static_assert(std::same_as<
-                          decltype(tok.get_string_view()),
-                          std::u32string_view>);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-    }
-
-    // no-ws lexer
-    {
-        auto const lexer = bp::lexer<char, my_tokens, bp::no_ws> |
-                           bp::token_spec<"foo", my_tokens::foo> |
-                           bp::token_spec<"bar", my_tokens::bar> |
-                           bp::token_spec<"baz", my_tokens::baz> |
-                           bp::token_chars<'='>;
-
-        std::string s = "foo=bar";
-        using tok_t = bp::token<char>;
-        tok_t const expected[] = {
-            tok_t((int)my_tokens::foo, 0, "foo"),
-            tok_t(bp::character_id, 0, (long long)'='),
-            tok_t((int)my_tokens::bar, 0, "bar")};
-
-        int position = 0;
-        for (auto tok : s | bp::to_tokens(lexer)) {
-            BOOST_TEST(tok == expected[position]);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-    }
-
-    // ws-as-token lexers
-    {
-        auto const lexer = bp::lexer<char, my_tokens, bp::no_ws> |
-                           bp::token_spec<"\\s+", my_tokens::ws> |
-                           bp::token_spec<"foo", my_tokens::foo> |
-                           bp::token_spec<"bar", my_tokens::bar> |
-                           bp::token_spec<"baz", my_tokens::baz> |
-                           bp::token_chars<'='>;
-
-        std::string s = "foo = bar";
-        using tok_t = bp::token<char>;
-        tok_t const expected[] = {
-            tok_t((int)my_tokens::foo, 0, "foo"),
-            tok_t((int)my_tokens::ws, 0, " "),
-            tok_t(bp::character_id, 0, (long long)'='),
-            tok_t((int)my_tokens::ws, 0, " "),
-            tok_t((int)my_tokens::bar, 0, "bar")};
-
-        int position = 0;
-        for (auto tok : s | bp::to_tokens(lexer)) {
-            BOOST_TEST(tok == expected[position]);
-            ++position;
-        }
-        BOOST_TEST(position == (int)std::size(expected));
-    }
-
-    // lexing errors
-    {
-        using namespace std::literals;
-
-        auto const lexer = bp::lexer<char, int> |
-                           bp::token_spec<"foo", 0, float> |
-                           bp::token_spec<"bar", 1, int> |
-                           bp::token_spec<"baz", 2, unsigned short> |
-                           bp::token_spec<"quux", 3, int, 8> |
-                           bp::token_spec<"next", 4, unsigned long long, 16>;
-
-        bool caught_exception = false;
-
-        caught_exception = false;
-        try {
-            for (auto tok : "foo" | bp::to_tokens(lexer)) {
-                (void)tok;
-            }
-        } catch (std::exception const & e) {
-            BOOST_TEST(e.what() == "32-bit floating-point number"sv);
-            caught_exception = true;
-        }
-        BOOST_TEST(caught_exception);
-
-        caught_exception = false;
-        try {
-            for (auto tok : "bar" | bp::to_tokens(lexer)) {
-                (void)tok;
-            }
-        } catch (std::exception const & e) {
-            BOOST_TEST(e.what() == "32-bit signed integer"sv);
-            caught_exception = true;
-        }
-        BOOST_TEST(caught_exception);
-
-        caught_exception = false;
-        try {
-            for (auto tok : "baz" | bp::to_tokens(lexer)) {
-                (void)tok;
-            }
-        } catch (std::exception const & e) {
-            BOOST_TEST(e.what() == "16-bit unsigned integer"sv);
-            caught_exception = true;
-        }
-        BOOST_TEST(caught_exception);
-
-        caught_exception = false;
-        try {
-            for (auto tok : "quux" | bp::to_tokens(lexer)) {
-                (void)tok;
-            }
-        } catch (std::exception const & e) {
-            BOOST_TEST(e.what() == "32-bit, base-8 signed integer"sv);
-            caught_exception = true;
-        }
-        BOOST_TEST(caught_exception);
-
-        caught_exception = false;
-        try {
-            for (auto tok : "next" | bp::to_tokens(lexer)) {
-                (void)tok;
-            }
-        } catch (std::exception const & e) {
-            BOOST_TEST(e.what() == "64-bit, base-16 unsigned integer"sv);
-            caught_exception = true;
-        }
-        BOOST_TEST(caught_exception);
-    }
-
-    return boost::report_errors();
-}

test/lexer_adobe_files.cpp

@@ -1,828 +0,0 @@
-/**
- *   Copyright (C) 2024 T. Zachary Laine
- *
- *   Distributed under the Boost Software License, Version 1.0. (See
- *   accompanying file LICENSE_1_0.txt or copy at
- *   http://www.boost.org/LICENSE_1_0.txt)
- */
-#define BOOST_PARSER_TESTING
-#include <boost/parser/lexer.hpp>
-
-#include <boost/parser/transcode_view.hpp>
-
-#include "ill_formed.hpp"
-#include "adobe_lexer.hpp"
-
-#include <boost/core/lightweight_test.hpp>
-#include <boost/container/small_vector.hpp>
-
-#include <deque>
-
-
-namespace bp = boost::parser;
-
-int main()
-{
-    {
-        static_assert(decltype(adobe_lexer)::size() == 29 + 1);
-        static_assert(
-            std::same_as<decltype(adobe_lexer)::id_type, adobe_tokens>);
-
-        // tokens_view from adobe_lexer
-        {
-            char const input[] = R"(/*
-    Copyright 2005-2007 Adobe Systems Incorporated
-    Distributed under the MIT License (see accompanying file LICENSE_1_0_0.txt
-    or a copy at http://stlab.adobe.com/licenses.html)
-*/
-
-sheet alert_dialog
-{
-output:
-    result <== { dummy_value: 42 };
-})";
-            // first, just make a ctre range
-            {
-                std::string_view const expected[] = {
-                    R"(/*
-    Copyright 2005-2007 Adobe Systems Incorporated
-    Distributed under the MIT License (see accompanying file LICENSE_1_0_0.txt
-    or a copy at http://stlab.adobe.com/licenses.html)
-*/)",
-                    R"(
-
-)",    R"(sheet)",  R"( )",  R"(alert_dialog)",
-                    R"(
-)",    R"({)",
-                    R"(
-)",    R"(output)", R"(:)",
-                    R"(
-    )",    R"(result)", R"( )",  R"(<==)",
-                    R"( )", R"({)",      R"( )",  R"(dummy_value)",
-                    R"(:)", R"( )",      R"(42)", R"( )",
-                    R"(})", R"(;)",
-                    R"(
-)",    R"(})"};
-                auto r = adobe_lexer.regex_range(input);
-                int position = 0;
-                for (auto subrange : r) {
-                    std::string_view sv = subrange;
-                    BOOST_TEST(sv == expected[position]);
-                    ++position;
-                }
-                BOOST_TEST(position == (int)std::size(expected));
-                std::cout << "\n";
-            }
-
-            using tok_t = bp::token<char>;
-            tok_t const expected[] = {
-                tok_t((int)adobe_tokens::lead_comment, 0, R"(/*
-    Copyright 2005-2007 Adobe Systems Incorporated
-    Distributed under the MIT License (see accompanying file LICENSE_1_0_0.txt
-    or a copy at http://stlab.adobe.com/licenses.html)
-*/)"),
-                tok_t((int)adobe_tokens::identifier, 0, "sheet"),
-                tok_t((int)adobe_tokens::identifier, 0, "alert_dialog"),
-                tok_t(bp::character_id, 0, (long long)'{'),
-                tok_t((int)adobe_tokens::identifier, 0, "output"),
-                tok_t(bp::character_id, 0, (long long)':'),
-                tok_t((int)adobe_tokens::identifier, 0, "result"),
-                tok_t((int)adobe_tokens::define, 0, "<=="),
-                tok_t(bp::character_id, 0, (long long)'{'),
-                tok_t((int)adobe_tokens::identifier, 0, "dummy_value"),
-                tok_t(bp::character_id, 0, (long long)':'),
-                tok_t((int)adobe_tokens::number, 0, (long double)42.0),
-                tok_t(bp::character_id, 0, (long long)'}'),
-                tok_t(bp::character_id, 0, (long long)';'),
-                tok_t(bp::character_id, 0, (long long)'}')};
-
-            // make a tokens_view
-            {
-                auto r = bp::tokens_view(input, adobe_lexer);
-                int position = 0;
-                for (auto tok : r) {
-                    BOOST_TEST(tok == expected[position]);
-                    ++position;
-                }
-                BOOST_TEST(position == (int)std::size(expected));
-            }
-
-            // to_tokens range adaptor
-            {
-                int position = 0;
-                for (auto tok : bp::to_tokens(input, adobe_lexer)) {
-                    BOOST_TEST(tok == expected[position]);
-                    ++position;
-                }
-                BOOST_TEST(position == (int)std::size(expected));
-            }
-            {
-                std::string const input_str = input;
-                int position = 0;
-                for (auto tok : bp::to_tokens(input_str, adobe_lexer)) {
-                    BOOST_TEST(tok == expected[position]);
-                    ++position;
-                }
-                BOOST_TEST(position == (int)std::size(expected));
-            }
-            {
-                int position = 0;
-                for (auto tok :
-                     std::string(input) | bp::to_tokens(adobe_lexer)) {
-                    BOOST_TEST(tok == expected[position]);
-                    ++position;
-                }
-                BOOST_TEST(position == (int)std::size(expected));
-            }
-
-            // using external caches
-            {
-                std::vector<bp::token<char>> cache;
-                int position = 0;
-                for (auto tok :
-                     bp::to_tokens(input, adobe_lexer, std::ref(cache))) {
-                    BOOST_TEST(tok == expected[position]);
-                    ++position;
-                }
-                BOOST_TEST(position == (int)std::size(expected));
-            }
-            {
-                boost::container::small_vector<bp::token<char>, 10> cache;
-                int position = 0;
-                for (auto tok :
-                     input | bp::to_tokens(adobe_lexer, std::ref(cache))) {
-                    BOOST_TEST(tok == expected[position]);
-                    ++position;
-                }
-                BOOST_TEST(position == (int)std::size(expected));
-            }
-
-            {
-                char const large_input[] = R"(/*
-    Copyright 2005-2007 Adobe Systems Incorporated
-    Distributed under the MIT License (see accompanying file LICENSE_1_0_0.txt
-    or a copy at http://stlab.adobe.com/licenses.html)
-*/
-
-sheet image_size
-{
-input:
-    original_width          : 1600;
-    original_height         : 1200;
-    original_resolution     : 300;
-
-constant:
-    original_doc_width      : original_width / original_resolution;
-    original_doc_height     : original_height / original_resolution;
-
-interface:
-    resample                : true;
-    unlink  constrain       : true <== resample ? constrain : true;
-    unlink  scale_styles    : true <== resample && constrain ? scale_styles : false;
-
-    resample_method         : @bicubic;
-
-    dim_width_pixels        : original_width <== resample ? round(dim_width_pixels) : original_width;
-    dim_width_percent       : 100 <== resample ? dim_width_percent : 100;
-
-    dim_height_pixels       : original_height <== resample ? round(dim_height_pixels) : original_height;
-    dim_height_percent      : 100 <== resample ? dim_height_percent : 100;
-
-    doc_width_inches        : original_doc_width;
-    doc_width_percent       : 100;
-
-    /*
-        Resolution must be initialized before width and height inches to allow proportions
-        to be constrained.
-    */
-    doc_resolution          : original_resolution;
-    
-    doc_height_inches       : original_doc_height;
-    doc_height_percent      : 100;
-
-    auto_quality            : @draft;
-    
-    screen_lpi; // initialized from doc_resolution
-
-logic:
-    relate {
-        doc_width_inches    <== doc_width_percent * original_doc_width / 100;
-        doc_width_percent   <== doc_width_inches * 100 / original_doc_width;
-    }
-
-    relate {
-        doc_height_inches   <== doc_height_percent * original_doc_height / 100;
-        doc_height_percent  <== doc_height_inches * 100 / original_doc_height;
-    }
-
-    relate {
-        screen_lpi      <== doc_resolution / (auto_quality == @draft ? 1 : (auto_quality == @good ? 1.5 : 2.0));
-        doc_resolution  <== screen_lpi * (auto_quality == @draft ? 1 : (auto_quality == @good ? 1.5 : 2.0));
-    }
-
-    when (resample) relate {
-        dim_width_pixels    <== dim_width_percent * original_width / 100;
-        dim_width_percent   <== dim_width_pixels * 100 / original_width;
-    }
-
-    when (resample) relate {
-        dim_height_pixels   <== dim_height_percent * original_height / 100;
-        dim_height_percent  <== dim_height_pixels * 100 / original_height;
-    }
-
-    when (resample) relate {
-        doc_width_inches    <== dim_width_pixels / doc_resolution;
-        dim_width_pixels    <== doc_width_inches * doc_resolution;
-        doc_resolution      <== dim_width_pixels / doc_width_inches;
-    }
-
-    when (resample) relate {
-        doc_height_inches   <== dim_height_pixels / doc_resolution;
-        dim_height_pixels   <== doc_height_inches * doc_resolution;
-        doc_resolution      <== dim_height_pixels / doc_height_inches;
-    }
-
-    when (!resample) relate {
-        doc_resolution      <== original_width / doc_width_inches;
-        doc_width_inches    <== original_width / doc_resolution;
-    }
-
-    when (!resample) relate {
-        doc_resolution      <== original_height / doc_height_inches;
-        doc_height_inches   <== original_height / doc_resolution;
-    }
-
-    when (constrain && resample) relate {
-        dim_width_percent       <== dim_height_percent;
-        dim_height_percent      <== dim_width_percent;
-    }
-
- output:
-    byte_count      <== dim_width_pixels * dim_height_pixels * 32;
-
-    result          <== resample ?  {
-                                        command:            @resize_image,
-                                        width:              dim_width_pixels,
-                                        height:             dim_height_pixels,
-                                        resolution:         doc_resolution,
-                                        scale_styles:       scale_styles,
-                                        resample_method:    resample_method
-                                    } : {
-                                        command:            @set_resolution,
-                                        resolution:         doc_resolution
-                                    };
-
- invariant:
-    width_max       <== dim_width_pixels <= 300000;
-    height_max      <== dim_height_pixels <= 300000;
-}
-)";
-
-                tok_t const expected[] = {
-                    tok_t((int)adobe_tokens::lead_comment, 0, R"(/*
-    Copyright 2005-2007 Adobe Systems Incorporated
-    Distributed under the MIT License (see accompanying file LICENSE_1_0_0.txt
-    or a copy at http://stlab.adobe.com/licenses.html)
-*/)"),
-                    tok_t((int)adobe_tokens::identifier, 0, "sheet"),
-                    tok_t((int)adobe_tokens::identifier, 0, "image_size"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, 0, "input"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)1600.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)1200.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier,
-                        0,
-                        "original_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)300.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "constant"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "original_doc_width"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t(
-                        (int)adobe_tokens::identifier,
-                        0,
-                        "original_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier,
-                        0,
-                        "original_doc_height"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t(
-                        (int)adobe_tokens::identifier,
-                        0,
-                        "original_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "interface"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 0, 1ll),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "unlink"),
-                    tok_t((int)adobe_tokens::identifier, 0, "constrain"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 0, 1ll),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, 0, "constrain"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 0, 1ll),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "unlink"),
-                    tok_t((int)adobe_tokens::identifier, 0, "scale_styles"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 0, 1ll),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)adobe_tokens::and_, 0, "&&"),
-                    tok_t((int)adobe_tokens::identifier, 0, "constrain"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, 0, "scale_styles"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::keyword_true_false, 0, 0ll),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample_method"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)bp::character_id, 0, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, 0, "bicubic"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, 0, "round"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t((int)adobe_tokens::identifier, 0, "round"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "original_doc_width"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_width_percent"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::lead_comment, 0, R"(/*
-        Resolution must be initialized before width and height inches to allow proportions
-        to be constrained.
-    */)"),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t(
-                        (int)adobe_tokens::identifier,
-                        0,
-                        "original_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t(
-                        (int)adobe_tokens::identifier,
-                        0,
-                        "original_doc_height"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_percent"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)bp::character_id, 0, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, 0, "draft"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "screen_lpi"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::trail_comment,
-                        0,
-                        "// initialized from doc_resolution"),
-                    tok_t((int)adobe_tokens::identifier, 0, "logic"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_width_percent"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "original_doc_width"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_width_percent"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "original_doc_width"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_percent"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t(
-                        (int)adobe_tokens::identifier,
-                        0,
-                        "original_doc_height"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_percent"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t(
-                        (int)adobe_tokens::identifier,
-                        0,
-                        "original_doc_height"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, 0, "screen_lpi"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
-                    tok_t((int)adobe_tokens::eq_op, 0, "=="),
-                    tok_t((int)bp::character_id, 0, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, 0, "draft"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)1.0),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
-                    tok_t((int)adobe_tokens::eq_op, 0, "=="),
-                    tok_t((int)bp::character_id, 0, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, 0, "good"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)1.5),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)2.0),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "screen_lpi"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
-                    tok_t((int)adobe_tokens::eq_op, 0, "=="),
-                    tok_t((int)bp::character_id, 0, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, 0, "draft"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)1.0),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "auto_quality"),
-                    tok_t((int)adobe_tokens::eq_op, 0, "=="),
-                    tok_t((int)bp::character_id, 0, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, 0, "good"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)1.5),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::number, 0, (long double)2.0),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "when"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "when"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)adobe_tokens::number, 0, (long double)100.0),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "when"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "when"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "when"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)bp::character_id, 0, (long long)'!'),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_width_inches"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_width"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "when"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)bp::character_id, 0, (long long)'!'),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "doc_height_inches"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "original_height"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "/"),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "when"),
-                    tok_t((int)bp::character_id, 0, (long long)'('),
-                    tok_t((int)adobe_tokens::identifier, 0, "constrain"),
-                    tok_t((int)adobe_tokens::and_, 0, "&&"),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)')'),
-                    tok_t((int)adobe_tokens::identifier, 0, "relate"),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_percent"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_width_percent"),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)adobe_tokens::identifier, 0, "output"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "byte_count"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::mul_op, 0, "*"),
-                    tok_t((int)adobe_tokens::number, 0, (long double)32.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "result"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample"),
-                    tok_t((int)bp::character_id, 0, (long long)'?'),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, 0, "command"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)bp::character_id, 0, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, 0, "resize_image"),
-                    tok_t((int)bp::character_id, 0, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, 0, "width"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)bp::character_id, 0, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, 0, "height"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)bp::character_id, 0, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, 0, "resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, 0, "scale_styles"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "scale_styles"),
-                    tok_t((int)bp::character_id, 0, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample_method"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "resample_method"),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)bp::character_id, 0, (long long)'{'),
-                    tok_t((int)adobe_tokens::identifier, 0, "command"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)bp::character_id, 0, (long long)'@'),
-                    tok_t((int)adobe_tokens::identifier, 0, "set_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)','),
-                    tok_t((int)adobe_tokens::identifier, 0, "resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "doc_resolution"),
-                    tok_t((int)bp::character_id, 0, (long long)'}'),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "invariant"),
-                    tok_t((int)bp::character_id, 0, (long long)':'),
-                    tok_t((int)adobe_tokens::identifier, 0, "width_max"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t((int)adobe_tokens::identifier, 0, "dim_width_pixels"),
-                    tok_t((int)adobe_tokens::rel_op, 0, "<"),
-                    tok_t((int)bp::character_id, 0, (long long)'='),
-                    tok_t((int)adobe_tokens::number, 0, (long double)300000.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)adobe_tokens::identifier, 0, "height_max"),
-                    tok_t((int)adobe_tokens::define, 0, "<=="),
-                    tok_t(
-                        (int)adobe_tokens::identifier, 0, "dim_height_pixels"),
-                    tok_t((int)adobe_tokens::rel_op, 0, "<"),
-                    tok_t((int)bp::character_id, 0, (long long)'='),
-                    tok_t((int)adobe_tokens::number, 0, (long double)300000.0),
-                    tok_t((int)bp::character_id, 0, (long long)';'),
-                    tok_t((int)bp::character_id, 0, (long long)'}')};
-
-                int position = 0;
-                for (auto tok :
-                     std::string(large_input) | bp::to_tokens(adobe_lexer)) {
-                    BOOST_TEST(tok == expected[position]);
-                    if (tok != expected[position]) {
-                        std::cout << "At pos=" << position << ": got " << tok
-                                  << " expected " << expected[position] << "\n";
-                    }
-                    ++position;
-                }
-                BOOST_TEST(position == (int)std::size(expected));
-            }
-        }
-    }
-
-    return boost::report_errors();
-}

test/lexer_and_parser.cpp

@@ -1,237 +0,0 @@
-/**
- *   Copyright (C) 2024 T. Zachary Laine
- *
- *   Distributed under the Boost Software License, Version 1.0. (See
- *   accompanying file LICENSE_1_0.txt or copy at
- *   http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#define BOOST_PARSER_TESTING
-//[ tokens_basics_headers
-#include <boost/parser/lexer.hpp>
-#include <boost/parser/parser.hpp>
-//]
-
-#include <boost/core/lightweight_test.hpp>
-
-#include "adobe_lexer.hpp"
-
-
-namespace bp = boost::parser;
-
-int main()
-{
-    // Minimal test; just instantiate the member functions, without involving
-    // the parse() API.
-    {
-        bp::token<char> tokens[1] = {};
-        auto p = bp::token_spec<"12", 12, int>;
-        auto first = std::begin(tokens);
-        auto const last = std::end(tokens);
-
-        bp::detail::nope globals;
-        bp::default_error_handler error_handler;
-
-        // From parse_impl().
-        bool success = true;
-        int trace_indent = 0;
-        bp::detail::symbol_table_tries_t symbol_table_tries;
-        bp::detail::pending_symbol_table_operations_t
-            pending_symbol_table_operations;
-        bp::detail::scoped_apply_pending_symbol_table_operations apply_pending(
-            pending_symbol_table_operations);
-        auto context = bp::detail::make_context<false, false>(
-            first,
-            last,
-            success,
-            trace_indent,
-            error_handler,
-            globals,
-            symbol_table_tries,
-            pending_symbol_table_operations);
-        auto const flags = bp::detail::flags::gen_attrs;
-
-        std::optional<int> result =
-            p(first, last, context, bp::ws, flags, success);
-        (void)result;
-    }
-
-    // Minimal tests of building parsers from token_parser and token_spec.
-    {
-        auto parser1 = true_false(true);
-        auto parser2 = true_false(false);
-        (void)parser1;
-        (void)parser2;
-    }
-    {
-        auto parser = identifier("foo") >> '=' >> true_false >> ';';
-        (void)parser;
-    }
-
-    // Minimal tests of using a lexer and parser together.
-    {
-        auto parser = identifier("foo") >> '=' >> true_false >> ';';
-        auto r = "some input" | bp::to_tokens(adobe_lexer);
-        auto result = bp::parse(r, parser);
-        BOOST_TEST(!result);
-
-        static_assert(!std::same_as<
-                      std::remove_cvref_t<
-                          decltype(bp::detail::tokens_view_or_nope(r))>,
-                      bp::detail::nope>);
-
-        auto const & cr = r;
-        static_assert(!std::same_as<
-                      std::remove_cvref_t<
-                          decltype(bp::detail::tokens_view_or_nope(cr))>,
-                      bp::detail::nope>);
-    }
-    {
-        auto parser = identifier >> '=' >> true_false >> ';';
-        auto r = "foo = false;" | bp::to_tokens(adobe_lexer);
-        auto result = bp::parse(r, parser);
-        BOOST_TEST(result);
-        BOOST_TEST(std::get<0>(*result) == "foo");
-        BOOST_TEST(std::get<1>(*result) == false);
-    }
-
-    // Test the use of an external token cache.
-    {
-        auto parser = identifier >> '=' >> true_false >> ';';
-        std::vector<bp::token<char>> cache;
-        auto r = "foo = false;" | bp::to_tokens(adobe_lexer, std::ref(cache));
-        auto result = bp::parse(r, parser);
-        BOOST_TEST(std::get<0>(*result) == "foo");
-        BOOST_TEST(std::get<1>(*result) == false);
-        BOOST_TEST(cache.size() == 4u);
-    }
-
-    // Test the clearing of the token cache at expectation points.
-    {
-        auto parser = identifier >> '=' > true_false >> ';';
-        std::vector<bp::token<char>> cache;
-        auto r = "foo = false;" | bp::to_tokens(adobe_lexer, std::ref(cache));
-        auto result = bp::parse(r, parser);
-        BOOST_TEST(std::get<0>(*result) == "foo");
-        BOOST_TEST(std::get<1>(*result) == false);
-        BOOST_TEST(cache.size() == 2u);
-    }
-
-    // doc examples
-    // clang-format off
-    {
-    //[ tokens_basics_lexer
-    auto const foo = bp::token_spec<"foo", 0>;
-    auto const bar = bp::token_spec<"b.r", 1>;
-    auto const baz = bp::token_spec<"b.z", 2>;
-
-    auto const lexer = bp::lexer<char, int> | foo | bar | baz;
-    //]
-
-    //[ tokens_basics_input_range
-    auto r = "foobazbar" | bp::to_tokens(lexer);
-    //]
-
-    //[ tokens_basics_parser
-    auto parser = foo >> baz >> bar;
-    //]
-
-    //[ tokens_basics_parse
-    auto result = bp::parse(r, parser);
-    assert(result);
-    assert(std::get<0>(*result) == "foo");
-    assert(std::get<1>(*result) == "baz");
-    assert(std::get<2>(*result) == "bar");
-    //]
-    }
-
-    {
-    //[ tokens_attrs
-    constexpr auto true_false = bp::token_spec<"true|false", 0, bool>;
-    constexpr auto identifier = bp::token_spec<"[a-zA-Z]\\w*", 1>;
-    constexpr auto number = bp::token_spec<"\\d+(?:\\.\\d*)?", 2, double>;
-    //]
-    (void)true_false;
-    (void)identifier;
-    (void)number;
-    }
-
-    {
-    //[ tokens_token_char
-    constexpr auto true_false = bp::token_spec<"true|false", 0, bool>;
-    constexpr auto identifier = bp::token_spec<"[a-zA-Z]\\w*", 1>;
-
-    constexpr auto lexer =
-        bp::lexer<char, int> | true_false | identifier | bp::token_chars<'=', ';'>;
-
-    auto parser = identifier >> '=' >> true_false >> ';';
-    auto r = "foo = false;" | bp::to_tokens(lexer);
-    auto result = bp::parse(r, parser);
-    assert(result);
-    assert(std::get<0>(*result) == "foo");
-    assert(std::get<1>(*result) == false);
-    //]
-    }
-
-    {
-    //[ tokens_caching_simple
-    constexpr auto true_false = bp::token_spec<"true|false", 0, bool>;
-    constexpr auto identifier = bp::token_spec<"[a-zA-Z]\\w*", 1>;
-
-    constexpr auto lexer =
-        bp::lexer<char, int> | true_false | identifier | bp::token_chars<'=', ';'>;
-
-    auto parser = identifier >> '=' >> true_false >> ';';
-    std::vector<bp::token<char>> cache;
-    auto r = "foo = false;" | bp::to_tokens(lexer, std::ref(cache));
-    auto result = bp::parse(r, parser);
-    assert(result);
-    assert(std::get<0>(*result) == "foo");
-    assert(std::get<1>(*result) == false);
-    assert(cache.size() == 4u);
-    //]
-    }
-
-    {
-    constexpr auto true_false = bp::token_spec<"true|false", 0, bool>;
-    constexpr auto identifier = bp::token_spec<"[a-zA-Z]\\w*", 1>;
-
-    constexpr auto lexer =
-        bp::lexer<char, int> | true_false | identifier | bp::token_chars<'=', ';'>;
-
-    //[ tokens_caching_expectation_point
-    auto parser = identifier >> '=' > true_false >> ';';
-    std::vector<bp::token<char>> cache;
-    auto r = "foo = false;" | bp::to_tokens(lexer, std::ref(cache));
-    auto result = bp::parse(r, parser);
-    assert(result);
-    assert(std::get<0>(*result) == "foo");
-    assert(std::get<1>(*result) == false);
-    assert(cache.size() == 2u);
-    //]
-    }
-
-    {
-    //[ tokens_string_in_character_vs_token_parsing
-    constexpr auto true_false = bp::token_spec<"true|false", 0, bool>;
-    constexpr auto identifier = bp::token_spec<"[a-zA-Z]\\w*", 1>;
-
-    constexpr auto lexer =
-        bp::lexer<char, int> | true_false | identifier | bp::token_chars<'=', ';'>;
-
-    auto parser = bp::string("=;");
-
-    // NOTE: Character parsing here.
-    auto character_parse_result = bp::parse("=;", parser);
-    assert(character_parse_result);
-    assert(*character_parse_result == "=;");
-
-    // NOTE: Token parsing here.
-    auto token_parse_result = bp::parse("=;" | bp::to_tokens(lexer), parser);
-    assert(!token_parse_result);
-    //]
-    }
-    // clang-format on
-
-    return boost::report_errors();
-}

test/lexer_and_parser_api.cpp

@@ -1,184 +0,0 @@
-/**
- *   Copyright (C) 2024 T. Zachary Laine
- *
- *   Distributed under the Boost Software License, Version 1.0. (See
- *   accompanying file LICENSE_1_0.txt or copy at
- *   http://www.boost.org/LICENSE_1_0.txt)
- */
-
-#define BOOST_PARSER_TESTING
-#include <boost/parser/lexer.hpp>
-#include <boost/parser/parser.hpp>
-
-#include <boost/core/lightweight_test.hpp>
-
-
-namespace bp = boost::parser;
-
-constexpr auto true_false = bp::token_spec<"true|false", 0, bool>;
-constexpr auto identifier = bp::token_spec<"[a-zA-Z]\\w*", 1>;
-
-struct tf_tag
-{};
-struct id_tag
-{};
-constexpr bp::callback_rule<tf_tag, bool> callback_true_false = "";
-constexpr bp::callback_rule<id_tag, std::string_view> callback_identifier = "";
-constexpr auto callback_true_false_def = true_false;
-constexpr auto callback_identifier_def = identifier;
-BOOST_PARSER_DEFINE_RULES(callback_true_false, callback_identifier);
-
-struct callbacks
-{
-    void operator()(id_tag, std::string_view sv) const { sv_ = sv; }
-    void operator()(tf_tag, bool b) const { b_ = b; }
-    std::string_view & sv_;
-    bool & b_;
-};
-
-int main()
-{
-    auto assign_bool_parser = identifier >> '=' >> true_false >> ';';
-    auto assign_bool_no_semi_parser = identifier >> '=' >> true_false;
-
-    constexpr auto lexer = bp::lexer<char, int> | true_false | identifier |
-                           bp::token_chars<'=', ';'>;
-    auto r = "foo = false;" | bp::to_tokens(lexer);
-
-    // prefix_parse() w/attr
-    {
-        auto f = r.begin();
-        auto const l = r.end();
-        std::tuple<std::string_view, bool> result;
-        auto success = bp::prefix_parse(f, l, assign_bool_parser, result);
-        BOOST_TEST(success);
-        BOOST_TEST(std::get<0>(result) == "foo");
-        BOOST_TEST(std::get<1>(result) == false);
-    }
-    {
-        auto f = r.begin();
-        auto const l = r.end();
-        std::tuple<std::string_view, bool> result;
-        auto success = bp::prefix_parse(f, l, assign_bool_no_semi_parser, result);
-        BOOST_TEST(success);
-        BOOST_TEST(std::get<0>(result) == "foo");
-        BOOST_TEST(std::get<1>(result) == false);
-        BOOST_TEST(f != l);
-    }
-
-    // parse() w/attr
-    {
-        std::tuple<std::string_view, bool> result;
-        auto success = bp::parse(r, assign_bool_parser, result);
-        BOOST_TEST(success);
-        BOOST_TEST(std::get<0>(result) == "foo");
-        BOOST_TEST(std::get<1>(result) == false);
-    }
-    {
-        constexpr auto lexer = bp::lexer<char8_t, int> | true_false |
-                               identifier | bp::token_chars<'=', ';'>;
-        auto r8 = u8"foo = false;" | bp::to_tokens(lexer);
-
-        std::tuple<std::u8string_view, bool> result;
-        auto success = bp::parse(r8, assign_bool_parser, result);
-        BOOST_TEST(success);
-        BOOST_TEST(std::get<0>(result) == u8"foo");
-        BOOST_TEST(std::get<1>(result) == false);
-    }
-    {
-        constexpr auto lexer = bp::lexer<char16_t, int> | true_false |
-                               identifier | bp::token_chars<'=', ';'>;
-        auto r16 = u"foo = false;" | bp::to_tokens(lexer);
-
-        std::tuple<std::u16string_view, bool> result;
-        auto success = bp::parse(r16, assign_bool_parser, result);
-        BOOST_TEST(success);
-        BOOST_TEST(std::get<0>(result) == u"foo");
-        BOOST_TEST(std::get<1>(result) == false);
-    }
-    {
-        constexpr auto lexer = bp::lexer<char32_t, int> | true_false |
-                               identifier | bp::token_chars<'=', ';'>;
-        auto r32 = U"foo = false;" | bp::to_tokens(lexer);
-
-        std::tuple<std::u32string_view, bool> result;
-        auto success = bp::parse(r32, assign_bool_parser, result);
-        BOOST_TEST(success);
-        BOOST_TEST(std::get<0>(result) == U"foo");
-        BOOST_TEST(std::get<1>(result) == false);
-    }
-
-    // prefix_parse() no attr
-    {
-        auto f = r.begin();
-        auto const l = r.end();
-        auto result = bp::prefix_parse(f, l, assign_bool_parser);
-        BOOST_TEST(result);
-        BOOST_TEST(std::get<0>(*result) == "foo");
-        BOOST_TEST(std::get<1>(*result) == false);
-    }
-    {
-        auto f = r.begin();
-        auto const l = r.end();
-        auto result = bp::prefix_parse(f, l, assign_bool_no_semi_parser);
-        BOOST_TEST(result);
-        BOOST_TEST(std::get<0>(*result) == "foo");
-        BOOST_TEST(std::get<1>(*result) == false);
-        BOOST_TEST(f != l);
-    }
-
-    // parse() no attr
-    {
-        auto result = bp::parse(r, assign_bool_parser);
-        BOOST_TEST(result);
-        BOOST_TEST(std::get<0>(*result) == "foo");
-        BOOST_TEST(std::get<1>(*result) == false);
-    }
-
-    // callback_prefix_parse()
-    {
-        auto assign_bool_parser =
-            callback_identifier >> '=' >> callback_true_false >> ';';
-
-        auto f = r.begin();
-        auto const l = r.end();
-        std::string_view sv;
-        bool b = false;
-        auto success = bp::callback_prefix_parse(
-            f, l, assign_bool_parser, callbacks{sv, b});
-        BOOST_TEST(success);
-        BOOST_TEST(sv == "foo");
-        BOOST_TEST(b == false);
-    }
-    {
-        auto assign_bool_no_semi_parser =
-            callback_identifier >> '=' >> callback_true_false;
-
-        auto f = r.begin();
-        auto const l = r.end();
-        std::string_view sv;
-        bool b = false;
-        auto success = bp::callback_prefix_parse(
-            f, l, assign_bool_no_semi_parser, callbacks{sv, b});
-        BOOST_TEST(success);
-        BOOST_TEST(sv == "foo");
-        BOOST_TEST(b == false);
-        BOOST_TEST(f != l);
-    }
-
-    // callback_parse()
-    {
-        auto assign_bool_parser =
-            callback_identifier >> '=' >> callback_true_false >> ';';
-
-        std::string_view sv;
-        bool b = false;
-        auto success =
-            bp::callback_parse(r, assign_bool_parser, callbacks{sv, b});
-        BOOST_TEST(success);
-        BOOST_TEST(sv == "foo");
-        BOOST_TEST(b == false);
-    }
-
-    return boost::report_errors();
-}

test/lexer_and_parser_symbol_table.cpp

@@ -1,112 +0,0 @@
-// Copyright (C) 2024 T. Zachary Laine
-//
-// Distributed under the Boost Software License, Version 1.0. (See
-// accompanying file LICENSE_1_0.txt or copy at
-// http://www.boost.org/LICENSE_1_0.txt)
-#define BOOST_PARSER_TESTING
-#include <boost/parser/lexer.hpp>
-#include <boost/parser/parser.hpp>
-
-#include <boost/core/lightweight_test.hpp>
-
-
-namespace bp = boost::parser;
-
-bp::rule<class symbol_rule, std::string_view> const symrule = "symbols";
-bp::symbols<std::string_view> rule_symbols;
-auto const fwd_attr = [](auto & ctx) { _val(ctx) = _attr(ctx); };
-auto symrule_def = rule_symbols[fwd_attr];
-BOOST_PARSER_DEFINE_RULES(symrule);
-
-constexpr auto I = bp::token_spec<"I", 0>;
-constexpr auto V = bp::token_spec<"V", 1>;
-constexpr auto X = bp::token_spec<"X", 2>;
-constexpr auto L = bp::token_spec<"L", 3>;
-constexpr auto C = bp::token_spec<"C", 4>;
-constexpr auto arabic_num = bp::token_spec<"\\d+", 5, int>;
-
-constexpr auto lexer = bp::lexer<char, int> | I | V | X | L | C | arabic_num;
-
-int main()
-{
-// symbols_empty
-{
-    bp::symbols<int> roman_numerals;
-    bp::symbols<std::string> named_strings;
-
-    auto r = "I" | bp::to_tokens(lexer);
-    BOOST_TEST(!bp::parse(r, roman_numerals));
-    BOOST_TEST(!bp::parse(r, named_strings));
-}
-
-// symbols_simple
-{
-    bp::symbols<int> const roman_numerals = {
-        {"I", 1}, {"V", 5}, {"X", 10}, {"L", 50}, {"C", 100}};
-    bp::symbols<std::string> const named_strings = {
-        {"I", "1"}, {"V", "5"}, {"X", "10"}, {"L", "50"}, {"C", "100"}};
-
-    {
-        auto const result =
-            bp::parse("I" | bp::to_tokens(lexer), roman_numerals);
-        BOOST_TEST(result);
-        BOOST_TEST(*result == 1);
-    }
-    {
-        auto const result =
-            bp::parse("I" | bp::to_tokens(lexer), named_strings);
-        BOOST_TEST(result);
-        BOOST_TEST(*result == "1");
-    }
-
-    {
-        auto const result =
-            bp::parse("L" | bp::to_tokens(lexer), roman_numerals);
-        BOOST_TEST(result);
-        BOOST_TEST(*result == 50);
-    }
-    {
-        auto const result =
-            bp::parse("L" | bp::to_tokens(lexer), named_strings);
-        BOOST_TEST(result);
-        BOOST_TEST(*result == "50");
-    }
-}
-
-// symbols_mutating
-{
-    bp::symbols<int> roman_numerals;
-    roman_numerals.insert_for_next_parse("I", 1);
-    roman_numerals.insert_for_next_parse("V", 5);
-    roman_numerals.insert_for_next_parse("X", 10);
-    auto const add_numeral = [&roman_numerals](auto & context) {
-        using namespace boost::parser::literals;
-        const std::string_view sv = bp::get(_attr(context), 0_c);
-        roman_numerals.insert(context, sv, bp::get(_attr(context), 1_c));
-    };
-    auto const numerals_parser =
-        ((I | V | X | L | C) >> arabic_num)[add_numeral] >> roman_numerals;
-
-    {
-        auto const result =
-            bp::parse("L50L" | bp::to_tokens(lexer), numerals_parser);
-        BOOST_TEST(result);
-        BOOST_TEST(*result == 50);
-        BOOST_TEST(!bp::parse("L", roman_numerals));
-    }
-    {
-        auto const result =
-            bp::parse("C100C" | bp::to_tokens(lexer), numerals_parser);
-        BOOST_TEST(result);
-        BOOST_TEST(*result == 100);
-        BOOST_TEST(!bp::parse("C", roman_numerals));
-    }
-    {
-        auto const result =
-            bp::parse("L50C" | bp::to_tokens(lexer), numerals_parser);
-        BOOST_TEST(!result);
-    }
-}
-
-return boost::report_errors();
-}

test/parser_perm.cpp

@@ -88,5 +88,66 @@ int main()
         }
     }
 
-return boost::report_errors();
+    {
+        using namespace bp::literals;
+        constexpr auto parser =
+            bp::delimiter(','_l)[bp::int_ || bp::string("foo") || bp::char_('g')];
+
+        {
+            auto result = bp::parse("42 foo g", parser, bp::ws);
+            BOOST_TEST(!result);
+            result = bp::parse("42, foo ,g", parser, bp::ws);
+            BOOST_TEST(result);
+            BOOST_TEST(
+                *result ==
+                (bp::tuple<int, std::string, double>(42, "foo"s, 'g')));
+        }
+        {
+            auto result = bp::parse(",42, g, foo", parser, bp::ws);
+            BOOST_TEST(!result);
+            result = bp::parse("42, g , foo", parser, bp::ws);
+            BOOST_TEST(result);
+            BOOST_TEST(
+                *result ==
+                (bp::tuple<int, std::string, double>(42, "foo"s, 'g')));
+        }
+        {
+            auto result = bp::parse("foo, 42, g,", parser, bp::ws);
+            BOOST_TEST(!result);
+            result = bp::parse("foo, 42, g", parser, bp::ws);
+            BOOST_TEST(result);
+            BOOST_TEST(
+                *result ==
+                (bp::tuple<int, std::string, double>(42, "foo"s, 'g')));
+        }
+        {
+            auto result = bp::parse("foo g 42", parser, bp::ws);
+            BOOST_TEST(!result);
+            result = bp::parse("foo, g, 42", parser, bp::ws);
+            BOOST_TEST(result);
+            BOOST_TEST(
+                *result ==
+                (bp::tuple<int, std::string, double>(42, "foo"s, 'g')));
+        }
+        {
+            auto result = bp::parse("g foo 42", parser, bp::ws);
+            BOOST_TEST(!result);
+            result = bp::parse("g ,foo ,42", parser, bp::ws);
+            BOOST_TEST(result);
+            BOOST_TEST(
+                *result ==
+                (bp::tuple<int, std::string, double>(42, "foo"s, 'g')));
+        }
+        {
+            auto result = bp::parse("g 42 foo", parser, bp::ws);
+            BOOST_TEST(!result);
+            result = bp::parse("g , 42 , foo", parser, bp::ws);
+            BOOST_TEST(result);
+            BOOST_TEST(
+                *result ==
+                (bp::tuple<int, std::string, double>(42, "foo"s, 'g')));
+        }
+    }
+
+    return boost::report_errors();
 }

test/parser_quoted_string.cpp

@@ -49,7 +49,7 @@ int main()
     }
 }
 
-// different_char
+// different_quote
 {
     constexpr auto parser = bp::quoted_string('\'');
 
@@ -75,9 +75,18 @@ int main()
         BOOST_TEST(result);
         BOOST_TEST(*result == "'foo'");
     }
+
+    {
+        constexpr auto parser = bp::quoted_string('\'', bp::char_("g\t"));
+
+        auto result = bp::parse(R"('\'ggg\'')", parser, bp::ws);
+        BOOST_TEST(result);
+        BOOST_TEST(*result == "'ggg'");
+        BOOST_TEST(!bp::parse(R"('\'fff\'')", parser, bp::ws));
+    }
 }
 
-// different_char_with_escapes
+// different_quote_with_escapes
 {
     {
         auto parser = bp::quoted_string('\'', cu_escapes);
@@ -119,6 +128,58 @@ int main()
     }
 }
 
+// different_quote_with_escapes_and_char_p
+{
+    {
+        auto parser = bp::quoted_string('\'', cu_escapes, bp::char_("g\t"));
+
+        {
+            auto result = bp::parse("", parser, bp::ws);
+            BOOST_TEST(!result);
+        }
+
+        {
+            auto result = bp::parse("'ggg'", parser, bp::ws);
+            BOOST_TEST(result);
+        }
+
+        {
+            auto result = bp::parse("'fff'", parser, bp::ws);
+            BOOST_TEST(!result);
+        }
+
+        {
+            auto result = bp::parse(R"('ggg\t')", parser, bp::ws);
+            BOOST_TEST(result);
+            BOOST_TEST(*result == "ggg\t");
+        }
+
+        {
+            auto result = bp::parse(R"('ggg\g')", parser, bp::ws);
+            BOOST_TEST(!result);
+        }
+    }
+    {
+        auto parser = bp::quoted_string('\'', cp_escapes);
+
+        {
+            auto result = bp::parse("", parser, bp::ws);
+            BOOST_TEST(!result);
+        }
+
+        {
+            auto result = bp::parse(R"('\tggg')", parser, bp::ws);
+            BOOST_TEST(result);
+            BOOST_TEST(*result == "\tggg");
+        }
+
+        {
+            auto result = bp::parse(R"('g\ggg')", parser, bp::ws);
+            BOOST_TEST(!result);
+        }
+    }
+}
+
 // char_set
 {
     constexpr auto parser = bp::quoted_string("'\"");
@@ -171,6 +232,15 @@ int main()
         // character.
         BOOST_TEST(!bp::parse(R"("\'foo")", parser, bp::ws));
     }
+
+    {
+        constexpr auto parser = bp::quoted_string("'\"", bp::char_("g"));
+
+        auto result = bp::parse(R"('\'ggg\'')", parser, bp::ws);
+        BOOST_TEST(result);
+        BOOST_TEST(*result == "'ggg'");
+        BOOST_TEST(!bp::parse(R"('\'fff\'')", parser, bp::ws));
+    }
 }
 
 // char_set_with_escapes
@@ -233,6 +303,15 @@ int main()
             BOOST_TEST(!result);
         }
     }
+
+    {
+        auto parser = bp::quoted_string("'\"", cu_escapes, bp::char_("g"));
+
+        auto result = bp::parse(R"('\'ggg\'')", parser, bp::ws);
+        BOOST_TEST(result);
+        BOOST_TEST(*result == "'ggg'");
+        BOOST_TEST(!bp::parse(R"('\'fff\'')", parser, bp::ws));
+    }
 }
 
 // doc_examples
@@ -244,8 +323,7 @@ int main()
     assert(result1);
     std::cout << *result1 << "\n"; // Prints: some text
 
-    auto result2 =
-        bp::parse("\"some \\\"text\\\"\"", bp::quoted_string, bp::ws);
+    auto result2 = bp::parse(R"("some \"text\"")", bp::quoted_string, bp::ws);
     assert(result2);
     std::cout << *result2 << "\n"; // Prints: some "text"
     //]
@@ -279,6 +357,16 @@ int main()
     assert(result5);
     std::cout << *result5 << "\n"; // Prints (with a CRLF newline): some text
     //]
+
+    //[ quoted_string_example_6
+    auto result6 = bp::parse(
+        "'some text'", bp::quoted_string("'\"", bp::char_('g')), bp::ws);
+    assert(!result6);
+    result6 =
+        bp::parse("'gggg'", bp::quoted_string("'\"", bp::char_('g')), bp::ws);
+    assert(result6);
+    std::cout << *result6 << "\n"; // Prints: gggg
+    //]
 }
 
 return boost::report_errors();

test/parser_symbol_table.cpp

@@ -316,7 +316,7 @@ int main()
             add_parser >> roman_numerals >> next_delete_parser >>
             roman_numerals);
         BOOST_TEST(result);
-        BOOST_TEST(*result == std::tuple(100, 100));
+        BOOST_TEST(*result == detail::hl::make_tuple(100, 100));
     }
 
     {

test/tracing.cpp

@@ -6,10 +6,6 @@
  *   http://www.boost.org/LICENSE_1_0.txt)
  */
 
-#include <boost/parser/config.hpp>
-#if BOOST_PARSER_USE_CONCEPTS
-#include <boost/parser/lexer.hpp>
-#endif
 #include <boost/parser/parser.hpp>
 
 
@@ -30,17 +26,6 @@ struct globals_t
 
 globals_t const globals;
 
-enum class unprintable_tokens { foo, bar };
-enum class printable_tokens { foo, bar };
-std::ostream & operator<<(std::ostream & os, printable_tokens tok)
-{
-    switch (tok) {
-    case printable_tokens::foo: os << "foo"; break;
-    case printable_tokens::bar: os << "bar"; break;
-    }
-    return os;
-}
-
 auto i = [](auto & ctx) { return _globals(ctx).i; };
 auto i2 = [](auto & ctx) { return _globals(ctx).i2; };
 auto u = [](auto & ctx) { return _globals(ctx).u; };
@@ -499,64 +484,4 @@ int main()
 
     PARSE_CHAR32(float_);
     PARSE_CHAR32(double_);
-
-#if BOOST_PARSER_USE_CONCEPTS
-    {
-        std::cout << "\n\n"
-                  << "----------------------------------------\n"
-                  << "| unprintable_foo (token_spec)          |\n"
-                  << "----------------------------------------\n";
-
-        constexpr auto unprintable_foo =
-            token_spec<"\\w\\w\\w", unprintable_tokens::foo>;
-        constexpr auto unprintable_lexer =
-            lexer<char, unprintable_tokens> | unprintable_foo;
-
-        std::cout << "token_spec<\"\\w\\w\\w\", unprintable_tokens::foo>:\n";
-        parse(str | to_tokens(unprintable_lexer), unprintable_foo, trace::on);
-
-        std::cout
-            << "token_spec<\"\\w\\w\\w\", unprintable_tokens::foo>(\"foo\"):\n";
-        parse(
-            str | to_tokens(unprintable_lexer),
-            unprintable_foo("foo"),
-            trace::on);
-    }
-
-    {
-        std::cout << "\n\n"
-                  << "----------------------------------------\n"
-                  << "| printable_foo (token_spec)            |\n"
-                  << "----------------------------------------\n";
-
-        constexpr auto printable_foo =
-            token_spec<"\\w\\w\\w", printable_tokens::foo>;
-        constexpr auto printable_lexer =
-            lexer<char, printable_tokens> | printable_foo;
-
-        std::cout << "token_spec<\"\\w\\w\\w\", printable_tokens::foo>:\n";
-        parse(str | to_tokens(printable_lexer), printable_foo, trace::on);
-
-        std::cout
-            << "token_spec<\"\\w\\w\\w\", printable_tokens::foo>(\"bar\"):\n";
-        parse(
-            str | to_tokens(printable_lexer), printable_foo("bar"), trace::on);
-    }
-
-    {
-        std::cout << "\n\n"
-                  << "----------------------------------------\n"
-                  << "| int_foo (token_spec)                  |\n"
-                  << "----------------------------------------\n";
-
-        constexpr auto int_foo = token_spec<"\\w\\w\\w", 42, int>;
-        constexpr auto int_lexer = lexer<char, int> | int_foo;
-
-        std::cout << "token_spec<\"\\w\\w\\w\", 42, int>:\n";
-        parse(str | to_tokens(int_lexer), int_foo, trace::on);
-
-        std::cout << "token_spec<\"\\w\\w\\w\", 42, int>(13):\n";
-        parse(str | to_tokens(int_lexer), int_foo(13), trace::on);
-    }
-#endif
 }