From 2e422497bc62abdfc31a126c6cb6402f63264a37 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 9 Nov 2006 14:35:26 +0000 Subject: [PATCH] Wave: modified the real_positions example to use a new tokentype carrying both positions, the original and the corrected one. [SVN r35950] --- ChangeLog | 3 +- .../wave/cpplexer/re2clex/cpp_re2c_lexer.hpp | 1 - samples/real_positions/build/Jamfile | 5 + samples/real_positions/build/Jamfile.v2 | 5 + .../correct_token_positions.hpp | 2 +- .../instantiate_cpp_exprgrammar.cpp | 41 ++++ .../instantiate_cpp_grammar.cpp | 45 ++++ .../instantiate_cpp_literalgrs.cpp | 42 ++++ .../instantiate_defined_grammar.cpp | 39 ++++ .../real_positions/instantiate_re2c_lexer.cpp | 60 ++++++ .../real_positions/real_position_token.hpp | 202 ++++++++++++++++++ samples/real_positions/real_positions.cpp | 37 ++-- 12 files changed, 462 insertions(+), 20 deletions(-) create mode 100644 samples/real_positions/instantiate_cpp_exprgrammar.cpp create mode 100644 samples/real_positions/instantiate_cpp_grammar.cpp create mode 100644 samples/real_positions/instantiate_cpp_literalgrs.cpp create mode 100644 samples/real_positions/instantiate_defined_grammar.cpp create mode 100644 samples/real_positions/instantiate_re2c_lexer.cpp create mode 100644 samples/real_positions/real_position_token.hpp diff --git a/ChangeLog b/ChangeLog index e8fbe7c..3bc92f9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -66,7 +66,8 @@ CHANGELOG be used to alter the token before it gets returned to the calling application. - Added a new sample 'real_positions' demonstrating the new generated_token() - preprocessing hook. + preprocessing hook and showing how to use Wave with a new token type without + using a new lexer type. Boost V1.34.0 - Wave Version 1.2.4 diff --git a/include/boost/wave/cpplexer/re2clex/cpp_re2c_lexer.hpp b/include/boost/wave/cpplexer/re2clex/cpp_re2c_lexer.hpp index 4044931..f70b4c2 100644 --- a/include/boost/wave/cpplexer/re2clex/cpp_re2c_lexer.hpp +++ b/include/boost/wave/cpplexer/re2clex/cpp_re2c_lexer.hpp @@ -33,7 +33,6 @@ #include #include -#include #include #include #include diff --git a/samples/real_positions/build/Jamfile b/samples/real_positions/build/Jamfile index 7b5fe54..2200c27 100644 --- a/samples/real_positions/build/Jamfile +++ b/samples/real_positions/build/Jamfile @@ -12,6 +12,11 @@ subproject libs/wave/samples/real_positions/build ; exe real_positions : ../real_positions.cpp + ../instantiate_cpp_exprgrammar.cpp + ../instantiate_cpp_grammar.cpp + ../instantiate_cpp_literalgrs.cpp + ../instantiate_defined_grammar.cpp + ../instantiate_re2c_lexer.cpp ../../../build/boost_wave ../../../../../libs/filesystem/build/boost_filesystem : diff --git a/samples/real_positions/build/Jamfile.v2 b/samples/real_positions/build/Jamfile.v2 index fa0f4f3..407bf59 100644 --- a/samples/real_positions/build/Jamfile.v2 +++ b/samples/real_positions/build/Jamfile.v2 @@ -10,6 +10,11 @@ exe real_positions : ../real_positions.cpp + ../instantiate_cpp_exprgrammar + ../instantiate_cpp_grammar + ../instantiate_cpp_literalgrs + ../instantiate_defined_grammar + ../instantiate_re2c_lexer /boost/wave//boost_wave /boost/wave//boost_filesystem ; diff --git a/samples/real_positions/correct_token_positions.hpp b/samples/real_positions/correct_token_positions.hpp index 1c8eedc..6232338 100644 --- a/samples/real_positions/correct_token_positions.hpp +++ b/samples/real_positions/correct_token_positions.hpp @@ -114,7 +114,7 @@ struct correct_token_position } // set the new position in the token to be returned - token.set_position(current_pos); + token.set_corrected_position(current_pos); return token; } diff --git a/samples/real_positions/instantiate_cpp_exprgrammar.cpp b/samples/real_positions/instantiate_cpp_exprgrammar.cpp new file mode 100644 index 0000000..50d66c6 --- /dev/null +++ b/samples/real_positions/instantiate_cpp_exprgrammar.cpp @@ -0,0 +1,41 @@ +/*============================================================================= + Boost.Wave: A Standard compliant C++ preprocessor library + Example: real_positions + + Explicit instantiation of the cpp_expression_grammar parsing function + + http://www.boost.org/ + + Copyright (c) 2001-2006 Hartmut Kaiser. Distributed under the Boost + Software License, Version 1.0. (See accompanying file + LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +=============================================================================*/ + +#include + +#if BOOST_WAVE_SEPARATE_GRAMMAR_INSTANTIATION != 0 + +#include + +#include + +#include "real_position_token.hpp" // token class +#include // lexer type + +#include + +/////////////////////////////////////////////////////////////////////////////// +// +// Explicit instantiation of the expression_grammar_gen template with the +// correct token type. This instantiates the corresponding parse function, +// which in turn instantiates the expression_grammar object (see +// wave/grammars/cpp_expression_grammar.hpp) +// +/////////////////////////////////////////////////////////////////////////////// + +typedef lex_token<> token_type; + +template struct boost::wave::grammars::expression_grammar_gen; + +#endif // #if BOOST_WAVE_SEPARATE_GRAMMAR_INSTANTIATION != 0 + diff --git a/samples/real_positions/instantiate_cpp_grammar.cpp b/samples/real_positions/instantiate_cpp_grammar.cpp new file mode 100644 index 0000000..f4275fb --- /dev/null +++ b/samples/real_positions/instantiate_cpp_grammar.cpp @@ -0,0 +1,45 @@ +/*============================================================================= + Boost.Wave: A Standard compliant C++ preprocessor library + Example: real_positions + + Explicit instantiation of the cpp_grammar parsing function + + http://www.boost.org/ + + Copyright (c) 2001-2006 Hartmut Kaiser. Distributed under the Boost + Software License, Version 1.0. (See accompanying file + LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +=============================================================================*/ + +#include + +#if BOOST_WAVE_SEPARATE_GRAMMAR_INSTANTIATION != 0 + +#include +#include + +#include + +#include "real_position_token.hpp" // token class +#include // lexer type + +#include + +/////////////////////////////////////////////////////////////////////////////// +// +// Explicit instantiation of the cpp_grammar_gen template with the correct +// token type. This instantiates the corresponding pt_parse function, which +// in turn instantiates the cpp_grammar object +// (see wave/grammars/cpp_grammar.hpp) +// +/////////////////////////////////////////////////////////////////////////////// + +typedef lex_token<> token_type; +typedef boost::wave::cpplexer::lex_iterator lexer_type; +typedef std::list > + token_sequence_type; + +template struct boost::wave::grammars::cpp_grammar_gen; + +#endif // #if BOOST_WAVE_SEPARATE_GRAMMAR_INSTANTIATION != 0 + diff --git a/samples/real_positions/instantiate_cpp_literalgrs.cpp b/samples/real_positions/instantiate_cpp_literalgrs.cpp new file mode 100644 index 0000000..faa02dd --- /dev/null +++ b/samples/real_positions/instantiate_cpp_literalgrs.cpp @@ -0,0 +1,42 @@ +/*============================================================================= + Boost.Wave: A Standard compliant C++ preprocessor library + Example: real_positions + + http://www.boost.org/ + + Copyright (c) 2001-2006 Hartmut Kaiser. Distributed under the Boost + Software License, Version 1.0. (See accompanying file + LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +=============================================================================*/ + +#include + +#if BOOST_WAVE_SEPARATE_GRAMMAR_INSTANTIATION != 0 + +#include + +#include + +#include "real_position_token.hpp" // token class +#include // lexer type + +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +// +// Explicit instantiation of the intlit_grammar_gen, chlit_grammar_gen and +// floatlit_grammar_gen templates with the correct token type. This +// instantiates the corresponding parse function, which in turn instantiates +// the corresponding parser object. +// +/////////////////////////////////////////////////////////////////////////////// + +typedef lex_token<> token_type; + +template struct boost::wave::grammars::intlit_grammar_gen; +template struct boost::wave::grammars::chlit_grammar_gen; + +#endif // #if BOOST_WAVE_SEPARATE_GRAMMAR_INSTANTIATION != 0 + diff --git a/samples/real_positions/instantiate_defined_grammar.cpp b/samples/real_positions/instantiate_defined_grammar.cpp new file mode 100644 index 0000000..95f1386 --- /dev/null +++ b/samples/real_positions/instantiate_defined_grammar.cpp @@ -0,0 +1,39 @@ +/*============================================================================= + Boost.Wave: A Standard compliant C++ preprocessor library + Example: real_positions + + http://www.boost.org/ + + Copyright (c) 2001-2006 Hartmut Kaiser. Distributed under the Boost + Software License, Version 1.0. (See accompanying file + LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +=============================================================================*/ + +#include + +#if BOOST_WAVE_SEPARATE_GRAMMAR_INSTANTIATION != 0 + +#include + +#include + +#include "real_position_token.hpp" // token class +#include // lexer type + +#include + +/////////////////////////////////////////////////////////////////////////////// +// +// Explicit instantiation of the defined_grammar_gen template +// with the correct token type. This instantiates the corresponding parse +// function, which in turn instantiates the defined_grammar +// object (see wave/grammars/cpp_defined_grammar.hpp) +// +/////////////////////////////////////////////////////////////////////////////// + +typedef boost::wave::cpplexer::lex_iterator > lexer_type; + +template struct boost::wave::grammars::defined_grammar_gen; + +#endif // #if BOOST_WAVE_SEPARATE_GRAMMAR_INSTANTIATION != 0 + diff --git a/samples/real_positions/instantiate_re2c_lexer.cpp b/samples/real_positions/instantiate_re2c_lexer.cpp new file mode 100644 index 0000000..266a614 --- /dev/null +++ b/samples/real_positions/instantiate_re2c_lexer.cpp @@ -0,0 +1,60 @@ +/*============================================================================= + Boost.Wave: A Standard compliant C++ preprocessor library + Explicit instantiation of the lex_functor generation function + + http://www.boost.org/ + + Copyright (c) 2001-2006 Hartmut Kaiser. Distributed under the Boost + Software License, Version 1.0. (See accompanying file + LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +=============================================================================*/ + +#include // configuration data + +#if BOOST_WAVE_SEPARATE_LEXER_INSTANTIATION != 0 + +#include + +#include + +#include "real_position_token.hpp" // token class +#include // lexer type + +/////////////////////////////////////////////////////////////////////////////// +// The following file needs to be included only once throughout the whole +// program. +#include + +// this must occur after all of the includes and before any code appears +#ifdef BOOST_HAS_ABI_HEADERS +#include BOOST_ABI_PREFIX +#endif + +/////////////////////////////////////////////////////////////////////////////// +// +// This instantiates the correct 'new_lexer' function, which generates the +// C++ lexer used in this sample. You will have to instantiate the +// new_lexer_gen<> template with the same iterator type, as you have used for +// instantiating the boost::wave::context<> object. +// +// This is moved into a separate compilation unit to decouple the compilation +// of the C++ lexer from the compilation of the other modules, which helps to +// reduce compilation time. +// +// The template parameter(s) supplied should be identical to the first +// parameter supplied while instantiating the boost::wave::context<> template +// (see the file cpp.cpp). +// +/////////////////////////////////////////////////////////////////////////////// + +template struct boost::wave::cpplexer::new_lexer_gen< + BOOST_WAVE_STRINGTYPE::iterator>; +template struct boost::wave::cpplexer::new_lexer_gen< + BOOST_WAVE_STRINGTYPE::const_iterator>; + +// the suffix header occurs after all of the code +#ifdef BOOST_HAS_ABI_HEADERS +#include BOOST_ABI_SUFFIX +#endif + +#endif // BOOST_WAVE_SEPARATE_LEXER_INSTANTIATION != 0 diff --git a/samples/real_positions/real_position_token.hpp b/samples/real_positions/real_position_token.hpp new file mode 100644 index 0000000..c6bbcbb --- /dev/null +++ b/samples/real_positions/real_position_token.hpp @@ -0,0 +1,202 @@ +/*============================================================================= + Boost.Wave: A Standard compliant C++ preprocessor library + + A C++ lexer token definition for the real_positions example + + http://www.boost.org/ + + Copyright (c) 2001-2006 Hartmut Kaiser. Distributed under the Boost + Software License, Version 1.0. (See accompanying file + LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +=============================================================================*/ + +#if !defined(REAL_POSITION_TOKEN_HPP_HK_061109_INCLUDED) +#define REAL_POSITION_TOKEN_HPP_HK_061109_INCLUDED + +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +namespace impl { + +template +class token_data +{ +public: + typedef StringTypeT string_type; + typedef PositionT position_type; + + token_data() + : id(boost::wave::T_EOI), refcnt(1) + {} + + token_data(boost::wave::token_id id_, string_type const &value_, + position_type const &pos_) + : id(id_), value(value_), pos(pos_), corrected_pos(pos_), refcnt(1) + {} + + token_data(token_data const& rhs) + : id(rhs.id), value(rhs.value), pos(rhs.pos), + corrected_pos(rhs.corrected_pos), refcnt(1) + {} + + ~token_data() + {} + + std::size_t addref() { return ++refcnt; } + std::size_t release() { return --refcnt; } + std::size_t get_refcnt() const { return refcnt; } + +// accessors + operator boost::wave::token_id() const { return id; } + string_type const &get_value() const { return value; } + position_type const &get_position() const { return pos; } + position_type const &get_corrected_position() const + { return corrected_pos; } + + void set_token_id (boost::wave::token_id id_) { id = id_; } + void set_value (string_type const &value_) { value = value_; } + void set_position (position_type const &pos_) { pos = pos_; } + void set_corrected_position (position_type const &pos_) + { corrected_pos = pos_; } + + friend bool operator== (token_data const& lhs, token_data const& rhs) + { + // two tokens are considered equal even if they refer to different + // positions + return (lhs.id == rhs.id && lhs.value == rhs.value) ? true : false; + } + +private: + boost::wave::token_id id; // the token id + string_type value; // the text, which was parsed into this token + position_type pos; // the original file position + position_type corrected_pos; // the original file position + std::size_t refcnt; +}; + +/////////////////////////////////////////////////////////////////////////////// +} // namespace impl + +/////////////////////////////////////////////////////////////////////////////// +// forward declaration of the token type +template +class lex_token; + +/////////////////////////////////////////////////////////////////////////////// +// +// lex_token +// +/////////////////////////////////////////////////////////////////////////////// + +template +class lex_token +{ +public: + typedef BOOST_WAVE_STRINGTYPE string_type; + typedef PositionT position_type; + + lex_token() + : data(new impl::token_data()) + {} + + lex_token(lex_token const& rhs) + : data(rhs.data) + { + data->addref(); + } + + lex_token(boost::wave::token_id id_, string_type const &value_, + PositionT const &pos_) + : data(new impl::token_data(id_, value_, pos_)) + {} + + ~lex_token() + { + if (0 == data->release()) + delete data; + data = 0; + } + + lex_token& operator=(lex_token const& rhs) + { + if (&rhs != this) { + if (0 == data->release()) + delete data; + + data = rhs.data; + data->addref(); + } + return *this; + } + +// accessors + operator boost::wave::token_id() const + { return boost::wave::token_id(*data); } + string_type const &get_value() const + { return data->get_value(); } + position_type const &get_position() const + { return data->get_position(); } + position_type const &get_corrected_position() const + { return data->get_corrected_position(); } + + void set_token_id (boost::wave::token_id id_) + { make_unique(); data->set_token_id(id_); } + void set_value (string_type const &value_) + { make_unique(); data->set_value(value_); } + void set_position (position_type const &pos_) + { make_unique(); data->set_position(pos_); } + void set_corrected_position (position_type const &pos_) + { make_unique(); data->set_corrected_position(pos_); } + + friend bool operator== (lex_token const& lhs, lex_token const& rhs) + { + return *(lhs.data) == *(rhs.data); + } + +// debug support +#if BOOST_WAVE_DUMP_PARSE_TREE != 0 +// access functions for the tree_to_xml functionality + static int get_token_id(lex_token const &t) + { return token_id(t); } + static string_type get_token_value(lex_token const &t) + { return t.get_value(); } +#endif + +#if defined(BOOST_SPIRIT_DEBUG) +// debug support + void print (std::ostream &stream) const + { + data->print(stream); + } +#endif // defined(BOOST_SPIRIT_DEBUG) + +private: +#if BOOST_WAVE_SERIALIZATION != 0 + friend class boost::serialization::access; + template + void serialize(Archive &ar, const unsigned int version) + { + data->serialize(ar, version); + } +#endif + + // make a unique copy of the current object + void make_unique() + { + if (1 == data->get_refcnt()) + return; + + impl::token_data *newdata = + new impl::token_data(*data); + + data->release(); // release this reference, can't get zero + data = newdata; + } + + impl::token_data *data; +}; + +#endif // !defined(REAL_POSITION_TOKEN_HPP_HK_061109_INCLUDED) diff --git a/samples/real_positions/real_positions.cpp b/samples/real_positions/real_positions.cpp index 2ddccc2..4a9034b 100644 --- a/samples/real_positions/real_positions.cpp +++ b/samples/real_positions/real_positions.cpp @@ -1,7 +1,7 @@ /*============================================================================= Boost.Wave: A Standard compliant C++ preprocessor library - Sample showing how to correct the positions inside the retunred tokens in + Sample showing how to correct the positions inside the returned tokens in a way that these appear to be consecutive (ignoring positions from macro definitions). @@ -24,8 +24,8 @@ /////////////////////////////////////////////////////////////////////////////// // Include the lexer stuff -#include // token class -#include // lexer class +#include "real_position_token.hpp" // token class +#include // lexer type #include "correct_token_positions.hpp" @@ -38,8 +38,7 @@ /////////////////////////////////////////////////////////////////////////////// template inline std::ostream & -operator<< (std::ostream &stream, - boost::wave::cpplexer::lex_token const &t) +operator<< (std::ostream &stream, lex_token const &t) { using namespace std; using namespace boost::wave; @@ -62,14 +61,9 @@ operator<< (std::ostream &stream, } } - stream - << ") at " << t.get_position().get_file() << " (" - << setw(3) << right << t.get_position().get_line() << "/" - << setw(2) << right << t.get_position().get_column() - << "): >"; + stream << "): >"; - typedef typename boost::wave::cpplexer::lex_token::string_type - string_type; + typedef typename lex_token::string_type string_type; string_type const& value = t.get_value(); for (std::size_t i = 0; i < value.size(); ++i) { @@ -82,8 +76,16 @@ operator<< (std::ostream &stream, break; } } - stream << "<"; - + stream << "<" << std::endl; + stream << " at: " << t.get_position().get_file() << " (" + << setw(3) << right << t.get_position().get_line() << "/" + << setw(2) << right << t.get_position().get_column() + << ")" << std::endl; + stream << " and: " << t.get_corrected_position().get_file() << " (" + << setw(3) << right << t.get_corrected_position().get_line() << "/" + << setw(2) << right << t.get_corrected_position().get_column() + << ")"; + return stream; } @@ -112,13 +114,14 @@ boost::wave::util::file_position_type current_position; instring = std::string(std::istreambuf_iterator(instream.rdbuf()), std::istreambuf_iterator()); - // The template boost::wave::cpplexer::lex_token<> is the token type to be + // The template real_positions::lex_token<> is the token type to be // used by the Wave library. - typedef boost::wave::cpplexer::lex_token<> token_type; + typedef lex_token<> token_type; // The template boost::wave::cpplexer::lex_iterator<> is the lexer type to // be used by the Wave library. - typedef boost::wave::cpplexer::lex_iterator lex_iterator_type; + typedef boost::wave::cpplexer::lex_iterator + lex_iterator_type; // This is the resulting context type to use. The first template parameter // should match the iterator type to be used during construction of the