From 9aaa78546c06ca24fc587e3af8ba5e6af5d4c4e1 Mon Sep 17 00:00:00 2001 From: Jeff Trull Date: Tue, 17 Nov 2020 08:16:22 -0800 Subject: [PATCH] Additional documentation (#121) * Add documentation for three undocumented hooks Those are found_unknown_directive, emit_line_directive, and locate_include_file Also tweak formatting, fix typos. * Add documentation for five samples * Add links from samples docs to the hooks they use --- doc/class_reference_ctxpolicy.html | 123 ++++++++++++++++++++++++++--- doc/samples.html | 35 +++++++- 2 files changed, 142 insertions(+), 16 deletions(-) diff --git a/doc/class_reference_ctxpolicy.html b/doc/class_reference_ctxpolicy.html index a07005f..a4fb880 100644 --- a/doc/class_reference_ctxpolicy.html +++ b/doc/class_reference_ctxpolicy.html @@ -45,27 +45,38 @@ look here.

The context policy is used to provide callback hooks, which are called from inside the library into the user code, whenever

This policy type is used as a template parameter to the boost::wave::context<> object, where the default policy provides empty hook functions only.

Header wave/preprocessing_hooks.hpp synopsis

namespace boost {
namespace wave {
namespace context_policies {
 
    struct default_preprocessing_hooks {

        // general hook functions
         template <typename ContextT, typename TokenT>
        bool found_directive(ContextT const &ctx, 
            TokenT const &directive);

+        template <typename ContextT, typename ContainerT>
+        bool found_unknown_directive(ContextT const& ctx,
+            ContainerT const& line, ContainerT& pending);
+
         template <typename ContextT, typename ExceptionT>
        void throw_exception(ContextT const &ctx, 
            ExceptionT const& e);

        // test, whether a given token may be skipped
        template <typename ContextT>
        bool may_skip_whitespace (ContextT const& ctx,
            TokenT &token, bool &skipped_newline);

+
         // Conditional compilation
         template <
            typename ContextT, typename TokenT, 
            typename ContainerT
        >
        bool evaluated_conditional_expression(
            ContextT const &ctx, TokenT const& directive, 
            ContainerT const& expression, bool expression_value);

-        template <typename ContextT, typename TokenT>
            void skipped_token(ContextT const &ctx, 
            TokenT const& token);

        template <typename ContextT, typename TokenT>
        TokenT const& generated_token(ContextT const &ctx, 
            TokenT const& token);

        // macro expansion tracing
+        template <typename ContextT, typename TokenT>
            void skipped_token(ContextT const &ctx, 
            TokenT const& token);

        template <typename ContextT, typename TokenT>
        TokenT const& generated_token(ContextT const &ctx, 
            TokenT const& token);


        // macro expansion tracing
         template <
             typename ContextT, typename TokenT, typename ContainerT,
            typename IteratorT
        >
        bool expanding_function_like_macro(
            ContextT const &ctx, TokenT const &macrodef, 
            std::vector<TokenT> const &formal_args, 
            ContainerT const &definition, TokenT const &macrocall, 
            std::vector<ContainerT> const &arguments,
            IteratorT const &seqstart, Iterator const &seqend);
 
        template <typename ContextT, typename TokenT, typename ContainerT>
        bool expanding_object_like_macro(
            ContextT const &ctx, TokenT const &macro, 
            ContainerT const &definition, TokenT const &macrocall);
 
        template <typename ContextT, typename ContainerT>
        void expanded_macro(ContextT const &ctx, 
            ContainerT const &result);
 
        template <typename ContextT, typename ContainerT>
        void rescanned_macro(ContextT const &ctx, 
            ContainerT const &result);


         // include file tracing functions
         template <typename ContextT>
        bool found_include_directive(ContextT const &ctx, 
            std::string const &filename, bool include_next);

+        template <typename ContextT>
+        bool locate_include_file(ContextT& ctx, std::string &file_path, 
+            bool is_system, char const *current_name,
+            std::string &dir_path, std::string &native_name);

         template <typename ContextT>
        void opened_include_file(ContextT const &ctx, 
            std::string const &relname, std::string const& absname,
            bool is_system_include); 

         template <typename ContextT>
        void returning_from_include_file(ContextT const &ctx);

         template <typename ContextT>
        void detected_include_guard(ContextT const &ctx, 
@@ -82,8 +93,13 @@
             typename ParametersT, typename DefinitionT
        >
        void defined_macro(ContextT const &ctx, TokenT const &name, 
             bool is_functionlike, ParametersT const &parameters, 
            DefinitionT const &definition, bool is_predefined);

        template <typename ContextT, typename TokenT>
        void undefined_macro(ContextT const &ctx, 
            TokenT const &name);


         // #error and #warning directive hooks
-        template <typename ContextT, typename ContainerT>
        bool found_warning_directive(ContextT const &ctx, 
            ContainerT const &message);

        template <typename ContextT, typename ContainerT>
        bool found_error_directive(ContextT const &ctx, 
            ContainerT const &message);

        // #line directive hook
-        template <typename ContextT, typename ContainerT>
        void found_line_directive(ContextT const &ctx, 
            ContainerT const &arguments, unsigned int line,
            std::string const& filename);
    };

}}}   // namespace boost::wave::context_policies
+ template <typename ContextT, typename ContainerT>
bool found_warning_directive(ContextT const &ctx,
ContainerT const &message);

template <typename ContextT, typename ContainerT>
bool found_error_directive(ContextT const &ctx,
ContainerT const &message);

// #line directive hooks + template <typename ContextT, typename ContainerT>
void found_line_directive(ContextT const &ctx,
ContainerT const &arguments, unsigned int line,
std::string const& filename);
+ template <typename ContextT, typename ContainerT> + bool emit_line_directive(ContextT const& ctx, ContainerT &pending, + typename ContextT::token_type const& act_token); + + };

}}} // namespace boost::wave::context_policies

Member functions

General hook functions

found_directive

@@ -99,18 +115,33 @@ directive is replaced by a single newline character. If the return value is false, the directive is processed in the normal manner.

-

throw_exception

-
    template <typename ContextT, typename ExceptionT>
    void throw_exception(ContextT const &ctx, 
            ExceptionT const& e);
+ +

found_unknown_directive

+
+    template <typename ContextT, typename ContainerT>
+    bool found_unknown_directive(ContextT const& ctx,
+        ContainerT const& line, ContainerT& pending);
+
-

he function throw_exception is called, whenever a preprocessing exception occurs .

+

The function  found_unknown_directive is called, whenever an unknown preprocessor directive was encountered.

+

The parameter  ctx is a reference to the context object used for instantiating the preprocessing iterators by the user.

+

The parameter  line holds the tokens of the entire source line containing the unknown directive.

+

The parameter  pending may be used to push tokens back into the input stream, which are to be used as the replacement text for the whole line containing the unknown directive.

+

The return value defines whether the given expression has been properly interpreted by the hook function or not. If this function returns  false, the library will raise an  ill_formed_directive exception. Otherwise the tokens pushed back into  pending are passed on to the user program.

+
+ +

throw_exception

+
    template <typename ContextT, typename ExceptionT>
    void throw_exception(ContextT const &ctx, 
        ExceptionT const& e);
+
+

The function throw_exception is called, whenever a preprocessing exception occurs .

The ctx parameter provides a reference to the context_type used during instantiation of the preprocessing iterators by the user. 

The parameter e is the exception object containing detailed error information.

-

may_skipwhitespace

+

may_skip_whitespace

    template <typename ContextT, typename TokenT>
    bool may_skip_whitespace(ContextT const& ctx, TokenT &token, 
        bool& skipped_newline);
-

The function may_skipwhitespace will be called by the library, whenever a token is about to be returned to the calling application.

+

The function may_skip_whitespace will be called by the library, whenever a token is about to be returned to the calling application.

The ctx parameter provides a reference to the context_type used during instantiation of the preprocessing iterators by the user. Note, this parameter was added for the Wave V1.2.4 release.

The token parameter holds a reference to the current token. The policy is free to change this token if needed.

@@ -263,6 +294,47 @@ return value is false, the directive is executed in a normal manner.

+

locate_include_file

+
+    template <typename ContextT>
+    bool locate_include_file(ContextT& ctx, std::string &file_path, 
+        bool is_system, char const *current_name, std::string &dir_path, 
+        std::string &native_name)
+
+
+

The function  locate_include_file is called, whenever a #include +directive was encountered. It is supposed to locate the given file and +should return the full file name of the located file. This file name +is expected to uniquely identify the referenced file.

+ +

The parameter  ctx is a reference to the context object used for +instantiating the preprocessing iterators by the user.

+ +

The parameter  file_path contains the (expanded) file name found after + the #include directive. This parameter holds the string as it is + specified in the #include directive, i.e. <file> or "file" will result + in a parameter value 'file'.

+ +

The parameter  is_system is set to  true if this call happens as a + result of a #include <file> directive, it is  false otherwise, i.e. + for #include "file" directives.

+ +

The parameter  current_name is only used if a #include_next directive + was encountered (and BOOST_WAVE_SUPPORT_INCLUDE_NEXT was defined to be +non-zero). In this case it points to unique full name of the current + include file (if any). Otherwise this parameter is set to NULL.

+ +

The parameter  dir_path on return is expected to hold the directory + part of the located file.

+ +

The parameter  native_name on return is expected to hold the unique + full file name of the located file.

+ +

The return value defines whether the file was located successfully.

+ +
+ +

opened_include_file

    template <typename ContextT>
    void opened_include_file(ContextT const& ctx, 
        std::string const &rel_filename, std::string const &abs_filename, 
        bool is_system_include);
@@ -398,7 +470,7 @@

The ctx parameter provides a reference to the context_type used during instantiation of the preprocessing iterators by the user.

The parameter message references the argument token sequence of the encountered #error directive.

If the return value is false, the library throws a preprocessor - exception of the type error_directive (normal execution continues), if the return value is true the execution continues as if no #error directive has been found and the overall directive is replaced by a single newline.

+ exception of the type error_directive (normal execution continues), if the return value is true the execution continues as if no #error directive has been found, and the overall directive is replaced by a single newline.

found_line_directive

    template <typename ContextT, typename ContainerT>
    void found_line_directive(ContextT const& ctx, 
        ContainerT const &arguments, unsigned int line,
        std::string const& filename);
@@ -411,6 +483,33 @@
   
The parameter filename references the recognized file name from the #line directive (if there was one given).

   

 
+
emit_line_directive

+
+    template <typename ContextT, typename ContainerT>
+    bool emit_line_directive(ContextT const& ctx, ContainerT &pending, 
+        typename ContextT::token_type const& act_token);
+

+

+  
The function emit_line_directive is called whenever a #line directive
+  has to be emitted into the generated output.

+
+
The parameter  ctx is a reference to the context object used for
+  instantiating the preprocessing iterators by the user.

+
+
The parameter  pending may be used to push tokens back into the input
+stream, which are to be used instead of the default output generated
+  for the #line directive.

+
+  
The parameter  act_token contains the actual #pragma token, which may
+be used for error output. The line number stored in this token can be
+    used as the line number emitted as part of the #line directive.

+
+  
If the return value is  false, a default #line directive is emitted
+by the library. A return value of  true will inhibit any further
+actions, the tokens contained in  pending will be copied verbatim
+    to the output.

+

+
 
diff --git a/doc/samples.html b/doc/samples.html
index 3f7d65a..dc61ed5 100644
--- a/doc/samples.html
+++ b/doc/samples.html
@@ -24,7 +24,7 @@
     

   
     

   

 

-
The Wave library contains several samples illustrating how to use the different features. This section describes these samples and its main characteristics. 

+
The Wave library contains several samples illustrating how to use the different features. This section describes these samples and their main characteristics. 

 
The quick_start sample

  
The quick_start sample shows a minimal way to use the Wave preprocessor library. It simply opens the file given as the first command line argument, preprocesses it assuming that there aren't any additional include paths or macros defined and outputs the textual representation of the tokens generated from the given input file. This sample may be used to introduce yourself to Wave, because it does not contain all the potential additional complexity exposed by more complex samples. 

  
The lexed_tokens sample

@@ -49,7 +49,7 @@
     #  include "file_b.h"
     #endif

The advanced_hooks sample

-

The advanced_hooks sample is based on the quick_start sample mentioned above. It shows how you may want to use the advanced preprocessing hooks of the Wave library to get in the output not only the preprocessed tokens from the evaluated conditional blocks, but also the tokens recognized inside the non-evaluated conditional blocks. To make the generated token stream useful for further processing the tokens from the non-evaluated conditional blocks are commented out.

+

The advanced_hooks sample is based on the quick_start sample mentioned above. It shows how you may want to use the advanced preprocessing hooks of the Wave library to get in the output not only the preprocessed tokens from the evaluated conditional blocks, but also the tokens recognized inside the non-evaluated conditional blocks. To make the generated token stream useful for further processing the tokens from the non-evaluated conditional blocks are commented out.

Here is a small sample what the advanced_hooks sample does. Consider the following input: 

    #define TEST 1
     #if defined(TEST)
@@ -67,7 +67,7 @@
 
The  wave sample

 
Because of its general usefulness the wave sample is not located in the sample directory of the library, but inside the tools directory of Boost. The wave sample is usable as a full fledged preprocessor executable 
   on top of any other C++ compiler. It outputs the textual representation of the 
-preprocessed tokens generated from a given input file. It is described in more details here. 

+preprocessed tokens generated from a given input file. It is described in more detail here. 

 
The  waveidl sample 

 
The main point of the waveidl sample is to show, how a completely independent lexer type may be used in conjunction with the default token type of the Wave library. The lexer used in this sample is supposed to be used for an IDL language based preprocessor. It is based on the Re2C tool too, but recognizes a different set of tokens as the default C++ lexer contained within the Wave library. So this lexer does not recognize any keywords (except true and false, which are needed by the preprocessor itself). This is needed because there exist different IDL languages, where identifiers of one language may be keywords of others. Certainly this implies to postpone keyword identification after the preprocessing, but allows to use Wave for all of the IDL derivatives. 

 
It is only possible to use the Wave library to write an IDL preprocessor, because the token sets for both languages are very similar. The tokens to be recognized by the waveidl IDL language preprocessor is nearly a complete subset of the full C++ token set. 

@@ -95,7 +95,7 @@ preprocessed tokens generated from a given input file. It is described in more d
 
It may be a good starting point for a grammar that can be used for things like   reverse engineering as some UML modelling tools do. Or whatever use you may find   for a grammar that gives you a list of all templates and classes in a file and their members. 

 

 
The check_macro_naming sample

-
The check_macro_naming sample demonstrates the use of context hooks to understand how macros are defined within a codebase. Some projects (such as Boost itself) have conventions on the names of macros. This sample will recursively search a directory, looking for header files and searching each for macro definitions. Any that do not match the supplied convention (which defaults to ^BOOST_.*) are reported, along with an annotation if they are used as an include guard. The user can also specify any number of directories to ignore in the process.

+
The check_macro_naming sample demonstrates the use of context hooks to understand how macros are defined within a codebase. Some projects (such as Boost itself) have conventions on the names of macros. This sample will recursively search a directory, looking for header files and searching each for macro definitions. Any that do not match the supplied convention (which defaults to ^BOOST_.*) are reported, along with an annotation if they are used as an include guard. The user can also specify any number of directories to ignore in the process.

 
Command line syntax:

 
 Usage: check_macro_naming [options] directory:
@@ -113,6 +113,33 @@ TRACE_CPP_TIME_CONVERSION include/boost/wave/util/time_conversion_helper.hpp
 spirit_append_actor include/boost/wave/util/time_conversion_helper.hpp
 spirit_assign_actor include/boost/wave/util/time_conversion_helper.hpp
 

+
The custom_directives sample

+
The custom_directives sample shows how to define your own directives by overriding the found_unknown_directive hook to add #version and #extension directives (from GLSL).

+
The emit_custom_line_directives sample

+
The emit_custom_line_directives sample shows how to use the emit_line_directive preprocessing hook
+  to customize the format of any generated #line directive. The sample will
+  emit #line directives formatted compatible with those generated by gcc:

+
+    # <lineno> <rel_file_name>
+

+
The preprocess_pragma_output sample

+
The preprocess_pragma_output sample demonstrates using the interpret_pragma hook to install custom pragma handling code. Specifically it defines a new #pragma wave pp() that interprets a string in the current preprocessor context. The sample input uses this mechanism to define new macros from within a quoted string passed to pp.
+
The real_positions sample

+The real_positions sample demonstrates:
+
+
by showing how to correct the positions inside the returned tokens in
+    a way that these appear to be consecutive (ignoring positions from macro
+  definitions).

+
+
The token_statistics sample

+The token_statistics sample collects counts of each type of token in the preprocessed input and reports it. It demonstrates:
+