From 4dea2a8ad5d97bcdb13234eed666cfb2459d9277 Mon Sep 17 00:00:00 2001 From: Daniel James Date: Wed, 2 Nov 2011 07:57:47 +0000 Subject: [PATCH] Quickbook: Keep files in memory, stop tracking position in iterators. Files are now permanently loaded. Quickbook substrings are stored as references into the file. Now positions are stored as iterators into the original file, the line and column is calculated when messages are output. This doesn't have much effect on efficiency but it simplifies a few things. [SVN r75253] --- src/Jamfile.v2 | 1 + src/actions.cpp | 282 ++++++++++++++++++----------------- src/actions.hpp | 44 +++--- src/actions_class.cpp | 10 +- src/actions_class.hpp | 2 +- src/actions_state.hpp | 2 +- src/code_snippet.cpp | 49 +++--- src/doc_info_actions.cpp | 105 ++++++------- src/doc_info_grammar.cpp | 2 +- src/files.cpp | 126 ++++++++++++++++ src/files.hpp | 37 +++++ src/fwd.hpp | 4 +- src/grammar.hpp | 2 +- src/input_path.cpp | 11 ++ src/input_path.hpp | 3 + src/iterator.hpp | 94 +++++++----- src/quickbook.cpp | 66 ++++---- src/quickbook.hpp | 7 +- src/string_ref.cpp | 6 + src/string_ref.hpp | 2 + src/syntax_highlight.cpp | 3 +- src/template_stack.cpp | 2 - src/template_stack.hpp | 10 +- src/utils.cpp | 103 ------------- src/utils.hpp | 21 +-- src/values.cpp | 257 ++++++++++--------------------- src/values.hpp | 41 ++--- src/values_parse.hpp | 19 ++- test/unit/Jamfile.v2 | 8 +- test/unit/iterator_tests.cpp | 71 --------- test/unit/values_test.cpp | 68 +-------- 31 files changed, 658 insertions(+), 800 deletions(-) create mode 100644 src/files.cpp create mode 100644 src/files.hpp delete mode 100644 test/unit/iterator_tests.cpp diff --git a/src/Jamfile.v2 b/src/Jamfile.v2 index eaa4610..9372559 100644 --- a/src/Jamfile.v2 +++ b/src/Jamfile.v2 @@ -26,6 +26,7 @@ exe quickbook doc_info_actions.cpp actions_class.cpp utils.cpp + files.cpp string_ref.cpp input_path.cpp values.cpp diff --git a/src/actions.cpp b/src/actions.cpp index 114f07b..bde3a9c 100644 --- a/src/actions.cpp +++ b/src/actions.cpp @@ -23,6 +23,7 @@ #include "quickbook.hpp" #include "actions.hpp" #include "utils.hpp" +#include "files.hpp" #include "markups.hpp" #include "actions_class.hpp" #include "actions_state.hpp" @@ -69,7 +70,7 @@ namespace quickbook void explicit_list_action(quickbook::actions&, value); void header_action(quickbook::actions&, value); void begin_section_action(quickbook::actions&, value); - void end_section_action(quickbook::actions&, value, file_position); + void end_section_action(quickbook::actions&, value, string_iterator); void block_action(quickbook::actions&, value); void block_empty_action(quickbook::actions&, value); void macro_definition_action(quickbook::actions&, value); @@ -77,7 +78,7 @@ namespace quickbook void variable_list_action(quickbook::actions&, value); void table_action(quickbook::actions&, value); void xinclude_action(quickbook::actions&, value); - void include_action(quickbook::actions&, value, file_position); + void include_action(quickbook::actions&, value, string_iterator); void image_action(quickbook::actions&, value); void anchor_action(quickbook::actions&, value); void link_action(quickbook::actions&, value); @@ -85,9 +86,9 @@ namespace quickbook void footnote_action(quickbook::actions&, value); void raw_phrase_action(quickbook::actions&, value); void source_mode_action(quickbook::actions&, value); - void do_template_action(quickbook::actions&, value, file_position); + void do_template_action(quickbook::actions&, value, string_iterator); - void element_action::operator()(iterator first, iterator) const + void element_action::operator()(parse_iterator first, parse_iterator) const { value_consumer values = actions.values.release(); if(!values.check() || !actions.conditional) return; @@ -112,7 +113,7 @@ namespace quickbook case block_tags::begin_section: return begin_section_action(actions, v); case block_tags::end_section: - return end_section_action(actions, v, first.get_position()); + return end_section_action(actions, v, first.base()); case block_tags::blurb: case block_tags::preformatted: case block_tags::blockquote: @@ -136,7 +137,7 @@ namespace quickbook return xinclude_action(actions, v); case block_tags::import: case block_tags::include: - return include_action(actions, v, first.get_position()); + return include_action(actions, v, first.base()); case phrase_tags::image: return image_action(actions, v); case phrase_tags::anchor: @@ -169,28 +170,27 @@ namespace quickbook case source_mode_tags::teletype: return source_mode_action(actions, v); case template_tags::template_: - return do_template_action(actions, v, first.get_position()); + return do_template_action(actions, v, first.base()); default: break; } } // Handles line-breaks (DEPRECATED!!!) - void break_action::operator()(iterator first, iterator) const + void break_action::operator()(parse_iterator first, parse_iterator) const { write_anchors(actions, phrase); - file_position const pos = first.get_position(); if(*first == '\\') { - detail::outwarn(actions.filename, pos.line) - << "in column:" << pos.column << ", " + detail::outwarn(actions.current_file, first.base()) + //<< "in column:" << pos.column << ", " << "'\\n' is deprecated, pleases use '[br]' instead" << ".\n"; } if(!actions.warned_about_breaks) { - detail::outwarn(actions.filename, pos.line) + detail::outwarn(actions.current_file, first.base()) << "line breaks generate invalid boostbook " "(will only note first occurrence).\n"; @@ -200,9 +200,9 @@ namespace quickbook phrase << detail::get_markup(phrase_tags::break_mark).pre; } - void error_message_action::operator()(iterator first, iterator last) const + void error_message_action::operator()(parse_iterator first, parse_iterator last) const { - file_position const pos = first.get_position(); + file_position const pos = get_position(first, actions.current_file->source); std::string value(first, last); std::string formatted_message = message; @@ -210,15 +210,16 @@ namespace quickbook boost::replace_all(formatted_message, "%c", boost::lexical_cast(pos.column)); - detail::outerr(actions.filename, pos.line) + detail::outerr(actions.current_file->path, pos.line) << detail::utf8(formatted_message) << std::endl; ++actions.error_count; } - void error_action::operator()(iterator first, iterator /*last*/) const + void error_action::operator()(parse_iterator first, parse_iterator /*last*/) const { - file_position const pos = first.get_position(); - detail::outerr(actions.filename, pos.line) + file_position const pos = get_position(first, actions.current_file->source); + + detail::outerr(actions.current_file->path, pos.line) << "Syntax Error near column " << pos.column << ".\n"; ++actions.error_count; } @@ -413,8 +414,10 @@ namespace quickbook if (saved_conditional) { - actions.conditional = - find(actions.macro, values.consume().get_quickbook().c_str()); + string_ref macro1 = values.consume().get_quickbook(); + std::string macro(macro1.begin(), macro1.end()); + + actions.conditional = find(actions.macro, macro.c_str()); if (!actions.conditional) { actions.phrase.push(); @@ -473,7 +476,7 @@ namespace quickbook std::string content = values.consume().get_boostbook(); values.finish(); - char mark = mark_value.get_quickbook()[0]; + char mark = *mark_value.get_quickbook().begin(); assert(mark == '*' || mark == '#'); if(list_indent == -1) { @@ -508,10 +511,9 @@ namespace quickbook if (mark != list_marks.top().first) // new_indent == list_indent { - file_position const pos = mark_value.get_position(); - detail::outerr(actions.filename, pos.line) - << "Illegal change of list style near column " << pos.column << ".\n"; - detail::outwarn(actions.filename, pos.line) + detail::outerr(mark_value.get_file(), mark_value.get_position()) + << "Illegal change of list style.\n"; + detail::outwarn(mark_value.get_file(), mark_value.get_position()) << "Ignoring change of list style" << std::endl; ++actions.error_count; } @@ -550,7 +552,7 @@ namespace quickbook // TODO: No need to check suppress since this is only used in the syntax // highlighter. I should move this or something. - void span::operator()(iterator first, iterator last) const + void span::operator()(parse_iterator first, parse_iterator last) const { if (name) out << ""; while (first != last) @@ -558,25 +560,25 @@ namespace quickbook if (name) out << ""; } - void span_start::operator()(iterator first, iterator last) const + void span_start::operator()(parse_iterator first, parse_iterator last) const { out << ""; while (first != last) detail::print_char(*first++, out.get()); } - void span_end::operator()(iterator first, iterator last) const + void span_end::operator()(parse_iterator first, parse_iterator last) const { while (first != last) detail::print_char(*first++, out.get()); out << ""; } - void unexpected_char::operator()(iterator first, iterator last) const + void unexpected_char::operator()(parse_iterator first, parse_iterator last) const { - file_position const pos = first.get_position(); + file_position const pos = get_position(first, actions.current_file->source); - detail::outwarn(actions.filename, pos.line) + detail::outwarn(actions.current_file->path, pos.line) << "in column:" << pos.column << ", unexpected character: " << detail::utf8(first, last) << "\n"; @@ -622,18 +624,18 @@ namespace quickbook detail::print_space(ch, out.get()); } - void space::operator()(iterator first, iterator last) const + void space::operator()(parse_iterator first, parse_iterator last) const { while (first != last) detail::print_space(*first++, out.get()); } - void pre_escape_back::operator()(iterator, iterator) const + void pre_escape_back::operator()(parse_iterator, parse_iterator) const { escape_actions.phrase.push(); // save the stream } - void post_escape_back::operator()(iterator, iterator) const + void post_escape_back::operator()(parse_iterator, parse_iterator) const { write_anchors(escape_actions, escape_actions.phrase); out << escape_actions.phrase.str(); @@ -645,7 +647,7 @@ namespace quickbook actions.source_mode = source_mode_tags::name(source_mode.get_tag()); } - void code_action::operator()(iterator first, iterator last) const + void code_action::operator()(parse_iterator first, parse_iterator last) const { write_anchors(actions, out); @@ -655,8 +657,16 @@ namespace quickbook if (program.size() == 0) return; // Nothing left to do here. The program is empty. - iterator first_(program.begin(), first.get_position()); - iterator last_(program.end()); + file fake_file; + + fake_file.path = actions.current_file->path; + fake_file.source = program; + + parse_iterator first_(fake_file.source.begin()); + parse_iterator last_(fake_file.source.end()); + + file const* saved_file = &fake_file; + boost::swap(actions.current_file, saved_file); // TODO: Shouldn't phrase be empty here? Why would it be output // after the code block? @@ -667,6 +677,7 @@ namespace quickbook std::string str = syntax_highlight(first_, last_, actions, actions.source_mode); phrase.swap(save); + boost::swap(actions.current_file, saved_file); // // We must not place a \n after the tag @@ -677,7 +688,7 @@ namespace quickbook out << "\n"; } - void inline_code_action::operator()(iterator first, iterator last) const + void inline_code_action::operator()(parse_iterator first, parse_iterator last) const { write_anchors(actions, out); @@ -701,14 +712,14 @@ namespace quickbook detail::print_char(ch, phrase.get()); } - void plain_char_action::operator()(iterator first, iterator /*last*/) const + void plain_char_action::operator()(parse_iterator first, parse_iterator /*last*/) const { write_anchors(actions, phrase); detail::print_char(*first, phrase.get()); } - void escape_unicode_action::operator()(iterator first, iterator last) const + void escape_unicode_action::operator()(parse_iterator first, parse_iterator last) const { write_anchors(actions, phrase); @@ -747,7 +758,7 @@ namespace quickbook pair.finish(); if(!attributes.insert(std::make_pair(name.get_quickbook(), value)).second) { - detail::outwarn(actions.filename, name.get_position().line) + detail::outwarn(name.get_file(), name.get_position()) << "Duplicate image attribute: " << detail::utf8(name.get_quickbook()) << std::endl; @@ -768,7 +779,7 @@ namespace quickbook if(fileref.find('\\') != std::string::npos) { - detail::outwarn(actions.filename, attributes["fileref"].get_position().line) + detail::outwarn(attributes["fileref"].get_file(), attributes["fileref"].get_position()) << "Image path isn't portable: '" << detail::utf8(fileref) << "'" @@ -952,11 +963,9 @@ namespace quickbook identifier, template_values, body, - actions.filename, &actions.templates.top_scope()))) { - file_position const pos = body.get_position(); - detail::outerr(actions.filename, pos.line) + detail::outwarn(body.get_file(), body.get_position()) << "Template Redefinition: " << detail::utf8(identifier) << std::endl; ++actions.error_count; } @@ -964,7 +973,7 @@ namespace quickbook namespace { - iterator find_first_seperator(iterator begin, iterator end) + string_iterator find_first_seperator(string_iterator begin, string_iterator end) { if(qbk_version_n < 105) { for(;begin != end; ++begin) @@ -1011,9 +1020,9 @@ namespace quickbook return begin; } - std::pair find_seperator(iterator begin, iterator end) + std::pair find_seperator(string_iterator begin, string_iterator end) { - iterator first = begin = find_first_seperator(begin, end); + string_iterator first = begin = find_first_seperator(begin, end); for(;begin != end; ++begin) { @@ -1032,11 +1041,10 @@ namespace quickbook return std::make_pair(first, begin); } - bool break_arguments( + void break_arguments( std::vector& args , std::vector const& params , fs::path const& filename - , file_position const& pos ) { // Quickbook 1.4-: If there aren't enough parameters seperated by @@ -1057,34 +1065,23 @@ namespace quickbook // arguments, or if there are no more spaces left. template_body& body = args.back(); - iterator begin = body.content.get_quickbook_range().begin(); - iterator end = body.content.get_quickbook_range().end(); + string_iterator begin = body.content.get_quickbook().begin(); + string_iterator end = body.content.get_quickbook().end(); - std::pair pos = + std::pair pos = find_seperator(begin, end); if (pos.second == end) break; template_body second( - qbk_value(pos.second, end, template_tags::phrase), - body.filename); + qbk_value_ref(body.content.get_file(), + pos.second, end, template_tags::phrase)); - body.content = qbk_value(begin, pos.first, - body.content.get_tag()); + // TODO: Make sure that this is overwriting a reference, not + // a value. + body.content = qbk_value_ref(body.content.get_file(), + begin, pos.first, body.content.get_tag()); args.push_back(second); } } - - if (args.size() != params.size()) - { - detail::outerr(filename, pos.line) - << "Invalid number of arguments passed. Expecting: " - << params.size() - << " argument(s), got: " - << args.size() - << " argument(s) instead." - << std::endl; - return false; - } - return true; } std::pair::const_iterator> @@ -1092,7 +1089,7 @@ namespace quickbook std::vector const& args , std::vector const& params , template_scope const& scope - , file_position const& pos + , string_iterator first , quickbook::actions& actions ) { @@ -1106,7 +1103,7 @@ namespace quickbook if (!actions.templates.add( template_symbol(*tpl, empty_params, *arg, &scope))) { - detail::outerr(actions.filename, pos.line) + detail::outerr(actions.current_file, first) << "Duplicate Symbol Found" << std::endl; ++actions.error_count; return std::make_pair(false, tpl); @@ -1122,32 +1119,38 @@ namespace quickbook , quickbook::actions& actions ) { - // How do we know if we are to parse the template as a block or - // a phrase? We apply a simple heuristic: if the body starts with - // a newline, then we regard it as a block, otherwise, we parse - // it as a phrase. - // - // Note: this is now done in the grammar. - - // TODO: For escape, should this be surrounded in escape comments? - if (body.type == template_body::raw_output || escape) + if (escape) { + // TODO: Should this be surrounded in escape comments? // escape the body of the template // we just copy out the literal body (body.is_block() ? actions.out : actions.phrase) << body.content.get_quickbook(); return true; } + else if (body.type == template_body::raw_output) + { + (body.is_block() ? actions.out : actions.phrase) << body.content.get_boostbook(); + return true; + } else { - actions.filename = body.filename; - iterator first = body.content.get_quickbook_range().begin(); - iterator last = body.content.get_quickbook_range().end(); - - return cl::parse(first, last, + file const* saved_current_file = actions.current_file; + + actions.current_file = body.content.get_file(); + string_ref source = body.content.get_quickbook(); + + parse_iterator first(source.begin()); + parse_iterator last(source.end()); + + bool r = cl::parse(first, last, body.is_block() ? actions.grammar().block : actions.grammar().simple_phrase ).full; + + boost::swap(actions.current_file, saved_current_file); + + return r; } } } @@ -1155,7 +1158,7 @@ namespace quickbook void call_template(quickbook::actions& actions, bool template_escape, template_symbol const* symbol, std::vector const& args, - file_position pos) + string_iterator first) { // The template arguments should have the scope that the template was // called from, not the template's own scope. @@ -1173,7 +1176,7 @@ namespace quickbook ++actions.template_depth; if (actions.template_depth > actions.max_template_depth) { - detail::outerr(actions.filename, pos.line) + detail::outerr(actions.current_file, first) << "Infinite loop detected" << std::endl; ++actions.error_count; return; @@ -1195,18 +1198,19 @@ namespace quickbook bool get_arg_result; std::vector::const_iterator tpl; boost::tie(get_arg_result, tpl) = - get_arguments(args, symbol->params, call_scope, pos, actions); + get_arguments(args, symbol->params, call_scope, first, actions); if (!get_arg_result) { return; } + /////////////////////////////////// // parse the template body: if (!parse_template(symbol->body, template_escape, actions)) { - detail::outerr(actions.filename, pos.line) + detail::outerr(actions.current_file, first) << "Expanding " << (symbol->body.is_block() ? "block" : "phrase") << " template: " << detail::utf8(symbol->identifier) << std::endl @@ -1221,7 +1225,7 @@ namespace quickbook if (actions.ids.section_level() != actions.min_section_level) { - detail::outerr(actions.filename, pos.line) + detail::outerr(actions.current_file, first) << "Mismatched sections in template " << detail::utf8(symbol->identifier) << std::endl; @@ -1247,8 +1251,8 @@ namespace quickbook void call_code_snippet(quickbook::actions& actions, bool template_escape, template_symbol const* symbol, - file_position pos) - { + string_iterator first) + { assert(symbol->body.is_block()); std::vector callout_ids; @@ -1266,13 +1270,13 @@ namespace quickbook code += "linkends=\"" + callout_id2 + "\" />"; args.push_back(template_body( - qbk_value(code, pos, template_tags::phrase), - actions.filename, template_body::raw_output)); + bbk_value(code, template_tags::phrase), + template_body::raw_output)); callout_ids.push_back(callout_id1); callout_ids.push_back(callout_id2); } - - call_template(actions, template_escape, symbol, args, pos); + + call_template(actions, template_escape, symbol, args, first); std::string block; @@ -1291,11 +1295,11 @@ namespace quickbook ++actions.template_depth; bool r = parse_template( - template_body(c, symbol->body.filename), false, actions); - + template_body(c), false, actions); + if(!r) { - detail::outerr(symbol->body.filename, c.get_position().line) + detail::outerr(c.get_file(), c.get_position()) << "Expanding callout." << std::endl << "------------------begin------------------" << std::endl << detail::utf8(c.get_quickbook()) @@ -1321,7 +1325,7 @@ namespace quickbook } void do_template_action(quickbook::actions& actions, value template_list, - file_position pos) + string_iterator first) { // Get the arguments value_consumer values = template_list; @@ -1336,7 +1340,7 @@ namespace quickbook BOOST_FOREACH(value arg, values) { - args.push_back(template_body(arg, actions.filename)); + args.push_back(template_body(arg)); } values.finish(); @@ -1350,29 +1354,39 @@ namespace quickbook if (!symbol->callouts.check()) { // Break the arguments for a template - - if (!break_arguments(args, symbol->params, actions.filename, pos)) + + break_arguments(args, symbol->params, actions.current_file->path); + + if (args.size() != symbol->params.size()) { + detail::outerr(actions.current_file, first) + << "Invalid number of arguments passed. Expecting: " + << symbol->params.size() + << " argument(s), got: " + << args.size() + << " argument(s) instead." + << std::endl; + ++actions.error_count; return; } - call_template(actions, template_escape, symbol, args, pos); + call_template(actions, template_escape, symbol, args, first); } else { if (!args.empty()) { - detail::outerr(actions.filename, pos.line) + detail::outerr(actions.current_file, first) << "Arguments for code snippet." <\n"; } - void end_section_action(quickbook::actions& actions, value end_section, file_position pos) + void end_section_action(quickbook::actions& actions, value end_section, string_iterator first) { write_anchors(actions, actions.out); if (actions.ids.section_level() <= actions.min_section_level) { - detail::outerr(actions.filename, pos.line) + file_position const pos = get_position(first, actions.current_file->source); + + detail::outerr(actions.current_file->path, pos.line) << "Mismatched [endsect] near column " << pos.column << ".\n"; ++actions.error_count; @@ -1580,10 +1596,9 @@ namespace quickbook actions.ids.end_section(); } - void element_id_warning_action::operator()(iterator first, iterator) const + void element_id_warning_action::operator()(parse_iterator first, parse_iterator) const { - file_position const pos = first.get_position(); - detail::outwarn(actions.filename, pos.line) << "Empty id.\n"; + detail::outwarn(actions.current_file, first.base()) << "Empty id.\n"; } // Not a general purpose normalization function, just @@ -1667,7 +1682,7 @@ namespace quickbook if(path_text.find('\\') != std::string::npos) { - detail::outwarn(actions.filename, path.get_position().line) + detail::outwarn(path.get_file(), path.get_position()) << "Path isn't portable: " << detail::utf8(path_text) << std::endl; @@ -1692,7 +1707,7 @@ namespace quickbook { return path_difference( actions.xinclude_base, - actions.filename.parent_path() / path); + actions.current_file->path.parent_path() / path); } } @@ -1725,7 +1740,7 @@ namespace quickbook include_search_return include_search(std::string const & name, quickbook::actions const& actions) { - fs::path current = actions.filename.parent_path(); + fs::path current = actions.current_file->path.parent_path(); fs::path path = detail::generic_to_path(name); // If the path is relative, try and resolve it. @@ -1772,17 +1787,16 @@ namespace quickbook qbk_version_n >= 106u ? file_state::scope_callables : file_state::scope_macros); - actions.imported = (load_type == block_tags::import); - actions.filename = paths.filename; + actions.current_file = load(paths.filename); // Throws load_error actions.filename_relative = paths.filename_relative; + actions.imported = (load_type == block_tags::import); // update the __FILENAME__ macro *boost::spirit::classic::find(actions.macro, "__FILENAME__") = detail::path_to_generic(actions.filename_relative); // parse the file - quickbook::parse_file(actions.filename, actions, - include_doc_id, true); + quickbook::parse_file(actions, include_doc_id, true); // Don't restore source_mode on older versions. if (keep_inner_source_mode) state.source_mode = actions.source_mode; @@ -1796,7 +1810,7 @@ namespace quickbook void load_source_file(quickbook::actions& actions, include_search_return const& paths, value::tag_type load_type, - file_position pos, + string_iterator first, value const& include_doc_id = value()) { assert(load_type == block_tags::include || @@ -1804,7 +1818,7 @@ namespace quickbook std::string ext = paths.filename.extension().generic_string(); std::vector storage; - // Throws detail::load_error + // Throws load_error actions.error_count += load_snippets(paths.filename, storage, ext, load_type); @@ -1821,7 +1835,7 @@ namespace quickbook ts.parent = &actions.templates.top_scope(); if (!actions.templates.add(ts)) { - detail::outerr(ts.body.filename, ts.body.content.get_position().line) + detail::outerr(ts.body.content.get_file(), ts.body.content.get_position()) << "Template Redefinition: " << detail::utf8(tname) << std::endl; ++actions.error_count; } @@ -1837,7 +1851,7 @@ namespace quickbook if (tname == "!") { ts.parent = &actions.templates.top_scope(); - call_code_snippet(actions, false, &ts, pos); + call_code_snippet(actions, false, &ts, first); } } @@ -1845,7 +1859,7 @@ namespace quickbook } } - void include_action(quickbook::actions& actions, value include, file_position pos) + void include_action(quickbook::actions& actions, value include, string_iterator first) { write_anchors(actions, actions.out); @@ -1869,7 +1883,7 @@ namespace quickbook } else { - load_source_file(actions, paths, include.get_tag(), pos, include_doc_id); + load_source_file(actions, paths, include.get_tag(), first, include_doc_id); } } else @@ -1880,14 +1894,14 @@ namespace quickbook } else { - load_source_file(actions, paths, include.get_tag(), pos, include_doc_id); + load_source_file(actions, paths, include.get_tag(), first, include_doc_id); } } } - catch (detail::load_error& e) { + catch (load_error& e) { ++actions.error_count; - detail::outerr(actions.filename, pos.line) + detail::outerr(actions.current_file, first) << "Loading file:" << paths.filename << ": " @@ -1896,7 +1910,7 @@ namespace quickbook } } - void phrase_to_docinfo_action_impl::operator()(iterator first, iterator last, + void phrase_to_docinfo_action_impl::operator()(parse_iterator first, parse_iterator last, value::tag_type tag) const { write_anchors(actions, actions.phrase); @@ -1904,15 +1918,15 @@ namespace quickbook std::string encoded; actions.phrase.swap(encoded); actions.values.builder.insert( - qbk_bbk_value(first, last, encoded, tag)); + qbk_bbk_value(actions.current_file, first.base(), last.base(), encoded, tag)); } - void phrase_to_docinfo_action_impl::operator()(iterator first, iterator last) const + void phrase_to_docinfo_action_impl::operator()(parse_iterator first, parse_iterator last) const { return (*this)(first, last, value::default_tag); } - void to_value_action::operator()(iterator, iterator) const + void to_value_action::operator()(parse_iterator, parse_iterator) const { std::string value; diff --git a/src/actions.hpp b/src/actions.hpp index 96ea1e5..0d77771 100644 --- a/src/actions.hpp +++ b/src/actions.hpp @@ -52,12 +52,12 @@ namespace quickbook return quickbook_range(0, max_); } - // Throws detail::load_error + // Throws load_error int load_snippets(fs::path const& file, std::vector& storage, std::string const& extension, value::tag_type load_type); std::string syntax_highlight( - iterator first, iterator last, + parse_iterator first, parse_iterator last, actions& escape_actions, std::string const& source_mode); @@ -70,7 +70,7 @@ namespace quickbook , message(m) {} - void operator()(iterator, iterator) const; + void operator()(parse_iterator, parse_iterator) const; quickbook::actions& actions; std::string message; @@ -83,7 +83,7 @@ namespace quickbook error_action(quickbook::actions& actions) : actions(actions) {} - void operator()(iterator first, iterator /*last*/) const; + void operator()(parse_iterator first, parse_iterator last) const; error_message_action operator()(std::string const& message) { @@ -98,7 +98,7 @@ namespace quickbook element_action(quickbook::actions& actions) : actions(actions) {} - void operator()(iterator, iterator) const; + void operator()(parse_iterator, parse_iterator) const; quickbook::actions& actions; }; @@ -113,7 +113,7 @@ namespace quickbook : actions(actions) {} void operator()() const; - void operator()(iterator, iterator) const { (*this)(); } + void operator()(parse_iterator, parse_iterator) const { (*this)(); } quickbook::actions& actions; }; @@ -154,7 +154,7 @@ namespace quickbook span(char const* name, collector& out) : name(name), out(out) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; char const* name; collector& out; @@ -165,7 +165,7 @@ namespace quickbook span_start(char const* name, collector& out) : name(name), out(out) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; char const* name; collector& out; @@ -176,7 +176,7 @@ namespace quickbook span_end(collector& out) : out(out) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; collector& out; }; @@ -191,7 +191,7 @@ namespace quickbook : out(out) , actions(actions) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; collector& out; quickbook::actions& actions; @@ -220,8 +220,8 @@ namespace quickbook space(collector& out) : out(out) {} - void operator()(iterator first, iterator last) const; void operator()(char ch) const; + void operator()(parse_iterator first, parse_iterator last) const; collector& out; }; @@ -233,7 +233,7 @@ namespace quickbook pre_escape_back(actions& escape_actions, std::string& save) : escape_actions(escape_actions), save(save) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; actions& escape_actions; std::string& save; @@ -246,7 +246,7 @@ namespace quickbook post_escape_back(collector& out, actions& escape_actions, std::string& save) : out(out), escape_actions(escape_actions), save(save) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; collector& out; actions& escape_actions; @@ -263,7 +263,7 @@ namespace quickbook , actions(actions) {} void operator()(char ch) const; - void operator()(iterator first, iterator /*last*/) const; + void operator()(parse_iterator first, parse_iterator last) const; collector& phrase; quickbook::actions& actions; @@ -274,7 +274,7 @@ namespace quickbook escape_unicode_action(collector& phrase, quickbook::actions& actions) : phrase(phrase) , actions(actions) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; collector& phrase; quickbook::actions& actions; @@ -294,7 +294,7 @@ namespace quickbook { } - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; collector& out; collector& phrase; @@ -312,7 +312,7 @@ namespace quickbook , actions(actions) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; collector& out; quickbook::actions& actions; @@ -325,7 +325,7 @@ namespace quickbook break_action(collector& phrase, quickbook::actions& actions) : phrase(phrase), actions(actions) {} - void operator()(iterator f, iterator) const; + void operator()(parse_iterator f, parse_iterator) const; collector& phrase; quickbook::actions& actions; @@ -336,7 +336,7 @@ namespace quickbook element_id_warning_action(quickbook::actions& actions_) : actions(actions_) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; quickbook::actions& actions; }; @@ -359,8 +359,8 @@ namespace quickbook phrase_to_docinfo_action_impl(quickbook::actions& actions) : actions(actions) {} - void operator()(iterator first, iterator last) const; - void operator()(iterator first, iterator last, value::tag_type) const; + void operator()(parse_iterator first, parse_iterator last) const; + void operator()(parse_iterator first, parse_iterator last, value::tag_type) const; quickbook::actions& actions; }; @@ -372,7 +372,7 @@ namespace quickbook to_value_action(quickbook::actions& actions) : actions(actions) {} - void operator()(iterator first, iterator last) const; + void operator()(parse_iterator first, parse_iterator last) const; quickbook::actions& actions; }; diff --git a/src/actions_class.cpp b/src/actions_class.cpp index cbbfe0a..e762cdd 100644 --- a/src/actions_class.cpp +++ b/src/actions_class.cpp @@ -40,7 +40,7 @@ namespace quickbook , doc_type() , macro() , source_mode("c++") - , filename(filein_) + , current_file(0) , filename_relative(filein_.filename()) , template_depth(0) @@ -48,9 +48,7 @@ namespace quickbook , out(out_) , phrase() - , values() - - // actions + , values(¤t_file) , to_value(*this) , docinfo_value(*this) @@ -98,7 +96,7 @@ namespace quickbook , qbk_version(qbk_version_n) , imported(a.imported) , doc_type(a.doc_type) - , filename(a.filename) + , current_file(a.current_file) , filename_relative(a.filename_relative) , source_mode(a.source_mode) , macro() @@ -118,7 +116,7 @@ namespace quickbook boost::swap(qbk_version_n, qbk_version); boost::swap(a.imported, imported); boost::swap(a.doc_type, doc_type); - boost::swap(a.filename, filename); + boost::swap(a.current_file, current_file); boost::swap(a.filename_relative, filename_relative); boost::swap(a.source_mode, source_mode); if (scope & scope_output) { diff --git a/src/actions_class.hpp b/src/actions_class.hpp index 686ecf9..f654dc0 100644 --- a/src/actions_class.hpp +++ b/src/actions_class.hpp @@ -55,7 +55,7 @@ namespace quickbook std::string doc_type; string_symbols macro; std::string source_mode; - fs::path filename; + file const* current_file; fs::path filename_relative; // for the __FILENAME__ macro. // (relative to the original file // or include path). diff --git a/src/actions_state.hpp b/src/actions_state.hpp index 16684fd..d8a4801 100644 --- a/src/actions_state.hpp +++ b/src/actions_state.hpp @@ -37,7 +37,7 @@ namespace quickbook unsigned qbk_version; bool imported; std::string doc_type; - fs::path filename; + file const* current_file; fs::path filename_relative; std::string source_mode; string_symbols macro; diff --git a/src/code_snippet.cpp b/src/code_snippet.cpp index a952e95..50dd406 100644 --- a/src/code_snippet.cpp +++ b/src/code_snippet.cpp @@ -17,6 +17,7 @@ #include "template_stack.hpp" #include "actions.hpp" #include "values.hpp" +#include "files.hpp" namespace quickbook { @@ -25,20 +26,20 @@ namespace quickbook struct code_snippet_actions { code_snippet_actions(std::vector& storage, - fs::path const& filename, - char const* source_type) + file const* source_file, + char const* source_type) : callout_id(0) , storage(storage) - , filename(filename) + , source_file(source_file) , source_type(source_type) {} void pass_thru_char(char); - void pass_thru(iterator first, iterator last); - void escaped_comment(iterator first, iterator last); - void start_snippet(iterator first, iterator last); - void end_snippet(iterator first, iterator last); - void callout(iterator first, iterator last); + void pass_thru(string_iterator first, string_iterator last); + void escaped_comment(string_iterator first, string_iterator last); + void start_snippet(string_iterator first, string_iterator last); + void end_snippet(string_iterator first, string_iterator last); + void callout(string_iterator first, string_iterator last); void append_code(); void close_code(); @@ -83,7 +84,7 @@ namespace quickbook std::string code; std::string id; std::vector& storage; - fs::path filename; + file const* source_file; char const* const source_type; }; @@ -310,7 +311,7 @@ namespace quickbook }; int load_snippets( - fs::path const& file + fs::path const& filename , std::vector& storage // snippets are stored in a // vector of template_symbols , std::string const& extension @@ -319,16 +320,14 @@ namespace quickbook assert(load_type == block_tags::include || load_type == block_tags::import); - std::string code; - detail::load(file, code); // Throws detail::load_error. - - iterator first(code.begin()); - iterator last(code.end()); - bool is_python = extension == ".py"; - code_snippet_actions a(storage, file, is_python ? "[python]" : "[c++]"); + code_snippet_actions a(storage, load(filename), is_python ? "[python]" : "[c++]"); + // TODO: Should I check that parse succeeded? + string_iterator first(a.source_file->source.begin()); + string_iterator last(a.source_file->source.end()); + if(is_python) { boost::spirit::classic::parse(first, last, python_code_snippet_grammar(a)); } @@ -378,7 +377,7 @@ namespace quickbook } } - void code_snippet_actions::pass_thru(iterator first, iterator last) + void code_snippet_actions::pass_thru(string_iterator first, string_iterator last) { if(!snippet_stack) return; code.append(first, last); @@ -390,16 +389,16 @@ namespace quickbook code += c; } - void code_snippet_actions::callout(iterator first, iterator last) + void code_snippet_actions::callout(string_iterator first, string_iterator last) { if(!snippet_stack) return; code += "``[[callout" + boost::lexical_cast(callout_id) + "]]``"; - snippet_stack->callouts.insert(qbk_value(first, last, template_tags::block)); + snippet_stack->callouts.insert(qbk_value_ref(source_file, first, last, template_tags::block)); ++callout_id; } - void code_snippet_actions::escaped_comment(iterator first, iterator last) + void code_snippet_actions::escaped_comment(string_iterator first, string_iterator last) { if (!snippet_stack) { @@ -425,14 +424,14 @@ namespace quickbook } } - void code_snippet_actions::start_snippet(iterator, iterator) + void code_snippet_actions::start_snippet(string_iterator, string_iterator) { append_code(); push_snippet_data(id, callout_id); id.clear(); } - void code_snippet_actions::end_snippet(iterator first, iterator) + void code_snippet_actions::end_snippet(string_iterator first, string_iterator) { // TODO: Error? if(!snippet_stack) return; @@ -462,9 +461,7 @@ namespace quickbook } // TODO: Save position in start_snippet - template_symbol symbol(snippet->id, params, - qbk_value(body, first.get_position(), template_tags::block), - filename); + template_symbol symbol(snippet->id, params, qbk_value(body, template_tags::block)); symbol.callouts = callouts; storage.push_back(symbol); diff --git a/src/doc_info_actions.cpp b/src/doc_info_actions.cpp index 3a5276d..40cb678 100644 --- a/src/doc_info_actions.cpp +++ b/src/doc_info_actions.cpp @@ -14,6 +14,7 @@ #include #include "quickbook.hpp" #include "utils.hpp" +#include "files.hpp" #include "input_path.hpp" #include "actions_class.hpp" #include "doc_info_tags.hpp" @@ -120,7 +121,7 @@ namespace quickbook if(!duplicates.empty()) { - detail::outwarn(actions.filename,1) + detail::outwarn(actions.current_file->path,1) << (duplicates.size() > 1 ? "Duplicate attributes" : "Duplicate attribute") << ":" << detail::utf8(boost::algorithm::join(duplicates, ", ")) @@ -128,15 +129,24 @@ namespace quickbook ; } + std::string include_doc_id_, id_; + + if (!include_doc_id.empty()) + include_doc_id_.assign( + include_doc_id.get_quickbook().begin(), + include_doc_id.get_quickbook().end()); + + if (!id.empty()) + id_.assign( + id.get_quickbook().begin(), + id.get_quickbook().end()); + // if we're ignoring the document info, just start the file // and we're done. if (!docinfo_type) { - actions.ids.start_file( - include_doc_id.empty() ? "" : include_doc_id.get_quickbook(), - id.empty() ? "" : id.get_quickbook(), - actions.doc_title_qbk); + actions.ids.start_file(include_doc_id_, id_, actions.doc_title_qbk); return; } @@ -151,7 +161,7 @@ namespace quickbook qbk_major_version = 1; qbk_minor_version = 1; qbk_version_n = 101; - detail::outwarn(actions.filename,1) + detail::outwarn(actions.current_file->path,1) << "Quickbook version undefined. " "Version 1.1 is assumed" << std::endl; } @@ -168,13 +178,13 @@ namespace quickbook if (qbk_version_n == 106) { - detail::outwarn(actions.filename,1) + detail::outwarn(actions.current_file->path,1) << "Quickbook 1.6 is still under development and is " "likely to change in the future." << std::endl; } else if(qbk_version_n < 100 || qbk_version_n > 106) { - detail::outerr(actions.filename,1) + detail::outerr(actions.current_file->path,1) << "Unknown version of quickbook: quickbook " << qbk_major_version << "." @@ -185,9 +195,7 @@ namespace quickbook id_manager::start_file_info start_file_info = actions.ids.start_file_with_docinfo( - qbk_version_n, - include_doc_id.empty() ? "" : include_doc_id.get_quickbook(), - id.empty() ? "" : id.get_quickbook(), + qbk_version_n, include_doc_id_, id_, actions.doc_title_qbk); // if we're ignoring the document info, we're done. @@ -202,33 +210,6 @@ namespace quickbook assert(doc_title.check() && !actions.doc_type.empty() && !start_file_info.doc_id.empty()); - // Set defaults for dirname + last_revision - - if (dirname.empty() && actions.doc_type == "library") { - if (!id.empty()) { - dirname = id; - } - else { - dirname = qbk_bbk_value(start_file_info.doc_id, - doc_info_attributes::dirname); - } - } - - if (last_revision.empty()) - { - // default value for last-revision is now - - char strdate[64]; - strftime( - strdate, sizeof(strdate), - (debug_mode ? - "DEBUG MODE Date: %Y/%m/%d %H:%M:%S $" : - "$" /* prevent CVS substitution */ "Date: %Y/%m/%d %H:%M:%S $"), - current_gm_time - ); - last_revision = qbk_bbk_value(strdate, doc_info_attributes::last_revision); - } - // Warn about invalid fields if (actions.doc_type != "library") @@ -246,7 +227,7 @@ namespace quickbook if(!invalid_attributes.empty()) { - detail::outwarn(actions.filename,1) + detail::outwarn(actions.current_file->path,1) << (invalid_attributes.size() > 1 ? "Invalid attributes" : "Invalid attribute") << " for '" << detail::utf8(actions.doc_type) << " document info': " @@ -285,16 +266,40 @@ namespace quickbook out << " name=\"" << doc_info_output(doc_title, 106) << "\"\n"; } - if(!dirname.empty()) + // Set defaults for dirname + last_revision + + if(!dirname.empty() || actions.doc_type == "library") { - out << " dirname=\"" - << doc_info_output(dirname, 106) - << "\"\n"; + out << " dirname=\""; + if (!dirname.empty()) + out << doc_info_output(dirname, 106); + else + out << start_file_info.doc_id; + out << "\"\n"; } - out << " last-revision=\"" - << doc_info_output(last_revision, 106) - << "\" \n" + out << " last-revision=\""; + if (!last_revision.empty()) + { + out << doc_info_output(last_revision, 106); + } + else + { + // default value for last-revision is now + + char strdate[64]; + strftime( + strdate, sizeof(strdate), + (debug_mode ? + "DEBUG MODE Date: %Y/%m/%d %H:%M:%S $" : + "$" /* prevent CVS substitution */ "Date: %Y/%m/%d %H:%M:%S $"), + current_gm_time + ); + + out << strdate; + } + + out << "\" \n" << " xmlns:xi=\"http://www.w3.org/2001/XInclude\">\n"; std::ostringstream tmp; @@ -329,7 +334,8 @@ namespace quickbook while(copyright.check(doc_info_tags::copyright_year)) { - int year_start = copyright.consume().get_int(); + value year_start_value = copyright.consume(); + int year_start = year_start_value.get_int(); int year_end = copyright.check(doc_info_tags::copyright_year_end) ? copyright.consume().get_int() : @@ -338,8 +344,7 @@ namespace quickbook if (year_end < year_start) { ++actions.error_count; - detail::outerr(actions.filename, - copyright.begin()->get_position().line) + detail::outerr(actions.current_file, copyright.begin()->get_position()) << "Invalid year range: " << year_start << "-" @@ -436,7 +441,7 @@ namespace quickbook // Close any open sections. if (docinfo_type && actions.ids.section_level() > 1) { - detail::outwarn(actions.filename) + detail::outwarn(actions.current_file->path) << "Missing [endsect] detected at end of file." << std::endl; diff --git a/src/doc_info_grammar.cpp b/src/doc_info_grammar.cpp index 05ea6e4..3a4fc3f 100644 --- a/src/doc_info_grammar.cpp +++ b/src/doc_info_grammar.cpp @@ -58,7 +58,7 @@ namespace quickbook : l(l) {} - void operator()(iterator, iterator) const { + void operator()(parse_iterator, parse_iterator) const { l.attribute_rule = l.doc_fallback; l.attribute_tag = value::default_tag; } diff --git a/src/files.cpp b/src/files.cpp new file mode 100644 index 0000000..ac313f7 --- /dev/null +++ b/src/files.cpp @@ -0,0 +1,126 @@ +/*============================================================================= + Copyright (c) 2002 2004 2006 Joel de Guzman + Copyright (c) 2004 Eric Niebler + http://spirit.sourceforge.net/ + + Use, modification and distribution is subject to the Boost Software + License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +=============================================================================*/ +#include "files.hpp" +#include +#include +#include + +namespace quickbook +{ + // Read the first few bytes in a file to see it starts with a byte order + // mark. If it doesn't, then write the characters we've already read in. + // Although, given how UTF-8 works, if we've read anything in, the files + // probably broken. + + template + bool check_bom(InputIterator& begin, InputIterator end, + OutputIterator out, char const* chars, int length) + { + char const* ptr = chars; + + while(begin != end && *begin == *ptr) { + ++begin; + ++ptr; + --length; + if(length == 0) return true; + } + + // Failed to match, so write the skipped characters to storage: + while(chars != ptr) *out++ = *chars++; + + return false; + } + + template + std::string read_bom(InputIterator& begin, InputIterator end, + OutputIterator out) + { + if(begin == end) return ""; + + const char* utf8 = "\xef\xbb\xbf" ; + const char* utf32be = "\0\0\xfe\xff"; + const char* utf32le = "\xff\xfe\0\0"; + + unsigned char c = *begin; + switch(c) + { + case 0xEF: { // UTF-8 + return check_bom(begin, end, out, utf8, 3) ? "UTF-8" : ""; + } + case 0xFF: // UTF-16/UTF-32 little endian + return !check_bom(begin, end, out, utf32le, 2) ? "" : + check_bom(begin, end, out, utf32le + 2, 2) ? "UTF-32" : "UTF-16"; + case 0: // UTF-32 big endian + return check_bom(begin, end, out, utf32be, 4) ? "UTF-32" : ""; + case 0xFE: // UTF-16 big endian + return check_bom(begin, end, out, utf32be + 2, 2) ? "UTF-16" : ""; + default: + return ""; + } + } + + // Copy a string, converting mac and windows style newlines to unix + // newlines. + + template + void normalize(InputIterator begin, InputIterator end, + OutputIterator out) + { + std::string encoding = read_bom(begin, end, out); + + if(encoding != "UTF-8" && encoding != "") + throw load_error(encoding + + " is not supported. Please use UTF-8."); + + while(begin != end) { + if(*begin == '\r') { + *out++ = '\n'; + ++begin; + if(begin != end && *begin == '\n') ++begin; + } + else { + *out++ = *begin++; + } + } + } + + namespace + { + boost::unordered_map files; + } + + file const* load(fs::path const& filename) + { + boost::unordered_map::iterator pos; + bool inserted; + + boost::tie(pos, inserted) = files.emplace(filename, file()); + + if (inserted) + { + pos->second.path = filename; + + fs::ifstream in(filename, std::ios_base::in); + + if (!in) + throw load_error("Could not open input file."); + + // Turn off white space skipping on the stream + in.unsetf(std::ios::skipws); + + normalize( + std::istream_iterator(in), + std::istream_iterator(), + std::back_inserter(pos->second.source)); + } + + return &pos->second; + } +} diff --git a/src/files.hpp b/src/files.hpp new file mode 100644 index 0000000..c5b5ca7 --- /dev/null +++ b/src/files.hpp @@ -0,0 +1,37 @@ +/*============================================================================= + Copyright (c) 2002 2004 2006 Joel de Guzman + Copyright (c) 2004 Eric Niebler + Copyright (c) 2011 Daniel James + + Use, modification and distribution is subject to the Boost Software + License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at + http://www.boost.org/LICENSE_1_0.txt) +=============================================================================*/ + +#if !defined(BOOST_QUICKBOOK_FILES_HPP) +#define BOOST_QUICKBOOK_FILES_HPP + +#include +#include +#include + +namespace quickbook { + + namespace fs = boost::filesystem; + + struct load_error : std::runtime_error + { + explicit load_error(std::string const& arg) + : std::runtime_error(arg) {} + }; + + struct file + { + fs::path path; + std::string source; + }; + + file const* load(fs::path const& filename); +} + +#endif // BOOST_QUICKBOOK_FILES_HPP diff --git a/src/fwd.hpp b/src/fwd.hpp index 9d4a9e9..5e8a0e1 100644 --- a/src/fwd.hpp +++ b/src/fwd.hpp @@ -20,8 +20,10 @@ namespace quickbook struct collector; struct id_manager; struct section_info; + struct file; - typedef position_iterator iterator; + typedef std::string::const_iterator string_iterator; + typedef lookback_iterator parse_iterator; inline void ignore_variable(void const*) {} } diff --git a/src/grammar.hpp b/src/grammar.hpp index f749588..d54ccf0 100644 --- a/src/grammar.hpp +++ b/src/grammar.hpp @@ -21,7 +21,7 @@ namespace quickbook // spirit implementation detail, but since classic is no longer under // development, it won't change. And spirit 2 won't require such a hack. - typedef cl::scanner > scanner; struct grammar diff --git a/src/input_path.cpp b/src/input_path.cpp index 67da796..43701ff 100644 --- a/src/input_path.cpp +++ b/src/input_path.cpp @@ -10,6 +10,7 @@ #include #include "input_path.hpp" #include "utils.hpp" +#include "files.hpp" #if QUICKBOOK_WIDE_PATHS || QUICKBOOK_WIDE_STREAMS #include @@ -229,6 +230,11 @@ namespace detail { } } + ostream& outerr(file const* f, string_iterator pos) + { + return outerr(f->path, get_position(pos, f->source).line); + } + ostream& outwarn(fs::path const& file, int line) { if (line >= 0) @@ -243,4 +249,9 @@ namespace detail { return error_stream() << path_to_stream(file) << ": warning: "; } } + + ostream& outwarn(file const* f, string_iterator pos) + { + return outwarn(f->path, get_position(pos, f->source).line); + } }} diff --git a/src/input_path.hpp b/src/input_path.hpp index 7d1f4e3..2f59eeb 100644 --- a/src/input_path.hpp +++ b/src/input_path.hpp @@ -14,6 +14,7 @@ #include #include #include +#include "fwd.hpp" #if defined(__cygwin__) || defined(__CYGWIN__) # define QUICKBOOK_CYGWIN_PATHS 1 @@ -91,6 +92,8 @@ namespace quickbook ostream& outerr(); ostream& outerr(fs::path const& file, int line = -1); ostream& outwarn(fs::path const& file, int line = -1); + ostream& outerr(file const*, string_iterator); + ostream& outwarn(file const*, string_iterator); struct utf8_proxy { diff --git a/src/iterator.hpp b/src/iterator.hpp index 41d6b43..46e8568 100644 --- a/src/iterator.hpp +++ b/src/iterator.hpp @@ -26,49 +26,31 @@ namespace quickbook }; template - struct position_iterator + struct lookback_iterator : boost::forward_iterator_helper< - position_iterator, + lookback_iterator, typename boost::iterator_value::type, typename boost::iterator_difference::type, typename boost::iterator_pointer::type, typename boost::iterator_reference::type > { - position_iterator() {} - explicit position_iterator(Iterator base) - : original_(base), base_(base), previous_('\0'), position_() {} - explicit position_iterator(Iterator base, file_position const& position) - : original_(base), base_(base), previous_('\0'), position_(position) {} + lookback_iterator() {} + explicit lookback_iterator(Iterator base) + : original_(base), base_(base) {} + explicit lookback_iterator(Iterator base, file_position const& position) + : original_(base), base_(base) {} friend bool operator==( - position_iterator const& x, - position_iterator const& y) + lookback_iterator const& x, + lookback_iterator const& y) { return x.base_ == y.base_; } - position_iterator& operator++() + lookback_iterator& operator++() { - char val = *base_; - - if (val == '\r') { - ++position_.line; - position_.column = 1; - } - else if (val == '\n') { - if (previous_ != '\r') { - ++position_.line; - position_.column = 1; - } - } - else { - ++position_.column; - } - - previous_ = val; ++base_; - return *this; } @@ -77,10 +59,6 @@ namespace quickbook return *base_; } - file_position const& get_position() const { - return position_; - } - Iterator base() const { return base_; } @@ -96,9 +74,57 @@ namespace quickbook private: Iterator original_; Iterator base_; - char previous_; - file_position position_; }; + + template + file_position get_position( + Iterator iterator, + String const& source) + { + file_position pos; + Iterator line_begin = source.begin(); + + Iterator begin = source.begin(); + while (begin != iterator) + { + assert(begin != source.end()); + + if (*begin == '\r') + { + ++begin; + ++pos.line; + line_begin = begin; + } + else if (*begin == '\n') + { + ++begin; + ++pos.line; + line_begin = begin; + if (begin == iterator) break; + assert(begin != source.end()); + if (*begin == '\r') + { + ++begin; + line_begin = begin; + } + } + else + { + ++begin; + } + } + + pos.column = iterator - line_begin + 1; + return pos; + } + + template + file_position get_position( + lookback_iterator iterator, + String const& source) + { + return get_position(iterator.base(), source); + } } #endif diff --git a/src/quickbook.cpp b/src/quickbook.cpp index a1dc828..2af0647 100644 --- a/src/quickbook.cpp +++ b/src/quickbook.cpp @@ -12,6 +12,7 @@ #include "actions_class.hpp" #include "post_process.hpp" #include "utils.hpp" +#include "files.hpp" #include "input_path.hpp" #include "id_manager.hpp" #include @@ -57,8 +58,8 @@ namespace quickbook it != end; ++it) { // TODO: Set filename in actor??? - iterator first(it->begin()); - iterator last(it->end()); + parse_iterator first(it->begin()); + parse_iterator last(it->end()); cl::parse(first, last, actor.grammar().command_line_macro); // TODO: Check result? @@ -70,14 +71,10 @@ namespace quickbook // Parse a file // /////////////////////////////////////////////////////////////////////////// - void parse_file(fs::path const& filein_, actions& actor, - value include_doc_id, bool nested_file) + void parse_file(actions& actor, value include_doc_id, bool nested_file) { - std::string storage; - detail::load(filein_, storage); // Throws detail::load_error - - iterator first(storage.begin()); - iterator last(storage.end()); + parse_iterator first(actor.current_file->source.begin()); + parse_iterator last(actor.current_file->source.end()); // This is awkward. When not ignoring docinfo, the source_mode should be // reset, but the code doesn't find out if the docinfo is ignored until @@ -86,7 +83,7 @@ namespace quickbook std::string saved_source_mode = actor.source_mode; if (qbk_version_n >= 106) actor.source_mode = "c++"; - cl::parse_info info = cl::parse(first, last, actor.grammar().doc_info); + cl::parse_info info = cl::parse(first, last, actor.grammar().doc_info); docinfo_types docinfo_type = !nested_file ? docinfo_main : @@ -108,34 +105,13 @@ namespace quickbook if (!info.full) { - file_position const& pos = info.stop.get_position(); - detail::outerr(actor.filename, pos.line) + file_position const& pos = get_position(info.stop, actor.current_file->source); + detail::outerr(actor.current_file->path, pos.line) << "Syntax Error near column " << pos.column << ".\n"; ++actor.error_count; } } - static int - parse_document( - fs::path const& filein_, - actions& actor) - { - try { - parse_file(filein_, actor); - - if(actor.error_count) { - detail::outerr() - << "Error count: " << actor.error_count << ".\n"; - } - } - catch (detail::load_error& e) { - ++actor.error_count; - detail::outerr(filein_) << detail::utf8(e.what()) << std::endl; - } - - return !!actor.error_count; - } - static int parse_document( fs::path const& filein_ @@ -147,10 +123,28 @@ namespace quickbook { string_stream buffer; id_manager ids; - actions actor(filein_, xinclude_base_, buffer, ids); - set_macros(actor); - int result = parse_document(filein_, actor); + int result = 0; + + try { + actions actor(filein_, xinclude_base_, buffer, ids); + set_macros(actor); + + actor.current_file = load(filein_); // Throws load_error + + parse_file(actor); + + if(actor.error_count) { + detail::outerr() + << "Error count: " << actor.error_count << ".\n"; + } + + result = !!actor.error_count; + } + catch (load_error& e) { + detail::outerr(filein_) << detail::utf8(e.what()) << std::endl; + result = 1; + } std::string stage2 = ids.replace_placeholders(buffer.str()); diff --git a/src/quickbook.hpp b/src/quickbook.hpp index 49ae6c8..e86ea48 100644 --- a/src/quickbook.hpp +++ b/src/quickbook.hpp @@ -29,10 +29,9 @@ namespace quickbook extern std::vector include_path; extern std::vector preset_defines; - void parse_file(fs::path const& filein_, actions& actor, - value include_doc_id = value(), - bool nested_file = false); - + void parse_file(actions& actor, + value include_doc_id = value(), + bool nested_file = false); // Some initialisation methods // // Declared here to avoid including other headers diff --git a/src/string_ref.cpp b/src/string_ref.cpp index a8043b5..b477557 100644 --- a/src/string_ref.cpp +++ b/src/string_ref.cpp @@ -9,6 +9,7 @@ #include "string_ref.hpp" #include #include +#include namespace quickbook { @@ -21,4 +22,9 @@ namespace quickbook { return boost::lexicographical_compare(x, y); } + + std::ostream& operator<<(std::ostream& out, string_ref const& x) + { + return out.write(&*x.begin(), x.end() - x.begin()); + } } diff --git a/src/string_ref.hpp b/src/string_ref.hpp index bf1f3e3..b712006 100644 --- a/src/string_ref.hpp +++ b/src/string_ref.hpp @@ -11,6 +11,7 @@ #include #include +#include namespace quickbook { @@ -56,6 +57,7 @@ namespace quickbook bool operator==(string_ref const& x, string_ref const& y); bool operator<(string_ref const& x, string_ref const& y); + std::ostream& operator<<(std::ostream&, string_ref const& x); inline bool operator==(string_ref const& x, std::string const& y) { diff --git a/src/syntax_highlight.cpp b/src/syntax_highlight.cpp index bacb23f..7a812ef 100644 --- a/src/syntax_highlight.cpp +++ b/src/syntax_highlight.cpp @@ -396,7 +396,8 @@ namespace quickbook }; std::string syntax_highlight( - iterator first, iterator last, + parse_iterator first, + parse_iterator last, actions& escape_actions, std::string const& source_mode) { diff --git a/src/template_stack.cpp b/src/template_stack.cpp index e6d916a..98514f9 100644 --- a/src/template_stack.cpp +++ b/src/template_stack.cpp @@ -18,11 +18,9 @@ namespace quickbook { template_body::template_body( value const& content, - fs::path const& filename, content_type type ) : content(content) - , filename(filename) , type(type) { assert(content.get_tag() == template_tags::block || diff --git a/src/template_stack.hpp b/src/template_stack.hpp index eedee15..d3fd24e 100644 --- a/src/template_stack.hpp +++ b/src/template_stack.hpp @@ -34,11 +34,10 @@ namespace quickbook raw_output }; - template_body(value const&, fs::path const&, content_type = quickbook); + template_body(value const&, content_type = quickbook); bool is_block() const; - stored_value content; - fs::path filename; + value content; content_type type; }; @@ -61,11 +60,10 @@ namespace quickbook std::string const& identifier, std::vector const& params, value const& content, - fs::path const& filename, template_scope const* parent = 0) : identifier(identifier) , params(params) - , body(content, filename) + , body(content) , parent(parent) , callouts() {} @@ -78,7 +76,7 @@ namespace quickbook // or static_parent for clarity. template_scope const* parent; - stored_value callouts; + value callouts; }; typedef boost::spirit::classic::symbols template_symbols; diff --git a/src/utils.cpp b/src/utils.cpp index bd81dce..3f9275a 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -8,14 +8,9 @@ http://www.boost.org/LICENSE_1_0.txt) =============================================================================*/ #include "utils.hpp" -#include -#include #include #include -#include -#include -#include #include namespace quickbook { namespace detail @@ -132,105 +127,7 @@ namespace quickbook { namespace detail } return uri; } - - // Read the first few bytes in a file to see it starts with a byte order - // mark. If it doesn't, then write the characters we've already read in. - // Although, given how UTF-8 works, if we've read anything in, the files - // probably broken. - - template - bool check_bom(InputIterator& begin, InputIterator end, - OutputIterator out, char const* chars, int length) - { - char const* ptr = chars; - - while(begin != end && *begin == *ptr) { - ++begin; - ++ptr; - --length; - if(length == 0) return true; - } - - // Failed to match, so write the skipped characters to storage: - while(chars != ptr) *out++ = *chars++; - - return false; - } - template - std::string read_bom(InputIterator& begin, InputIterator end, - OutputIterator out) - { - if(begin == end) return ""; - - const char* utf8 = "\xef\xbb\xbf" ; - const char* utf32be = "\0\0\xfe\xff"; - const char* utf32le = "\xff\xfe\0\0"; - - unsigned char c = *begin; - switch(c) - { - case 0xEF: { // UTF-8 - return check_bom(begin, end, out, utf8, 3) ? "UTF-8" : ""; - } - case 0xFF: // UTF-16/UTF-32 little endian - return !check_bom(begin, end, out, utf32le, 2) ? "" : - check_bom(begin, end, out, utf32le + 2, 2) ? "UTF-32" : "UTF-16"; - case 0: // UTF-32 big endian - return check_bom(begin, end, out, utf32be, 4) ? "UTF-32" : ""; - case 0xFE: // UTF-16 big endian - return check_bom(begin, end, out, utf32be + 2, 2) ? "UTF-16" : ""; - default: - return ""; - } - } - - // Copy a string, converting mac and windows style newlines to unix - // newlines. - - template - void normalize(InputIterator begin, InputIterator end, - OutputIterator out) - { - std::string encoding = read_bom(begin, end, out); - - if(encoding != "UTF-8" && encoding != "") - throw load_error(encoding + - " is not supported. Please use UTF-8."); - - while(begin != end) { - if(*begin == '\r') { - *out++ = '\n'; - ++begin; - if(begin != end && *begin == '\n') ++begin; - } - else { - *out++ = *begin++; - } - } - } - - void load(fs::path const& filename, std::string& storage) - { - using std::endl; - using std::ios; - using std::ifstream; - using std::istream_iterator; - - fs::ifstream in(filename, std::ios_base::in); - - if (!in) - throw load_error("Could not open input file."); - - // Turn off white space skipping on the stream - in.unsetf(ios::skipws); - - normalize( - istream_iterator(in), - istream_iterator(), - std::back_inserter(storage)); - } - file_type get_file_type(std::string const& extension) { static std::map ftypes; diff --git a/src/utils.hpp b/src/utils.hpp index 1326ad4..d4521ad 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -11,19 +11,11 @@ #define BOOST_SPIRIT_QUICKBOOK_UTILS_HPP #include -#include -#include -#include +#include #include #include - -namespace quickbook { - - namespace fs = boost::filesystem; - -namespace detail -{ +namespace quickbook { namespace detail { void print_char(char ch, std::ostream& out); void print_string(std::basic_string const& str, std::ostream& out); void print_space(char ch, std::ostream& out); @@ -45,15 +37,6 @@ namespace detail void unindent(std::string& program); std::string escape_uri(std::string uri); - - class load_error : public std::runtime_error - { - public: - explicit load_error(std::string const& arg) - : std::runtime_error(arg) {} - }; - - void load(fs::path const& filename, std::string& storage); // given a file extension, return the type of the source file // we'll have an internal database for known file types. diff --git a/src/values.cpp b/src/values.cpp index ce55838..7d32cc1 100644 --- a/src/values.cpp +++ b/src/values.cpp @@ -7,6 +7,7 @@ =============================================================================*/ #include "values.hpp" +#include "files.hpp" #include #include #include @@ -47,12 +48,10 @@ namespace quickbook value_node::~value_node() { } - value_node* value_node::store() { return this; } - - file_position value_node::get_position() const { UNDEFINED_ERROR(); } + file const* value_node::get_file() const { UNDEFINED_ERROR(); } + string_iterator value_node::get_position() const { UNDEFINED_ERROR(); } int value_node::get_int() const { UNDEFINED_ERROR(); } - std::string value_node::get_quickbook() const { UNDEFINED_ERROR(); } - value_node::qbk_range value_node::get_quickbook_range() const { UNDEFINED_ERROR(); } + string_ref value_node::get_quickbook() const { UNDEFINED_ERROR(); } std::string value_node::get_boostbook() const { UNDEFINED_ERROR(); } value_node* value_node::get_list() const { UNDEFINED_ERROR(); } @@ -231,30 +230,6 @@ namespace quickbook return *this; } - //////////////////////////////////////////////////////////////////////////// - // stored_value - - stored_value::stored_value() - : detail::value_counted() - { - } - - stored_value::stored_value(stored_value const& x) - : detail::value_counted(x) - { - } - - stored_value::stored_value(detail::value_base const& x) - : detail::value_counted(x.value_->store()) - { - } - - stored_value& stored_value::operator=(stored_value x) - { - swap(x); - return *this; - } - //////////////////////////////////////////////////////////////////////////// // Integers @@ -268,7 +243,6 @@ namespace quickbook char const* type_name() const { return "integer"; } virtual value_node* clone() const; virtual int get_int() const; - virtual std::string get_quickbook() const; virtual std::string get_boostbook() const; virtual bool empty() const; virtual bool equals(value_node*) const; @@ -291,11 +265,6 @@ namespace quickbook return value_; } - std::string value_int_impl::get_quickbook() const - { - return boost::lexical_cast(value_); - } - std::string value_int_impl::get_boostbook() const { return boost::lexical_cast(value_); @@ -346,46 +315,46 @@ namespace quickbook struct value_qbk_string_impl : public value_node { public: - explicit value_qbk_string_impl( - std::string const&, file_position, value::tag_type); - explicit value_qbk_string_impl( - quickbook::iterator begin, quickbook::iterator end, - value::tag_type); + explicit value_qbk_string_impl(std::string const&, value::tag_type); + explicit value_qbk_string_impl(file const&, value::tag_type); private: char const* type_name() const { return "quickbook"; } virtual ~value_qbk_string_impl(); virtual value_node* clone() const; - virtual file_position get_position() const; - virtual std::string get_quickbook() const; - qbk_range get_quickbook_range() const; + virtual file const* get_file() const; + virtual string_iterator get_position() const; + virtual string_ref get_quickbook() const; virtual bool is_string() const; virtual bool empty() const; virtual bool equals(value_node*) const; - - std::string value_; - file_position position_; + + file fake_file_; }; struct value_qbk_ref_impl : public value_node { public: - explicit value_qbk_ref_impl(quickbook::iterator begin, quickbook::iterator end, value::tag_type); + explicit value_qbk_ref_impl( + file const*, + string_iterator begin, + string_iterator end, + value::tag_type); private: char const* type_name() const { return "quickbook"; } virtual ~value_qbk_ref_impl(); virtual value_node* clone() const; - virtual value_node* store(); - virtual file_position get_position() const; - virtual std::string get_quickbook() const; - qbk_range get_quickbook_range() const; + virtual file const* get_file() const; + virtual string_iterator get_position() const; + virtual string_ref get_quickbook() const; virtual bool is_string() const; virtual bool empty() const; virtual bool equals(value_node*) const; - quickbook::iterator begin_; - quickbook::iterator end_; + file const* file_; + string_iterator begin_; + string_iterator end_; }; struct value_qbk_bbk_impl : public value_node @@ -393,32 +362,27 @@ namespace quickbook private: char const* type_name() const { return "quickbook/boostbook"; } - value_qbk_bbk_impl( - std::string const& qbk, std::string const& bbk, - file_position const&, value::tag_type); - value_qbk_bbk_impl(std::string const&, value::tag_type); - value_qbk_bbk_impl( - quickbook::iterator, quickbook::iterator, + value_qbk_bbk_impl(file const*, + string_iterator, string_iterator, std::string const&, value::tag_type); virtual ~value_qbk_bbk_impl(); virtual value_node* clone() const; - virtual file_position get_position() const; - virtual std::string get_quickbook() const; - qbk_range get_quickbook_range() const; + virtual file const* get_file() const; + virtual string_iterator get_position() const; + virtual string_ref get_quickbook() const; virtual std::string get_boostbook() const; virtual bool is_string() const; virtual bool empty() const; virtual bool equals(value_node*) const; - std::string qbk_value_; + file const* file_; + string_iterator begin_; + string_iterator end_; std::string bbk_value_; - file_position position_; friend quickbook::value quickbook::qbk_bbk_value( - std::string const&, quickbook::value::tag_type); - friend quickbook::value quickbook::qbk_bbk_value( - quickbook::iterator, quickbook::iterator, + file const*, string_iterator, string_iterator, std::string const&, quickbook::value::tag_type); }; @@ -463,49 +427,47 @@ namespace quickbook value_qbk_string_impl::value_qbk_string_impl( std::string const& v, - file_position p, value::tag_type tag) : value_node(tag) - , value_(v) - , position_(p) - {} + , fake_file_() + { + fake_file_.source = v; + fake_file_.path = "(generated code)"; + } value_qbk_string_impl::value_qbk_string_impl( - quickbook::iterator begin, quickbook::iterator end, - value::tag_type tag) + file const& f, value::tag_type tag) : value_node(tag) - , value_(begin, end) - , position_(begin.get_position()) - {} + , fake_file_(f) + { + } value_qbk_string_impl::~value_qbk_string_impl() {} value_node* value_qbk_string_impl::clone() const { - return new value_qbk_string_impl(value_, position_, tag_); + return new value_qbk_string_impl(fake_file_, tag_); } - file_position value_qbk_string_impl::get_position() const - { return position_; } + file const* value_qbk_string_impl::get_file() const + { return &fake_file_; } - std::string value_qbk_string_impl::get_quickbook() const - { return value_; } + string_iterator value_qbk_string_impl::get_position() const + { return fake_file_.source.begin(); } - value::qbk_range value_qbk_string_impl::get_quickbook_range() const - { return qbk_range( - iterator(value_.begin(), position_), - iterator(value_.end())); } + string_ref value_qbk_string_impl::get_quickbook() const + { return string_ref(fake_file_.source); } bool value_qbk_string_impl::is_string() const { return true; } bool value_qbk_string_impl::empty() const - { return value_.empty(); } + { return fake_file_.source.empty(); } bool value_qbk_string_impl::equals(value_node* other) const { try { - return value_ == other->get_quickbook(); + return fake_file_.source == other->get_quickbook(); } catch(value_undefined_method&) { return false; @@ -515,9 +477,11 @@ namespace quickbook // value_qbk_ref_impl value_qbk_ref_impl::value_qbk_ref_impl( - quickbook::iterator begin, quickbook::iterator end, + file const* f, + string_iterator begin, + string_iterator end, value::tag_type tag - ) : value_node(tag), begin_(begin), end_(end) + ) : value_node(tag), file_(f), begin_(begin), end_(end) { } @@ -527,22 +491,17 @@ namespace quickbook value_node* value_qbk_ref_impl::clone() const { - return new value_qbk_ref_impl(begin_, end_, tag_); + return new value_qbk_ref_impl(file_, begin_, end_, tag_); } - value_node* value_qbk_ref_impl::store() - { - return new value_qbk_string_impl(begin_, end_, tag_); - } + file const* value_qbk_ref_impl::get_file() const + { return file_; } - file_position value_qbk_ref_impl::get_position() const - { return begin_.get_position(); } + string_iterator value_qbk_ref_impl::get_position() const + { return begin_; } - std::string value_qbk_ref_impl::get_quickbook() const - { return std::string(begin_.base(), end_.base()); } - - value::qbk_range value_qbk_ref_impl::get_quickbook_range() const - { return qbk_range(begin_, end_); } + string_ref value_qbk_ref_impl::get_quickbook() const + { return string_ref(begin_, end_); } bool value_qbk_ref_impl::is_string() const { return true; } @@ -562,41 +521,20 @@ namespace quickbook // value_qbk_bbk_impl value_qbk_bbk_impl::value_qbk_bbk_impl( - std::string const& qbk, - std::string const& bbk, - file_position const& pos, - value::tag_type tag) - : value_node(tag) - , qbk_value_(qbk) - , bbk_value_(bbk) - , position_(pos) - - { - } - - value_qbk_bbk_impl::value_qbk_bbk_impl( - quickbook::iterator begin, - quickbook::iterator end, + file const* f, + string_iterator begin, + string_iterator end, std::string const& bbk, value::tag_type tag) : value_node(tag) - , qbk_value_(begin.base(), end.base()) + , file_(f) + , begin_(begin) + , end_(end) , bbk_value_(bbk) - , position_(begin.get_position()) { } - value_qbk_bbk_impl::value_qbk_bbk_impl( - std::string const& val, - value::tag_type tag) - : value_node(tag) - , qbk_value_(val) - , bbk_value_(val) - , position_() - { - } - value_qbk_bbk_impl::~value_qbk_bbk_impl() { } @@ -604,19 +542,17 @@ namespace quickbook value_node* value_qbk_bbk_impl::clone() const { return new value_qbk_bbk_impl( - qbk_value_, bbk_value_, position_, tag_); + file_, begin_, end_, bbk_value_, tag_); } - file_position value_qbk_bbk_impl::get_position() const - { return position_; } + file const* value_qbk_bbk_impl::get_file() const + { return file_; } - std::string value_qbk_bbk_impl::get_quickbook() const - { return qbk_value_; } + string_iterator value_qbk_bbk_impl::get_position() const + { return begin_; } - value::qbk_range value_qbk_bbk_impl::get_quickbook_range() const - { return qbk_range( - iterator(qbk_value_.begin(), position_), - iterator(qbk_value_.end())); } + string_ref value_qbk_bbk_impl::get_quickbook() const + { return string_ref(begin_, end_); } std::string value_qbk_bbk_impl::get_boostbook() const { return bbk_value_; } @@ -643,14 +579,14 @@ namespace quickbook } } - value qbk_value(iterator x, iterator y, value::tag_type t) + value qbk_value_ref(file const* f, string_iterator x, string_iterator y, value::tag_type t) { - return value(new detail::value_qbk_ref_impl(x, y, t)); + return value(new detail::value_qbk_ref_impl(f, x, y, t)); } - value qbk_value(std::string const& x, file_position pos, value::tag_type t) + value qbk_value(std::string const& x, value::tag_type t) { - return value(new detail::value_qbk_string_impl(x, pos, t)); + return value(new detail::value_qbk_string_impl(x, t)); } value bbk_value(std::string const& x, value::tag_type t) @@ -658,16 +594,11 @@ namespace quickbook return value(new detail::value_string_impl(x, t)); } - value qbk_bbk_value(std::string const& x, value::tag_type t) - { - return value(new detail::value_qbk_bbk_impl(x,t)); - } - value qbk_bbk_value( - iterator x, iterator y, + file const* f, string_iterator x, string_iterator y, std::string const& z, value::tag_type t) { - return value(new detail::value_qbk_bbk_impl(x,y,z,t)); + return value(new detail::value_qbk_bbk_impl(f,x,y,z,t)); } ////////////////////////////////////////////////////////////////////////// @@ -789,7 +720,6 @@ namespace quickbook virtual ~value_list_impl(); virtual value_node* clone() const; - virtual value_node* store(); virtual bool empty() const; virtual bool equals(value_node*) const; @@ -825,35 +755,6 @@ namespace quickbook return new value_list_impl(*this); } - value_node* value_list_impl::store() - { - value_node* pos = head_; - boost::intrusive_ptr new_node; - - for(;;) { - if(pos == &value_list_end_impl::instance) - return this; - new_node = pos->store(); - if(new_node.get() != pos) break; - pos = pos->next_; - } - - value_list_builder build; - - value_node* pos2 = head_; - - for(;pos2 != pos; pos2 = pos2->next_) - build.append(pos2); - - build.append(new_node.get()); - pos2 = pos2->next_; - - for(;pos2 != &value_list_end_impl::instance; pos2 = pos2->next_) - build.append(pos2->store()); - - return new value_list_impl(build, tag_); - } - bool value_list_impl::empty() const { return head_ == &value_list_end_impl::instance; diff --git a/src/values.hpp b/src/values.hpp index e24c98e..8111a6e 100644 --- a/src/values.hpp +++ b/src/values.hpp @@ -16,14 +16,13 @@ #include #include #include -#include #include #include "fwd.hpp" +#include "string_ref.hpp" namespace quickbook { struct value; - struct stored_value; struct value_builder; struct value_error; @@ -40,7 +39,6 @@ namespace quickbook public: typedef int tag_type; - typedef boost::iterator_range qbk_range; protected: explicit value_node(tag_type); @@ -49,12 +47,11 @@ namespace quickbook public: virtual char const* type_name() const = 0; virtual value_node* clone() const = 0; - virtual value_node* store(); - virtual file_position get_position() const; - virtual std::string get_quickbook() const; + virtual file const* get_file() const; + virtual string_iterator get_position() const; + virtual string_ref get_quickbook() const; virtual std::string get_boostbook() const; - virtual qbk_range get_quickbook_range() const; virtual int get_int() const; virtual bool check() const; @@ -89,7 +86,6 @@ namespace quickbook typedef iterator const_iterator; typedef value_node::tag_type tag_type; enum { default_tag = 0 }; - typedef boost::iterator_range qbk_range; protected: explicit value_base(value_node* base) @@ -112,12 +108,12 @@ namespace quickbook // Item accessors int get_tag() const { return value_->tag_; } - file_position get_position() const + file const* get_file() const + { return value_->get_file(); } + string_iterator get_position() const { return value_->get_position(); } - std::string get_quickbook() const + string_ref get_quickbook() const { return value_->get_quickbook(); } - qbk_range get_quickbook_range() const - { return value_->get_quickbook_range(); } std::string get_boostbook() const { return value_->get_boostbook(); } int get_int() const @@ -134,7 +130,6 @@ namespace quickbook // value_builder needs to access 'value_' to get the node // from a value. friend struct quickbook::value_builder; - friend struct quickbook::stored_value; }; //////////////////////////////////////////////////////////////////////// @@ -237,16 +232,6 @@ namespace quickbook void swap(value& x) { detail::value_counted::swap(x); } }; - struct stored_value : public detail::value_counted - { - public: - stored_value(); - stored_value(stored_value const&); - stored_value(detail::value_base const&); - stored_value& operator=(stored_value); - void swap(stored_value& x) { detail::value_counted::swap(x); } - }; - // Empty value empty_value(value::tag_type = value::default_tag); @@ -254,14 +239,10 @@ namespace quickbook value int_value(int, value::tag_type = value::default_tag); // Boostbook and quickbook strings - value qbk_value(iterator, iterator, value::tag_type = value::default_tag); - value qbk_value(std::string const&, - file_position = file_position(), - value::tag_type = value::default_tag); + value qbk_value_ref(file const*, string_iterator, string_iterator, value::tag_type = value::default_tag); + value qbk_value(std::string const&, value::tag_type = value::default_tag); value bbk_value(std::string const&, value::tag_type = value::default_tag); - value qbk_bbk_value(std::string const&, - value::tag_type = value::default_tag); - value qbk_bbk_value(iterator, iterator, std::string const&, + value qbk_bbk_value(file const*, string_iterator, string_iterator, std::string const&, value::tag_type = value::default_tag); //////////////////////////////////////////////////////////////////////////// diff --git a/src/values_parse.hpp b/src/values_parse.hpp index a65a3fd..7ae3196 100644 --- a/src/values_parse.hpp +++ b/src/values_parse.hpp @@ -57,22 +57,20 @@ namespace quickbook { typedef void type; }; - value_entry(value_builder& b) - : b(b) {} + value_entry(value_builder& b, file const** current_file) + : b(b), current_file(current_file) {} - template - void operator()(Iterator begin, Iterator end, + void operator()(parse_iterator begin, parse_iterator end, value::tag_type tag = value::default_tag) const { - b.insert(qbk_value(begin, end, tag)); + b.insert(qbk_value_ref(*current_file, begin.base(), end.base(), tag)); } - template - void operator()(Iterator begin, Iterator, + void operator()(parse_iterator begin, parse_iterator, std::string const& v, value::tag_type tag = value::default_tag) const { - b.insert(qbk_value(v, begin.get_position(), tag)); + b.insert(qbk_value(v, tag)); } void operator()(int v, @@ -82,6 +80,7 @@ namespace quickbook { } value_builder& b; + file const** current_file; }; struct value_reset @@ -114,11 +113,11 @@ namespace quickbook { struct value_parser { - value_parser() + value_parser(file const** current_file) : builder() , save(builder) , list(builder) - , entry(builder) + , entry(value_entry(builder, current_file)) , reset(builder) , sort(builder) {} diff --git a/test/unit/Jamfile.v2 b/test/unit/Jamfile.v2 index 19b4d1a..3871fbb 100644 --- a/test/unit/Jamfile.v2 +++ b/test/unit/Jamfile.v2 @@ -9,12 +9,14 @@ import testing ; project quickbook-unit-tests - : requirements ../../src all + : requirements + ../../src + all + /boost//filesystem ; -run values_test.cpp ../../src/values.cpp ; +run values_test.cpp ../../src/values.cpp ../../src/string_ref.cpp ; run post_process_test.cpp ../../src/post_process.cpp ; -run iterator_tests.cpp ../../src/values.cpp ; # Copied from spirit run symbols_tests.cpp ; diff --git a/test/unit/iterator_tests.cpp b/test/unit/iterator_tests.cpp deleted file mode 100644 index c163ace..0000000 --- a/test/unit/iterator_tests.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/*============================================================================= - Copyright (c) 2011 Daniel James - - Use, modification and distribution is subject to the Boost Software - License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at - http://www.boost.org/LICENSE_1_0.txt) -=============================================================================*/ - -// Use boost's iterator concept tests for our iterators. - -#include "fwd.hpp" -#include "values.hpp" -#include -#include -#include - -void iterator_concept_checks() -{ - typedef quickbook::iterator Iter; - boost::function_requires< boost::ForwardIterator >(); - boost::function_requires< boost_concepts::ReadableIteratorConcept >(); - boost::function_requires< boost_concepts::LvalueIteratorConcept >(); - boost::function_requires< boost_concepts::ForwardTraversalConcept >(); -} - -void value_iterator_concept_checks() -{ - typedef quickbook::value::iterator Iter; - boost::function_requires< boost::ForwardIterator >(); - boost::function_requires< boost_concepts::ReadableIteratorConcept >(); - boost::function_requires< boost_concepts::ForwardTraversalConcept >(); -} - -void iterator_runtime_checks() -{ - std::string x = "Hello World"; - - quickbook::iterator i1(x.begin(), quickbook::file_position(10, 5)); - quickbook::iterator i2(++x.begin(), quickbook::file_position(10, 6)); - - boost::forward_readable_iterator_test(i1, i2, 'H', 'e'); - boost::constant_lvalue_iterator_test(i1, 'H'); -} - -void value_iterator_runtime_checks() -{ - quickbook::value v1 = quickbook::bbk_value("a", 10); - quickbook::value v2 = quickbook::int_value(25, 32); - - quickbook::value_builder b; - b.insert(v1); - b.insert(v2); - quickbook::value x = b.release(); - - quickbook::value::iterator i1 = x.begin(); - quickbook::value::iterator i2 = ++x.begin(); - - boost::forward_readable_iterator_test(i1, i2, v1, v2); -} - -int main() -{ - // I know I don't have to run the concept checks. - // I'm a bit irrational like that. - iterator_concept_checks(); - value_iterator_concept_checks(); - iterator_runtime_checks(); - value_iterator_runtime_checks(); - - return boost::report_errors(); -} diff --git a/test/unit/values_test.cpp b/test/unit/values_test.cpp index 1b24354..8747f40 100644 --- a/test/unit/values_test.cpp +++ b/test/unit/values_test.cpp @@ -13,6 +13,7 @@ #include #include #include "values.hpp" +#include "files.hpp" void empty_tests() { @@ -24,11 +25,13 @@ void empty_tests() void qbk_tests() { - std::string src = "Source"; - quickbook::value q = quickbook::qbk_value( - quickbook::iterator(src.begin()), - quickbook::iterator(src.end())); - BOOST_TEST_EQ(q.get_quickbook(), src); + quickbook::file fake_file; + fake_file.source = "Source"; + quickbook::value q = quickbook::qbk_value_ref( + &fake_file, + fake_file.source.begin(), + fake_file.source.end()); + BOOST_TEST_EQ(q.get_quickbook(), fake_file.source); } void sort_test() @@ -79,59 +82,6 @@ void multiple_list_test() BOOST_TEST(!l2.check()); } -void store_test1() -{ - quickbook::stored_value q; - - { - std::string src = "Hello"; - quickbook::value q1 = quickbook::qbk_value( - quickbook::iterator(src.begin()), - quickbook::iterator(src.end()), - 5); - - BOOST_TEST_EQ(q1.get_quickbook(), "Hello"); - q = q1; - BOOST_TEST_EQ(q.get_quickbook(), "Hello"); - BOOST_TEST_EQ(q1.get_quickbook(), "Hello"); - } - - BOOST_TEST_EQ(q.get_quickbook(), "Hello"); -} - -void store_test2_check(quickbook::value const& q) -{ - quickbook::value_consumer l1 = q; - BOOST_TEST(l1.check(5)); - BOOST_TEST_EQ(l1.consume(5).get_quickbook(), "Hello"); - BOOST_TEST(l1.check(10)); - BOOST_TEST_EQ(l1.consume(10).get_boostbook(), "World"); - BOOST_TEST(!l1.check()); -} - -void store_test2() -{ - quickbook::stored_value q; - - { - quickbook::value_builder list1; - std::string src = "Hello"; - list1.insert(quickbook::qbk_value( - quickbook::iterator(src.begin()), - quickbook::iterator(src.end()), - 5)); - list1.insert(quickbook::bbk_value("World", 10)); - - quickbook::value q2 = list1.release(); - - store_test2_check(q2); - q = q2; - store_test2_check(q); - store_test2_check(q2); - } - store_test2_check(q); -} - void equality_tests() { std::vector distinct_values; @@ -165,8 +115,6 @@ int main() qbk_tests(); sort_test(); multiple_list_test(); - store_test1(); - store_test2(); equality_tests(); return boost::report_errors();