Implement command database generation (i.e. compile_commands.json). (#399)

This implements obtaining and generating compile commands from toolsets that compile C or C++ sources. I.e. implements both the --command-database=json amd --command-database-out=<filename> CLI options. Although it implements the toolset changes for most compilers, only a few are tested.

Fixes #395

.vscode/launch.json

@@ -93,7 +93,7 @@
 			"type": "cppdbg",
 			"request": "launch",
 			"program": "${workspaceFolder}/.build/gcc-13/debug/cxxstd-11-iso/threading-multi/b2",
-			"args": [],
+			"args": ["-d1", "--command-database=json", "b2"],
 			"stopAtEntry": false,
 			"cwd": "${workspaceFolder}",
 			"environment": [],

Jamroot.jam

@@ -22,7 +22,7 @@ import virtual-target ;
 
 path-constant SELF : . ;
 
-project b2
+project /bfgroup/b2
     : build-dir .build
     : requirements
         <cxxstd>11
@@ -34,6 +34,9 @@ project b2
         <toolset>clang-win:<define>_CRT_NONSTDC_NO_DEPRECATE=1
         <toolset>clang:<cxxflags>-Wno-deprecated-declarations
         <toolset>gcc,<variant>debug,<target-os>linux:<linkflags>-rdynamic
+        # Don't warn on ignored/unknown attributes.
+        <toolset>gcc:<cxxflags>-Wno-attributes
+        <toolset>clang:<cxxflags>-Wno-attributes
     ;
 
 #|

doc/src/history.adoc

@@ -3,6 +3,9 @@
 
 == Version 5.2.0
 
+* *New*: Add support for generating `compile_commands.json` command database
+  for some IDE integration.
+  -- _René Ferdinand Rivera Morell_
 * *New*: Addition of a module (`db`) for structured data management. Has a
   `property-db` class that can write out as JSON data.
   -- _René Ferdinand Rivera Morell_

doc/src/overview.adoc

@@ -626,8 +626,7 @@ B2 recognizes the following command line options.
 `-d0`::
   Suppress all informational messages.
 `-d N`::
-  Enable cumulative debugging levels from 1 to n. Values are:
-  +
+  Enable cumulative debugging levels from 1 to n. Values are: +
   1.  Show the actions taken for building targets, as they are executed
   (the default).
   2.  Show "quiet" actions and display all action text, as they are
@@ -653,6 +652,11 @@ B2 recognizes the following command line options.
 `-s var=value`::
   Set the variable `var` to `value` in the global scope of the jam language
   interpreter, overriding variables imported from the environment.
+`--command-database=_format_`::
+  Output a compile commands database as _format_. Currently _format_ can be:
+  `json`. (See xref:tasks#b2.tasks.commanddb[Command Database] for details.)
+`--command-database-out=_file_`::
+  Specify the _file_ path to output the commands database to.
 
 [[bbv2.overview.invocation.properties]]
 === Properties

doc/src/tasks.adoc

@@ -821,3 +821,50 @@ When loading a `*.jam` file as the _path_ it is equivalent to calling:
 In this case it means that the file will be loaded as part of the referenced
 project and hence any bare targets or information it declares will be part of
 the project.
+
+[[b2.tasks.commanddb]]
+== Command Database, and IDE Integration
+
+Many IDE programs accept the use of a
+https://clang.llvm.org/docs/JSONCompilationDatabase.html[`compile_commands.json`]
+file to learn what and how your project builds. B2 supports generating such
+files for any build you make. B2 supports this through a generic facility to
+extract commands from the actions it executes. There are two options that
+control this. The `--command-database=_format_` option indicates to generate the
+file for the given _format_. It has a couple of effects when specified:
+
+* It tells B2 to start observing and extracting commands from actions (as
+  specified by the toolset).
+* It disables execution of actions. I.e. equivalent to adding the `-n` option.
+* It enables building all default and specified targets. I.e. the equivalent to
+  adding the `-a` option.
+* It disables all action execution output. I.e. as if specifying `-d0` option.
+* At the end of the main build it writes out the results of what it observed
+  to the database file.
+
+Currently on `json` is supported as a format that follows the
+https://clang.llvm.org/docs/JSONCompilationDatabase.html[Clang JSON Compilation
+Database Format Specification].
+
+The `--command-database-out=_file_` option controls the name, and optionally
+location, of the generated file. By default the _file_ is
+`compile_commands.json` to follow the ecosystem convention. And it is generated,
+by default, in one of the following locations:
+
+* Relative to the `build-dir` of the root project, if it's specified by the
+  project. With the default _file_ name or as given.
+* At the absolute _file_ path if it is rooted.
+* At the _current working directory_.
+
+The following fields are populated in the generated database:
+
+* `directory` - This will always be the current directory as B2 makes all paths
+  relative to that (or absolute).
+* `file` - The first source of each action recorded.
+* `command` - The quoted, full, command as extracted by the toolset.
+* `output` - The first target file of each action recorded. As B2 can build
+  multiple variants at once this is required to differentiate between multiple
+  compilations of the same source file.
+
+NOTE: Only one command database file is generated per `b2` invocation. And each
+time it is generated it overwrites any previous such file.

src/build/project.jam

@@ -42,6 +42,7 @@ import print ;
 import property-set ;
 import regex ;
 import sequence ;
+import command-db ;
 
 
 .debug-loading = [ MATCH ^(--debug-loading)$ : [ modules.peek : ARGV ] ] ;
@@ -712,11 +713,12 @@ rule initialize (
 
     # load-parent can end up loading this module again. Make sure this is not
     # duplicated.
+    local attributes ;
     if ! $($(module-name).attributes)
     {
         $(module-name).attributes = [ new project-attributes $(location)
             $(module-name) ] ;
-        local attributes = $($(module-name).attributes) ;
+        attributes = $($(module-name).attributes) ;
 
         if $(location)
         {
@@ -791,6 +793,12 @@ rule initialize (
     }
 
     .current-project = [ target $(module-name) ] ;
+
+    if $(jamroot) && [ $(attributes).get build-dir ]
+    {
+        command-db.set-output-dir
+            [ path.native [ $(attributes).get build-dir ] ] ;
+    }
 }
 
 
@@ -1312,6 +1320,7 @@ module project-rules
     {
         import path ;
         import project ;
+        import command-db ;
 
         local caller = [ CALLER_MODULE ] ;
         local attributes = [ project.attributes $(caller) ] ;
@@ -1382,6 +1391,12 @@ module project-rules
                 }
             }
         }
+
+        if $(explicit-build-dir)
+        {
+            command-db.set-output-dir
+                [ path.native [ $(attributes).get build-dir ] ] ;
+        }
     }
 
     # Declare and set a project global constant. Project global constants are

src/engine/bindjam.cpp

@@ -21,6 +21,7 @@ Distributed under the Boost Software License, Version 1.0.
 #include "value.h"
 #include "variable.h"
 
+#include "mod_command_db.h"
 #include "mod_db.h"
 #include "mod_jam_builtin.h"
 #include "mod_jam_class.h"
@@ -823,7 +824,8 @@ void bind_jam(FRAME * f)
 		.bind(string_module())
 		.bind(sysinfo_module())
 		.bind(version_module())
-		.bind(db_module());
+		.bind(db_module())
+		.bind(command_db_module());
 }
 
 }} // namespace b2::jam

src/engine/build.bat

@@ -170,6 +170,7 @@ set B2_SOURCES=
 set B2_SOURCES=%B2_SOURCES% bindjam.cpp builtins.cpp class.cpp
 set B2_SOURCES=%B2_SOURCES% command.cpp compile.cpp constants.cpp cwd.cpp
 set B2_SOURCES=%B2_SOURCES% debug.cpp debugger.cpp
+set B2_SOURCES=%B2_SOURCES% events.cpp
 set B2_SOURCES=%B2_SOURCES% execcmd.cpp execnt.cpp execunix.cpp filent.cpp filesys.cpp fileunix.cpp frames.cpp function.cpp
 set B2_SOURCES=%B2_SOURCES% glob.cpp hash.cpp hcache.cpp hdrmacro.cpp headers.cpp jam.cpp
 set B2_SOURCES=%B2_SOURCES% jamgram.cpp lists.cpp make.cpp make1.cpp md5.cpp mem.cpp modules.cpp
@@ -177,6 +178,7 @@ set B2_SOURCES=%B2_SOURCES% native.cpp option.cpp output.cpp parse.cpp pathnt.cp
 set B2_SOURCES=%B2_SOURCES% pathsys.cpp pathunix.cpp regexp.cpp rules.cpp scan.cpp search.cpp jam_strings.cpp
 set B2_SOURCES=%B2_SOURCES% startup.cpp tasks.cpp
 set B2_SOURCES=%B2_SOURCES% timestamp.cpp value.cpp variable.cpp w32_getreg.cpp
+set B2_SOURCES=%B2_SOURCES% mod_command_db.cpp
 set B2_SOURCES=%B2_SOURCES% mod_db.cpp
 set B2_SOURCES=%B2_SOURCES% mod_jam_builtin.cpp
 set B2_SOURCES=%B2_SOURCES% mod_jam_class.cpp

src/engine/build.sh

@@ -448,6 +448,7 @@ constants.cpp \
 cwd.cpp \
 debug.cpp \
 debugger.cpp \
+events.cpp \
 execcmd.cpp \
 execnt.cpp \
 execunix.cpp \
@@ -487,6 +488,7 @@ timestamp.cpp \
 value.cpp \
 variable.cpp \
 w32_getreg.cpp \
+mod_command_db.cpp \
 mod_db.cpp \
 mod_jam_builtin.cpp \
 mod_jam_class.cpp \

src/engine/events.cpp

@@ -0,0 +1,123 @@
+/*
+Copyright 2024 René Ferdinand Rivera Morell
+Distributed under the Boost Software License, Version 1.0.
+(See accompanying file LICENSE.txt or https://www.bfgroup.xyz/b2/LICENSE.txt)
+*/
+
+#include "events.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <vector>
+
+namespace b2 {
+
+struct event_base
+{
+	event_tag tag;
+	int32_t priority;
+	uint64_t id;
+
+	inline bool operator<(const event_base & o) const
+	{
+		if (tag < o.tag) return true;
+		if ((0 - priority) < (0 - o.priority)) return true;
+		return id < o.id;
+	}
+};
+
+template <typename F>
+struct event : public event_base
+{
+	F call;
+};
+
+struct events
+{
+	struct ecmp
+	{
+		inline bool operator()(const event_base * a, const event_base * b) const
+		{
+			return (*a) < (*b);
+		}
+	};
+	std::set<const event_base *, ecmp> sorted_items;
+	std::vector<std::unique_ptr<event_base>> items;
+
+	static events & get()
+	{
+		static events e;
+		return e;
+	}
+
+	template <typename F>
+	uint64_t add(event_tag tag, F && call, int32_t priority)
+	{
+		uint64_t id = items.empty() ? 1 : items.back()->id + 1;
+		std::unique_ptr<event<F>> e(new event<F>);
+		e->tag = tag;
+		e->priority = priority;
+		e->id = id;
+		e->call = call;
+		sorted_items.insert(e.get());
+		items.push_back(std::move(e));
+		return id;
+	}
+
+	void remove(uint64_t e)
+	{
+		auto i = std::lower_bound(items.begin(), items.end(), e,
+			[](const std::unique_ptr<event_base> & a, uint64_t id) -> bool {
+				return a->id < id;
+			});
+		if (i != items.end() && (*i)->id == e)
+		{
+			items.erase(i);
+		}
+	}
+
+	template <typename... Args>
+	void trigger(event_tag tag, Args... args)
+	{
+		using E = event<std::function<void(Args...)>>;
+		static event_base x = { tag, std::numeric_limits<int32_t>::max(), 0 };
+		auto i = sorted_items.lower_bound(&x);
+		static event_base y = { tag, std::numeric_limits<int32_t>::min(), 0 };
+		auto j = sorted_items.lower_bound(&y);
+		if (j != sorted_items.end()) ++j;
+		for (; i != j; ++i)
+		{
+			if ((*i)->tag != tag) break;
+			static_cast<const E *>(*i)->call(args...);
+		}
+	}
+};
+
+void remove_event_callback(uint64_t e) { events::get().remove(e); }
+
+template <>
+uint64_t add_event_callback(
+	event_tag tag, std::function<void(TARGET *)> && call, int32_t priority)
+{
+	return events::get().add(tag, std::move(call), priority);
+}
+
+void trigger_event_pre_exec_cmd(TARGET * t)
+{
+	events::get().trigger<TARGET *>(event_tag::pre_exec_cmd, t);
+}
+
+template <>
+uint64_t add_event_callback(
+	event_tag tag, std::function<void(int)> && call, int32_t priority)
+{
+	return events::get().add(tag, std::move(call), priority);
+}
+
+void trigger_event_exit_main(int status)
+{
+	events::get().trigger<int>(event_tag::exit_main, status);
+}
+
+} // namespace b2

src/engine/events.h

@@ -0,0 +1,36 @@
+/*
+Copyright 2024 René Ferdinand Rivera Morell
+Distributed under the Boost Software License, Version 1.0.
+(See accompanying file LICENSE.txt or https://www.bfgroup.xyz/b2/LICENSE.txt)
+*/
+
+#ifndef B2_EVENTS_H
+#define B2_EVENTS_H
+
+#include "config.h"
+
+#include "rules.h"
+
+#include <cstdint>
+#include <functional>
+
+namespace b2 {
+
+enum class event_tag : uint16_t
+{
+	unknown = 0,
+	pre_exec_cmd,
+	exit_main
+};
+
+template <typename F>
+uint64_t add_event_callback(
+	event_tag tag, std::function<F> && call, int32_t priority = 0);
+void remove_event_callback(uint64_t e);
+
+void trigger_event_pre_exec_cmd(TARGET * t);
+void trigger_event_exit_main(int status);
+
+} // namespace b2
+
+#endif

src/engine/jam.cpp

@@ -118,6 +118,7 @@
 #include "compile.h"
 #include "constants.h"
 #include "debugger.h"
+#include "events.h"
 #include "filesys.h"
 #include "function.h"
 #include "hcache.h"
@@ -137,6 +138,7 @@
 #include "variable.h"
 #include "execcmd.h"
 #include "mod_sysinfo.h"
+#include "mod_command_db.h"
 
 #include <errno.h>
 #include <string.h>
@@ -157,6 +159,8 @@
 # include <windows.h>
 #endif
 
+#include "ext_bfgroup_lyra.h"
+
 struct globs globs =
 {
     0,          /* noexec */
@@ -338,6 +342,12 @@ int guarded_main( int argc, char * * argv )
         return EXITOK;
     }
 
+	// Process options.
+	lyra::cli cli;
+	b2::command_db::declare_args(cli);
+	cli |= lyra::arg([](const std::string&){}, "").cardinality(0,0);
+	auto cli_parse_result = cli.parse({arg_c, arg_v});
+
     /* Pick up interesting options. */
     if ( ( s = getoptval( optv, 'n', 0 ) ) )
     {
@@ -613,6 +623,7 @@ int guarded_main( int argc, char * * argv )
 
         PROFILE_EXIT( MAIN );
     }
+	b2::trigger_event_exit_main(status ? EXITBAD : EXITOK);
 
     return status ? EXITBAD : EXITOK;
 }

src/engine/jam.h

@@ -504,6 +504,8 @@ struct globs
 
 extern struct globs globs;
 
+extern int anyhow;
+
 #define DEBUG_MAKE     ( globs.debug[ 1 ] )   /* show actions when executed */
 #define DEBUG_MAKEQ    ( globs.debug[ 2 ] )   /* show even quiet actions */
 #define DEBUG_EXEC     ( globs.debug[ 2 ] )   /* show text of actons */

src/engine/lists.h

@@ -504,6 +504,7 @@ struct list_ref : private list_cref
 
 	// list operations
 	inline list_ref & slice(size_type i, size_type j = -1);
+	inline list_cref cref() const;
 };
 // end::reference[]
 /* tag::reference[]
@@ -724,6 +725,8 @@ inline list_ref & list_ref::slice(size_type i, size_type j)
 	return *this;
 }
 
+inline list_cref list_ref::cref() const { return list_cref(list_obj); }
+
 /* tag::reference[]
 
 = `b2::lists`

src/engine/make1.cpp

@@ -41,6 +41,7 @@
 
 #include "command.h"
 #include "compile.h"
+#include "events.h"
 #include "execcmd.h"
 #include "headers.h"
 #include "lists.h"
@@ -597,6 +598,9 @@ static void make1c( state const * const pState )
             exec_flags |= EXEC_CMD_QUIET;
         }
 
+        // Signal that we are about to execute a command.
+        b2::trigger_event_pre_exec_cmd(pState->t);
+
         /* Execute the actual build command or fake it if no-op. */
         if ( globs.noexec || cmd->noop )
         {

src/engine/mod_command_db.cpp

@@ -0,0 +1,165 @@
+/*
+Copyright 2024 René Ferdinand Rivera Morell
+Distributed under the Boost Software License, Version 1.0.
+(See accompanying file LICENSE.txt or https://www.bfgroup.xyz/b2/LICENSE.txt)
+*/
+
+#include "mod_command_db.h"
+
+#include "command.h"
+#include "cwd.h"
+#include "events.h"
+#include "lists.h"
+#include "mod_db.h"
+#include "output.h"
+#include "pathsys.h"
+#include "regexp.h"
+
+#include "ext_bfgroup_lyra.h"
+
+#include <unordered_map>
+
+namespace b2 { namespace command_db {
+
+struct database
+{
+	bool output_flag = false;
+	std::string output_format = "json";
+	std::string output_filename = "compile_commands.json";
+	std::string output_directory;
+	std::string db_directory;
+	std::unordered_map<std::string, std::unique_ptr<regex::program>>
+		regex_cache;
+	std::unique_ptr<property_db> prop_db;
+	uint64_t command_index = 0;
+
+	database()
+	{
+		// B2 doesn't change directories. And runs everything relative to CWD.
+		// So we can cache the value to fill into the database.
+		db_directory = b2::cwd_str();
+		output_directory = db_directory;
+	}
+
+	static database & get()
+	{
+		static database db;
+		return db;
+	}
+
+	void set_output_format(const std::string & f)
+	{
+		// Set up for the format and create the output database.
+		output_flag = true;
+		output_format = f;
+		prop_db.reset(new property_db);
+
+		// Default to all targets and no regular action output.
+		++globs.noexec;
+		for (int i = 0; i < DEBUG_MAX; ++i) globs.debug[i] = 0;
+		++anyhow;
+
+		// Events to track the commands and exit to generate the output.
+		add_event_callback(event_tag::pre_exec_cmd,
+			std::function<void(TARGET *)>(
+				[](TARGET * t) { database::get().pre_exec_cmd(t); }));
+		add_event_callback(event_tag::exit_main,
+			std::function<void(int)>(
+				[](int s) { database::get().exit_main(s); }));
+	}
+
+	void set_output_filename(const std::string & f) { output_filename = f; }
+
+	void pre_exec_cmd(TARGET * t)
+	{
+		CMD * cmd = (CMD *)t->cmds;
+		auto args_out = list_cref(lol_get((LOL *)&cmd->args, 0));
+		auto args_in = list_cref(lol_get((LOL *)&cmd->args, 1));
+		auto settings = t->settings;
+		for (; settings; settings = settings->next)
+		{
+			value_ref symbol(settings->symbol);
+			list_cref value(settings->value);
+			if (symbol == "COMMAND_DATABASE" && !value.empty())
+			{
+				auto db_file = args_in[0]->str();
+				auto db_output = args_out[0]->str();
+				auto db_command
+					= extract_command(value[0]->str(), cmd->buf->value);
+				list_ref db_node;
+				db_node.push_back("[]").push_back(double(command_index++));
+				prop_db->emplace(
+					list_ref(db_node).push_back("output").cref(), db_output);
+				prop_db->emplace(
+					list_ref(db_node).push_back("file").cref(), db_file);
+				prop_db->emplace(
+					list_ref(db_node).push_back("directory").cref(),
+					db_directory);
+				prop_db->emplace(
+					list_ref(db_node).push_back("command").cref(), db_command);
+			}
+		}
+	}
+
+	std::string extract_command(const char * cmd_regex, const char * cmd_text)
+	{
+		regex::program * regex_prog = nullptr;
+		auto regex_prog_i = regex_cache.find(cmd_regex);
+		if (regex_prog_i == regex_cache.end())
+		{
+			auto regex_prog_e = std::unique_ptr<regex::program>(
+				new regex::program(cmd_regex));
+			regex_prog = regex_prog_e.get();
+			regex_cache.emplace(cmd_regex, std::move(regex_prog_e));
+		}
+		else
+		{
+			regex_prog = regex_prog_i->second.get();
+		}
+		auto regex_sub = regex_prog->search(cmd_text)[0];
+		return std::string(
+			regex_sub.cbegin(), regex_sub.cend() - regex_sub.begin());
+	}
+
+	void exit_main(int status)
+	{
+		if (status == EXIT_FAIL) return;
+		std::string filename = output_filename;
+		if (!b2::paths::is_rooted(output_filename))
+		{
+			if (!b2::paths::is_rooted(output_directory))
+				filename = b2::cwd_str() + "/";
+			filename += output_directory + "/" + output_filename;
+			filename = b2::paths::normalize(filename);
+		}
+		if (prop_db->write_file(filename, output_format))
+			out_printf("...wrote command database '%s'...\n", filename.c_str());
+		else
+			err_printf("...writing to command database '%s' FAILED...\n",
+				filename.c_str());
+	}
+};
+
+void declare_args(lyra::cli & cli)
+{
+	cli |= lyra::opt(
+		[](const std::string & f) { database::get().set_output_format(f); },
+		"format")
+			   .name("--command-database")
+			   .help("Output a compile commands database as format.");
+	cli |= lyra::opt(
+		[](const std::string & f) { database::get().set_output_filename(f); },
+		"filename")
+			   .name("--command-database-out")
+			   .help(
+				   "Filename to output the command database to. "
+				   "A relative path for the filename is rooted to the project "
+				   "build-dir.");
+}
+
+void set_output_dir(value_ref dirname)
+{
+	database::get().output_directory = dirname;
+}
+
+}} // namespace b2::command_db

src/engine/mod_command_db.h

@@ -0,0 +1,44 @@
+/*
+Copyright 2024 René Ferdinand Rivera Morell
+Distributed under the Boost Software License, Version 1.0.
+(See accompanying file LICENSE.txt or https://www.bfgroup.xyz/b2/LICENSE.txt)
+*/
+
+#ifndef B2_COMMAND_DB_H
+#define B2_COMMAND_DB_H
+
+#include "config.h"
+
+#include "bind.h"
+#include "value.h"
+
+namespace lyra {
+class cli;
+} // namespace lyra
+
+namespace b2 {
+
+namespace command_db {
+
+void declare_args(lyra::cli &);
+
+void set_output_dir(value_ref dirname);
+
+} // namespace command_db
+
+struct command_db_module : b2::bind::module_<command_db_module>
+{
+	const char * module_name = "command-db";
+
+	template <class Binder>
+	void def(Binder & binder)
+	{
+		binder.def(
+			&command_db::set_output_dir, "set-output-dir", ("dirname" * _1));
+		binder.loaded();
+	}
+};
+
+} // namespace b2
+
+#endif

src/engine/mod_db.cpp

@@ -42,13 +42,14 @@ void b2::property_db::emplace(list_cref k, value_ref v)
 	db[nk] = v;
 }
 
-void b2::property_db::write_file(value_ref filename, value_ref format)
+bool b2::property_db::write_file(value_ref filename, value_ref format)
 {
 	if (!format->has_value())
 	{
 		format = b2::value::make("json");
 	}
-	if (format == "json") write_file_json(filename);
+	if (format == "json") return write_file_json(filename);
+	return false;
 }
 
 std::string b2::property_db::dump(value_ref format)
@@ -101,19 +102,23 @@ void build_json_from_db(const T & db, nlohmann::json & out)
 }
 } // namespace
 
-void b2::property_db::write_file_json(value_ref filename)
+bool b2::property_db::write_file_json(value_ref filename)
 {
 	nlohmann::json out;
 	build_json_from_db(db, out);
 	FILE * file = std::fopen(filename->str(), "w");
-	try
+	if (file)
 	{
-		auto data = out.dump(0);
-		std::fwrite(data.c_str(), data.size(), 1, file);
+		try
+		{
+			auto data = out.dump(0);
+			return std::fwrite(data.c_str(), data.size(), 1, file) == 1;
+		}
+		catch (const std::exception &)
+		{}
+		std::fclose(file);
 	}
-	catch (const std::exception &)
-	{}
-	std::fclose(file);
+	return false;
 }
 
 std::string b2::property_db::dump_json()

src/engine/mod_db.h

@@ -36,7 +36,7 @@ end::reference[] */
 struct property_db : public object
 {
 	void emplace(list_cref k, value_ref v);
-	void write_file(value_ref filename, value_ref format);
+	bool write_file(value_ref filename, value_ref format);
 	std::string dump(value_ref format);
 
 	private:
@@ -58,7 +58,7 @@ struct property_db : public object
 	};
 	using db_type = std::map<list_ref, value_ref, key_less>;
 	db_type db;
-	void write_file_json(value_ref filename);
+	bool write_file_json(value_ref filename);
 	std::string dump_json();
 };
 

src/tools/como-linux.jam

@@ -12,6 +12,7 @@ import feature ;
 import toolset : flags ;
 import common ;
 import generators ;
+import regex ;
 
 import unix ;
 import como ;
@@ -85,11 +86,27 @@ actions link.dll bind LIBRARIES
     $(CONFIG_COMMAND) $(LINKFLAGS) -shared -o "$(<[1])" "$(>)" -L"$(LINKPATH)" -Wl,-R$(SPACE)-Wl,"$(RPATH)" -Wl,-rpath-link$(SPACE)-Wl,"$(RPATH_LINK)" "$(LIBRARIES)"  "$(LIBRARIES)" -l$(FINDLIBS-SA) -l$(FINDLIBS-ST) 2>&1
 }
 
+rule compile.c ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "($(config_command).*) 2\\>\\&1" ;
+}
+
 actions compile.c
 {
     $(CONFIG_COMMAND) -c --c99 --long_long -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -o "$(<)" "$(>)" 2>&1
 }
 
+rule compile.c++ ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "($(config_command).*) 2\\>\\&1" ;
+}
+
 actions compile.c++
 {
     $(CONFIG_COMMAND) -tused -c --long_long -U$(UNDEFS) -D$(DEFINES) $(CFLAGS) $(C++FLAGS) -I"$(HDRS)" -I"$(STDHDRS)"  -o "$(<)" "$(>)" 2>&1

src/tools/como-win.jam

@@ -15,6 +15,7 @@ import como ;
 import feature ;
 import generators ;
 import toolset : flags ;
+import regex ;
 
 feature.extend-subfeature toolset como : platform : win ;
 
@@ -100,11 +101,27 @@ actions link bind LIBRARIES
     $(CONFIG_COMMAND) --no_version --no_prelink_verbose $(LINKFLAGS) -o "$(<[1]:S=)" @"@($(<[1]:W).rsp:E=$(nl)"$(>)")" "$(LIBRARIES)" "$(FINDLIBS:S=.lib)"
 }
 
+rule compile.c ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "($(config_command)[^\n]*)" ;
+}
+
 actions compile.c
 {
     $(CONFIG_COMMAND) -c --c99 -e5 --no_version --display_error_number --diag_suppress=9,21,161,748,940,962 -U$(UNDEFS) -D$(DEFINES) $(WARN) $(CFLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -I"$(SYSHDRS)" -o "$(<:D=)" "$(>)"
 }
 
+rule compile.c++ ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "($(config_command)[^\n]*)" ;
+}
+
 actions compile.c++
 {
     $(CONFIG_COMMAND) -c -e5 --no_version --no_prelink_verbose --display_error_number --long_long --diag_suppress=9,21,161,748,940,962 --diag_error=461 -D__STL_LONG_LONG -U$(UNDEFS) -D$(DEFINES) $(WARN) $(CFLAGS) $(C++FLAGS) -I"$(HDRS)" -I"$(STDHDRS)" -I"$(SYSHDRS)" -o "$(<)" "$(>)"

src/tools/gcc.jam

@@ -478,6 +478,14 @@ generators.register-c-compiler gcc.compile.mm  : OBJECTIVE_CPP : OBJ : <toolset>
 generators.register [ new fortran-compiling-generator
     gcc.compile.fortran : FORTRAN FORTRAN90 : OBJ : <toolset>gcc ] ;
 
+rule set_command_db ( targets + )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
 rule compile.c++.preprocess ( targets * : sources * : properties * )
 {
     # Some extensions are compiled as C++ by default. For others, we need to
@@ -486,6 +494,7 @@ rule compile.c++.preprocess ( targets * : sources * : properties * )
     {
         LANG on $(<) = "-x c++" ;
     }
+    gcc.set_command_db $(targets) ;
     DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
 }
 
@@ -498,6 +507,7 @@ rule compile.c.preprocess ( targets * : sources * : properties * )
     #{
         LANG on $(<) = "-x c" ;
     #}
+    gcc.set_command_db $(targets) ;
     DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
 }
 
@@ -509,6 +519,7 @@ rule compile.c++ ( targets * : sources * : properties * )
     {
         LANG on $(<) = "-x c++" ;
     }
+    gcc.set_command_db $(targets) ;
     DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
 }
 
@@ -521,11 +532,13 @@ rule compile.c ( targets * : sources * : properties * )
     #{
         LANG on $(<) = "-x c" ;
     #}
+    gcc.set_command_db $(targets) ;
     DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
 }
 
 rule compile.fortran ( targets * : sources * : properties * )
 {
+    gcc.set_command_db $(targets) ;
 }
 
 actions compile.c++ bind PCH_FILE
@@ -555,6 +568,7 @@ actions compile.fortran
 
 rule compile.asm ( targets * : sources * : properties * )
 {
+    gcc.set_command_db $(targets) ;
     LANG on $(<) = "-x assembler-with-cpp" ;
 }
 

src/tools/hp_cxx.jam

@@ -69,6 +69,8 @@ Specifies additional command line options that will be passed to the linker.
 #   optimization is needed.
 #
 
+import regex ;
+
 import feature generators common ;
 import toolset : flags ;
 
@@ -182,6 +184,12 @@ actions link.dll bind LIBRARIES
 # Note: Relaxed ANSI mode (-std) is used for compilation because in strict ANSI
 # C89 mode (-std1) the compiler doesn't accept C++ comments in C files. As -std
 # is the default, no special flag is needed.
+
+rule compile.c ( targets * : sources * : properties * )
+{
+    COMMAND_DATABASE on $(targets) = "(cc [^\n]*)" ;
+}
+
 actions compile.c
 {
     $(.root:E=)cc -c $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -o "$(<)" "$(>)"
@@ -193,7 +201,7 @@ actions compile.c
 # Note: We deliberately don't suppress any warnings on the compiler command
 # line, the user can always do this in a customized toolset later on.
 
-rule compile.c++
+rule compile.c++ ( targets * : sources * : properties * )
 {
     # We preprocess the TEMPLATE_DEPTH command line option here because we found
     # no way to do it correctly in the actual action code. There we either get
@@ -202,6 +210,10 @@ rule compile.c++
     # "-pending_instantiations ".
     local template-depth = [ on $(1) return $(TEMPLATE_DEPTH) ] ;
     TEMPLATE_DEPTH on $(1) = "-pending_instantiations "$(template-depth) ;
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "($(config_command)[^\n]*)" ;
 }
 
 actions compile.c++

src/tools/intel-darwin.jam

@@ -13,6 +13,7 @@ import gcc ;
 import common ;
 import errors ;
 import generators ;
+import regex ;
 
 feature.extend-subfeature toolset intel : platform : darwin ;
 
@@ -159,11 +160,27 @@ flags intel-darwin.compile.c++ OPTIONS <warnings>extra : -w3 ;
 flags intel-darwin.compile.c++ OPTIONS <warnings>pedantic : -w3 -Wcheck ;
 flags intel-darwin.compile.c++ OPTIONS <warnings-as-errors>on : -Werror-all ;
 
+rule compile.c ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
 actions compile.c
 {
     "$(CONFIG_COMMAND)" -xc $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
 }
 
+rule compile.c++ ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
 actions compile.c++
 {
     "$(CONFIG_COMMAND)" -xc++ $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"

src/tools/intel-linux.jam

@@ -243,8 +243,17 @@ rule init ( version ? :  command * : options * )
 
 _ = " " ;
 
+rule set_command_db ( targets + )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
 rule compile.c++ ( targets * : sources * : properties * )
 {
+    set_command_db $(targets) ;
     DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
 }
 
@@ -255,6 +264,7 @@ actions compile.c++ bind PCH_FILE
 
 rule compile.c ( targets * : sources * : properties * )
 {
+    set_command_db $(targets) ;
     DEPENDS $(<) : [ on $(<) return $(PCH_FILE) ] ;
 }
 
@@ -263,6 +273,16 @@ actions compile.c bind PCH_FILE
     LD_LIBRARY_PATH="$(RUN_PATH)" "$(CONFIG_COMMAND)" -c -xc $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -use-pch"$(PCH_FILE)" -c -o "$(<)" "$(>)"
 }
 
+rule compile.c++.pch ( targets * : sources * : properties * )
+{
+    set_command_db $(targets) ;
+}
+
+rule compile.c.pch ( targets * : sources * : properties * )
+{
+    set_command_db $(targets) ;
+}
+
 #
 # Compiling a pch first deletes any existing *.pchi file, as Intel's compiler
 # won't over-write an existing pch: instead it creates filename$1.pchi, filename$2.pchi

src/tools/intel-vxworks.jam

@@ -12,6 +12,7 @@ import gcc ;
 import common ;
 import errors ;
 import generators ;
+import regex ;
 
 feature.extend-subfeature toolset intel : platform : vxworks ;
 
@@ -137,11 +138,29 @@ flags intel-vxworks.compile.c++ OPTIONS <warnings>extra : -w3 ;
 flags intel-vxworks.compile.c++ OPTIONS <warnings>pedantic : -w3 -Wcheck ;
 flags intel-vxworks.compile.c++ OPTIONS <warnings-as-errors>on : -Werror-all ;
 
+rule set_command_db ( targets + )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
+rule compile.c ( targets * : sources * : properties * )
+{
+    set_command_db $(targets) ;
+}
+
 actions compile.c
 {
     "$(CONFIG_COMMAND)" -xc $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
 }
 
+rule compile.c++ ( targets * : sources * : properties * )
+{
+    set_command_db $(targets) ;
+}
+
 actions compile.c++
 {
     "$(CONFIG_COMMAND)" -xc++ $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"

src/tools/mipspro.jam

@@ -11,6 +11,7 @@ import feature ;
 import fortran ;
 import type ;
 import common ;
+import regex ;
 
 feature.extend toolset : mipspro ;
 toolset.inherit mipspro : unix ;
@@ -81,11 +82,27 @@ flags mipspro.compile INCLUDES <include> ;
 
 flags mipspro.compile.fortran OPTIONS <fflags> ;
 
+rule compile.c ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_C_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
 actions compile.c
 {
     "$(CONFIG_C_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"
 }
 
+rule compile.c++ ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
 actions compile.c++
 {
     "$(CONFIG_COMMAND)" -FE:template_in_elf_section -ptused $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"

src/tools/msvc.jam

@@ -229,6 +229,7 @@ import project ;
 import property ;
 import property-set ;
 import rc ;
+import regex ;
 import sequence ;
 import set ;
 import testing ;
@@ -683,6 +684,19 @@ rule compile.c ( targets + : sources * : properties * )
     set-setup-command $(targets) : $(properties) ;
     get-rspline $(targets) : -TC CFLAGS  ;
     compile-c-c++ $(<) : $(>) ;
+
+    local cc = [ on $(targets[1]) return $(.CC) ] ;
+    cc = [ regex.escape "$(cc:J= )" : "|()[]\\+.*^$\" " : "\\" ] ;
+    local eol = [ on $(targets[1]) return $(.CC.FILTER) ] ;
+    if $(eol)
+    {
+        eol = ".*" [ regex.escape "$(eol:J= )" : "|()[]\\+.*^$\" " : "\\" ] ;
+    }
+    else
+    {
+        eol = "[^\n]*" "" ;
+    }
+    COMMAND_DATABASE on $(targets) = "($(cc)$(eol[1]))$(eol[2])" ;
 }
 
 
@@ -751,6 +765,19 @@ rule compile-c-c++ ( targets + : sources * )
     LOCATE on $(<[1]:S=.pdb) = [ on $(<[1]) return $(LOCATE) ] ;
     local pch-header = [ on $(<[1]) return $(PCH_HEADER) ] ;
     PCH_HEADER_AS_SPELLED on $(<[1]) = $(pch-header:G=) ;
+
+    local cc = [ on $(targets[1]) return $(.CC) ] ;
+    cc = [ regex.escape "$(cc:J= )" : "|()[]\\+.*^$\" " : "\\" ] ;
+    local eol = [ on $(targets[1]) return $(.CC.FILTER) ] ;
+    if $(eol)
+    {
+        eol = ".*" [ regex.escape "$(eol:J= )" : "|()[]\\+.*^$\" " : "\\" ] ;
+    }
+    else
+    {
+        eol = "[^\n]*" "" ;
+    }
+    COMMAND_DATABASE on $(targets) = "($(cc)$(eol[1]))$(eol[2])" ;
 }
 
 rule preprocess-c-c++ ( targets + : sources * )

src/tools/pathscale.jam

@@ -11,6 +11,7 @@ import type ;
 import os ;
 import common ;
 import fortran ;
+import regex ;
 
 feature.extend toolset : pathscale ;
 toolset.inherit pathscale : unix ;
@@ -100,6 +101,22 @@ flags pathscale.compile INCLUDES <include> ;
 flags pathscale.compile.fortran USER_OPTIONS <fflags> ;
 flags pathscale.compile.fortran90 USER_OPTIONS <fflags> ;
 
+rule compile.c ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_C_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
+rule compile.c++ ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
 actions compile.c
 {
     "$(CONFIG_C_COMMAND)" $(OPTIONS) $(USER_OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"

src/tools/pgi.jam

@@ -13,6 +13,7 @@ import fortran ;
 import type ;
 import common ;
 import gcc ;
+import regex ;
 
 feature.extend toolset : pgi ;
 toolset.inherit pgi : unix ;
@@ -81,6 +82,22 @@ flags pgi.compile INCLUDES <include> ;
 
 flags pgi.compile.fortran OPTIONS <fflags> ;
 
+rule compile.c ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_C_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
+rule compile.c++ ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
 actions compile.c
 {
     "$(CONFIG_C_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"

src/tools/qcc.jam

@@ -14,6 +14,7 @@ import feature ;
 import generators ;
 import os ;
 import property ;
+import regex ;
 import set ;
 import toolset ;
 import type ;
@@ -140,6 +141,11 @@ rule compile.c++
     }
 
     check-target-platform $(1) ;
+
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
 }
 
 actions compile.c++
@@ -150,6 +156,11 @@ actions compile.c++
 rule compile.c
 {
     check-target-platform $(1) ;
+
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
 }
 
 actions compile.c

src/tools/sun.jam

@@ -66,6 +66,7 @@ import toolset : flags ;
 import feature ;
 import type ;
 import common ;
+import regex ;
 
 feature.extend toolset : sun ;
 toolset.inherit  sun : unix ;
@@ -162,6 +163,22 @@ flags sun.compile.c++ OPTIONS <cxxflags> ;
 flags sun.compile DEFINES <define> ;
 flags sun.compile INCLUDES <include> ;
 
+rule compile.c ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_C_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
+rule compile.c++ ( targets * : sources * : properties * )
+{
+    local config_command = [ regex.escape
+        [ on $(targets[1]) return $(CONFIG_COMMAND) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "(\"$(config_command)\"[^\n]*)" ;
+}
+
 actions compile.c
 {
     "$(CONFIG_C_COMMAND)" $(OPTIONS) -D$(DEFINES) -I"$(INCLUDES)" -c -o "$(<)" "$(>)"

src/tools/vmsdecc.jam

@@ -17,6 +17,7 @@ import type ;
 import common ;
 import unix ;
 import path ;
+import regex ;
 
 
 if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
@@ -170,6 +171,10 @@ rule compile.c++ ( targets * : sources * : properties * )
 
   TARGET-CXX-REPO on $(targets) =  [ on $(targets[1]) get-target-cxx-repo $(LOCATE) ] ;
   CXX-REPOS on $(targets) = [ on $(targets) return $(TARGET-CXX-REPO) $(CXX-REPOS) ] ;
+
+    local config_command = [ regex.escape [ on $(targets[1]) return $(.CXX) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "($(config_command)[^\n]*)" ;
 }
 
 
@@ -178,6 +183,10 @@ rule compile.c ( targets * : sources * : properties * )
   DEPENDS $(targets) : [ on $(targets) return $(SOURCE-INCLUDES) ] ;
 
   INCLUDES on $(targets) = [ on $(targets) get-includes $(sources) : $(INCLUDES) ] ;
+
+    local config_command = [ regex.escape [ on $(targets[1]) return $(.CC) ]
+        : "()[]\\+.*^$\"" : "\\" ] ;
+    COMMAND_DATABASE on $(targets) = "($(config_command)[^\n]*)" ;
 }
 
 actions compile.c