Merge branch 'develop'

.gitattributes

@@ -0,0 +1,96 @@
+* text=auto !eol svneol=native#text/plain
+*.gitattributes text svneol=native#text/plain
+
+# Scriptish formats
+*.bat        text svneol=native#text/plain
+*.bsh        text svneol=native#text/x-beanshell
+*.cgi        text svneol=native#text/plain
+*.cmd        text svneol=native#text/plain
+*.js         text svneol=native#text/javascript
+*.php        text svneol=native#text/x-php
+*.pl         text svneol=native#text/x-perl
+*.pm         text svneol=native#text/x-perl
+*.py         text svneol=native#text/x-python
+*.sh         eol=lf svneol=LF#text/x-sh
+configure    eol=lf svneol=LF#text/x-sh
+
+# Image formats
+*.bmp        binary svneol=unset#image/bmp
+*.gif        binary svneol=unset#image/gif
+*.ico        binary svneol=unset#image/ico
+*.jpeg       binary svneol=unset#image/jpeg
+*.jpg        binary svneol=unset#image/jpeg
+*.png        binary svneol=unset#image/png
+*.tif        binary svneol=unset#image/tiff
+*.tiff       binary svneol=unset#image/tiff
+*.svg        text svneol=native#image/svg%2Bxml
+
+# Data formats
+*.pdf        binary svneol=unset#application/pdf
+*.avi        binary svneol=unset#video/avi
+*.doc        binary svneol=unset#application/msword
+*.dsp        text svneol=crlf#text/plain
+*.dsw        text svneol=crlf#text/plain
+*.eps        binary svneol=unset#application/postscript
+*.gz         binary svneol=unset#application/gzip
+*.mov        binary svneol=unset#video/quicktime
+*.mp3        binary svneol=unset#audio/mpeg
+*.ppt        binary svneol=unset#application/vnd.ms-powerpoint
+*.ps         binary svneol=unset#application/postscript
+*.psd        binary svneol=unset#application/photoshop
+*.rdf        binary svneol=unset#text/rdf
+*.rss        text svneol=unset#text/xml
+*.rtf        binary svneol=unset#text/rtf
+*.sln        text svneol=native#text/plain
+*.swf        binary svneol=unset#application/x-shockwave-flash
+*.tgz        binary svneol=unset#application/gzip
+*.vcproj     text svneol=native#text/xml
+*.vcxproj    text svneol=native#text/xml
+*.vsprops    text svneol=native#text/xml
+*.wav        binary svneol=unset#audio/wav
+*.xls        binary svneol=unset#application/vnd.ms-excel
+*.zip        binary svneol=unset#application/zip
+
+# Text formats
+.htaccess    text svneol=native#text/plain
+*.bbk        text svneol=native#text/xml
+*.cmake      text svneol=native#text/plain
+*.css        text svneol=native#text/css
+*.dtd        text svneol=native#text/xml
+*.htm        text svneol=native#text/html
+*.html       text svneol=native#text/html
+*.ini        text svneol=native#text/plain
+*.log        text svneol=native#text/plain
+*.mak        text svneol=native#text/plain
+*.qbk        text svneol=native#text/plain
+*.rst        text svneol=native#text/plain
+*.sql        text svneol=native#text/x-sql
+*.txt        text svneol=native#text/plain
+*.xhtml      text svneol=native#text/xhtml%2Bxml
+*.xml        text svneol=native#text/xml
+*.xsd        text svneol=native#text/xml
+*.xsl        text svneol=native#text/xml
+*.xslt       text svneol=native#text/xml
+*.xul        text svneol=native#text/xul
+*.yml        text svneol=native#text/plain
+boost-no-inspect text svneol=native#text/plain
+CHANGES      text svneol=native#text/plain
+COPYING      text svneol=native#text/plain
+INSTALL      text svneol=native#text/plain
+Jamfile      text svneol=native#text/plain
+Jamroot      text svneol=native#text/plain
+Jamfile.v2   text svneol=native#text/plain
+Jamrules     text svneol=native#text/plain
+Makefile*    text svneol=native#text/plain
+README       text svneol=native#text/plain
+TODO         text svneol=native#text/plain
+
+# Code formats
+*.c          text svneol=native#text/plain
+*.cpp        text svneol=native#text/plain
+*.h          text svneol=native#text/plain
+*.hpp        text svneol=native#text/plain
+*.ipp        text svneol=native#text/plain
+*.tpp        text svneol=native#text/plain
+*.jam        text svneol=native#text/plain
+*.java       text svneol=native#text/plain

build/Jamfile.v2

@@ -0,0 +1,30 @@
+# Copyright 2014, 2015 Peter Dimov
+#
+# Distributed under the Boost Software License, Version 1.0.
+# See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt
+
+project : default-build release ;
+
+local SOURCES =
+
+  bpm.cpp cmd_headers.cpp cmd_index.cpp cmd_install.cpp
+  cmd_list.cpp cmd_remove.cpp config.cpp dependencies.cpp
+  error.cpp file_reader.cpp fs.cpp http_reader.cpp json.cpp
+  lzma_reader.cpp message.cpp options.cpp package_path.cpp
+  string.cpp tar.cpp tcp_reader.cpp lzma/LzmaDec.c ;
+
+lib ws2_32 ;
+
+exe bpm : ../src/$(SOURCES) :
+
+          <os>NT:<library>ws2_32
+
+          <toolset>msvc:<runtime-link>static
+          <toolset>msvc:<cxxflags>/wd4996 ;
+
+install dist-bin : bpm : <location>../../../dist/bin ;
+explicit dist-bin ;
+
+alias install : dist-bin ;
+explicit install ;

scripts/package.bat

@@ -0,0 +1,28 @@
+@REM This cmd.exe batch script generates
+@REM   a package-based Boost distribution
+@REM   for use with bpm.
+@REM 
+@REM It needs to be run from the Boost root.
+@REM 
+@REM Copyright 2014, 2015 Peter Dimov
+@REM 
+@REM Distributed under the Boost Software License, Version 1.0.
+@REM See accompanying file LICENSE_1_0.txt or copy at
+@REM http://www.boost.org/LICENSE_1_0.txt
+
+FOR /f %%i IN ('git rev-parse HEAD') DO @SET REV=%%i
+
+FOR /f %%i IN ('git rev-parse --short HEAD') DO @SET SHREV=%%i
+
+FOR /f %%i IN ('git rev-parse --abbrev-ref HEAD') DO @SET BRANCH=%%i
+
+SET OUTDIR=..\pkg-%BRANCH%-%SHREV%
+
+mkdir %OUTDIR%
+
+dist\bin\boostdep.exe --track-sources --list-dependencies | xz --format=lzma > %OUTDIR%\dependencies.txt.lzma
+dist\bin\boostdep.exe --list-buildable | xz --format=lzma > %OUTDIR%\buildable.txt.lzma
+
+FOR /d %%i IN (libs/*) DO tar cf %OUTDIR%\%%i.tar.lzma --lzma libs/%%i/
+
+tar cf %OUTDIR%\build.tar.lzma --lzma b2.exe boost-build.jam boostcpp.jam Jamroot libs/Jamfile.v2 tools/build/

src/basic_reader.hpp

@@ -0,0 +1,34 @@
+#ifndef BASIC_READER_HPP_INCLUDED
+#define BASIC_READER_HPP_INCLUDED
+
+//
+// Copyright 2014 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include <string>
+#include <cstddef>
+
+class basic_reader
+{
+protected:
+
+    ~basic_reader()
+    {
+    }
+
+public:
+
+    virtual std::string name() const = 0;
+
+    // read throws on error, and returns number of bytes read
+    // which can be less than n when the end of the stream has
+    // been reached
+
+    virtual std::size_t read( void * p, std::size_t n ) = 0;
+};
+
+#endif // #ifndef BASIC_READER_HPP_INCLUDED

src/bpm.cpp

@@ -0,0 +1,155 @@
+//
+// bpm - a tool to install Boost modules
+//
+// Copyright 2014, 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "message.hpp"
+#include "error.hpp"
+#include "config.hpp"
+#include "options.hpp"
+#include "cmd_install.hpp"
+#include "cmd_headers.hpp"
+#include "cmd_index.hpp"
+#include "cmd_remove.hpp"
+#include "cmd_list.hpp"
+#include <string>
+#include <exception>
+#include <stdexcept>
+#include <stdio.h>
+#include <stdlib.h>
+
+static void handle_option( std::string const & opt )
+{
+    if( opt == "-v" )
+    {
+        increase_message_level();
+    }
+    else if( opt == "-q" )
+    {
+        decrease_message_level();
+    }
+    else
+    {
+        throw std::runtime_error( "invalid option: '" + opt + "'" );
+    }
+}
+
+static void usage()
+{
+    printf( "%s",
+
+        "Usage: bpm [-v] [-q] command [options] [modules]\n\n"
+
+        "  -v:  Be verbose\n"
+        "  -vv: Be more verbose\n"
+        "  -q:  Be quiet\n\n"
+
+        "  bpm install [-n] [+d] [-k] [-a] [-i] [-p] <module> <module>...\n\n"
+
+        "    Installs the specified modules and their dependencies into\n"
+        "    the current directory.\n\n"
+
+        "    -n: Only output what would be installed\n"
+        "    +d: Do not install dependencies\n"
+        "    -k: Do not remove partial installations on error\n"
+        "    -a: All modules (use instead of a module list)\n"
+        "    -i: Installed modules\n"
+        "    -p: Partially installed modules\n\n"
+
+        "  bpm remove [-n] [-f] [-d] [-a] [-p] <package> <package>...\n\n"
+
+        "    Removes the specified packages.\n\n"
+
+        "    (A package, like 'numeric', can contain more than one module.)\n\n"
+
+        "    -n: Only output what would be removed\n"
+        "    -f: Force removal even when dependents exist\n"
+        "    -d: Remove dependents as well. Requires -f\n"
+        "    -a: Remove all packages. Requires -f\n"
+        "    -p: Remove partially installed packages\n\n"
+
+        "  bpm list [-a] [-i] [-p] [-b] [prefix]\n\n"
+
+        "    Lists modules matching [prefix].\n\n"
+
+        "    -a: All modules (default when no -b)\n"
+        "    -i: Installed modules (default when -b)\n"
+        "    -p: Partially installed modules\n"
+        "    -b: Modules that require building\n\n"
+
+        "  bpm headers\n\n"
+
+        "    Recreates the header links in the include/ subdirectory of\n"
+        "    the current directory.\n\n"
+
+        "  bpm index\n\n"
+
+        "    Recreates the file index.html, which lists the installed\n"
+        "    modules, in the current directory.\n"
+
+    );
+}
+
+int main( int argc, char const * argv[] )
+{
+    if( argc < 2 || std::string( argv[ 1 ] ) == "--help" )
+    {
+        usage();
+        return 0;
+    }
+
+    try
+    {
+        ++argv;
+
+        config_read_file( "bpm.conf" );
+
+        parse_options( argv, handle_option );
+
+        if( *argv == 0 )
+        {
+            throw std::runtime_error( "missing command" );
+        }
+
+        std::string command( *argv++ );
+
+        if( command == "install" )
+        {
+            cmd_install( argv );
+        }
+        else if( command == "remove" )
+        {
+            cmd_remove( argv );
+        }
+        else if( command == "headers" )
+        {
+            cmd_headers( argv );
+        }
+        else if( command == "index" )
+        {
+            cmd_index( argv );
+        }
+        else if( command == "list" )
+        {
+            cmd_list( argv );
+        }
+        else if( command == "help" )
+        {
+            usage();
+        }
+        else
+        {
+            throw std::runtime_error( "invalid command: '" + command + "'" );
+        }
+    }
+    catch( std::exception const & x )
+    {
+        fprintf( stderr, "bpm: %s\n", x.what() );
+        return EXIT_FAILURE;
+    }
+}

src/cmd_headers.cpp

@@ -0,0 +1,274 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "cmd_headers.hpp"
+#include "options.hpp"
+#include "message.hpp"
+#include "error.hpp"
+#include "fs.hpp"
+#include <stdexcept>
+#include <map>
+#include <cassert>
+#include <errno.h>
+
+static void handle_option( std::string const & opt )
+{
+    if( opt == "-v" )
+    {
+        increase_message_level();
+    }
+    else if( opt == "-q" )
+    {
+        decrease_message_level();
+    }
+    else
+    {
+        throw std::runtime_error( "invalid headers option: '" + opt + "'" );
+    }
+}
+
+void cmd_headers( char const * argv[] )
+{
+    parse_options( argv, handle_option );
+
+    if( *argv )
+    {
+        throw std::runtime_error( std::string( "unexpected argument '" ) + *argv + "'" );
+    }
+
+    cmd_headers();
+}
+
+static void removing( std::string const & path )
+{
+    msg_printf( 2, "removing '%s'", path.c_str() );
+}
+
+static void rmerror( std::string const & path, int err )
+{
+    throw_errno_error( path, "remove error", err );
+}
+
+static void build_dir_map2( std::string const & path, std::string const & suffix, std::map< std::string, std::vector< std::string > > & dirs )
+{
+    // enumerate header directories in 'path' + 'suffix'
+
+    std::vector< std::string > entries;
+    int r = fs_readdir( path + suffix, entries );
+
+    if( r != 0 )
+    {
+        throw_errno_error( path + suffix, "read error", errno );
+    }
+
+    for( std::vector< std::string >::const_iterator i = entries.begin(); i != entries.end(); ++i )
+    {
+        if( *i == "." || *i == ".." ) continue;
+
+        std::string sx2 = suffix + "/" + *i;
+
+        std::string p2 = path + sx2;
+
+        if( !fs_is_dir( p2 ) ) continue;
+
+        dirs[ sx2 ].push_back( p2 );
+
+        build_dir_map2( path, sx2, dirs );
+    }
+}
+
+static void build_dir_map( std::string const & path, std::map< std::string, std::vector< std::string > > & dirs )
+{
+    // enumerate modules in 'path'
+
+    std::vector< std::string > entries;
+    int r = fs_readdir( path, entries );
+
+    if( r != 0 )
+    {
+        throw_errno_error( path, "read error", errno );
+    }
+
+    for( std::vector< std::string >::const_iterator i = entries.begin(); i != entries.end(); ++i )
+    {
+        if( *i == "." || *i == ".." ) continue;
+
+        std::string p2 = path + "/" + *i;
+
+        if( fs_is_dir( p2 ) && fs_is_dir( p2 + "/include" ) )
+        {
+            build_dir_map2( p2 + "/", "include", dirs );
+        }
+
+        if( fs_exists( p2 + "/sublibs" ) )
+        {
+            // enumerate submodules
+            build_dir_map( p2, dirs );
+        }
+    }
+}
+
+static void link_directory( std::string const & dir, std::map< std::string, std::vector< std::string > > & dirs );
+
+static void link_files( std::string const & path, std::string const & target, std::map< std::string, std::vector< std::string > > & dirs )
+{
+    // msg_printf( 1, "linking files from '%s' into '%s'", path.c_str(), target.c_str() );
+
+    std::vector< std::string > entries;
+    int r = fs_readdir( path, entries );
+
+    if( r != 0 )
+    {
+        throw_errno_error( path, "read error", errno );
+    }
+
+    for( std::vector< std::string >::const_iterator i = entries.begin(); i != entries.end(); ++i )
+    {
+        if( *i == "." || *i == ".." ) continue;
+
+        std::string p2 = path + "/" + *i;
+        std::string t2 = target + "/" + *i;
+
+        if( fs_is_dir( p2 ) )
+        {
+            link_directory( t2, dirs );
+        }
+        else
+        {
+            // file
+            msg_printf( 1, "linking '%s' to '%s'", t2.c_str(), p2.c_str() );
+
+            if( fs_link_file( t2, p2 ) != 0 )
+            {
+                throw_errno_error( t2, "link create error", errno );
+            }
+        }
+    }
+}
+
+static void remove_directory_from_list( std::string const & dir, std::map< std::string, std::vector< std::string > > & dirs )
+{
+    for( std::map< std::string, std::vector< std::string > >::iterator i = dirs.find( dir ); i != dirs.end(); )
+    {
+        std::string d2 = i->first;
+
+        if( d2 == dir || d2.substr( 0, 1 + dir.size() ) == dir + "/" )
+        {
+            dirs.erase( i++ );
+        }
+        else
+        {
+            break;
+        }
+    }
+}
+
+static void link_directory( std::string const & dir, std::string const & target )
+{
+    msg_printf( 1, "linking '%s' to '%s'", dir.c_str(), target.c_str() );
+
+    if( fs_link_dir( dir, target ) != 0 )
+    {
+        throw_errno_error( dir, "link create error", errno );
+    }
+}
+
+static void create_directory( std::string const & dir )
+{
+    msg_printf( 1, "creating '%s'", dir.c_str() );
+
+    if( fs_mkdir( dir, 0755 ) != 0 )
+    {
+        throw_errno_error( dir, "create error", errno );
+    }
+}
+
+static void link_directory( std::string const & dir, std::map< std::string, std::vector< std::string > > & dirs )
+{
+    if( dirs.count( dir ) == 0 )
+    {
+        // already processed
+        return;
+    }
+
+    std::vector< std::string > const & d2 = dirs[ dir ];
+
+    assert( !d2.empty() );
+
+    if( d2.size() == 1 )
+    {
+        link_directory( dir, d2.front() );
+    }
+    else
+    {
+        create_directory( dir );
+
+        for( std::vector< std::string >::const_iterator i = d2.begin(); i != d2.end(); ++i )
+        {
+            link_files( *i, dir, dirs );
+        }
+    }
+
+    remove_directory_from_list( dir, dirs );
+}
+
+static void remove_directory( std::string const & dir )
+{
+    if( fs_exists( dir ) )
+    {
+        fs_remove_all( dir, removing, rmerror );
+    }
+}
+
+static void touch_file( std::string const & path )
+{
+    int fd = fs_creat( path, 0644 );
+
+    if( fd >= 0 )
+    {
+        fs_close( fd );
+    }
+}
+
+void cmd_headers()
+{
+    msg_printf( 0, "recreating header links" );
+
+    msg_printf( 1, "removing old header links" );
+
+    remove_directory( "include" );
+    remove_directory( "boost" );
+
+    if( !fs_exists( "libs" ) )
+    {
+        return;
+    }
+
+    std::map< std::string, std::vector< std::string > > dirs;
+    build_dir_map( "libs", dirs );
+
+    if( dirs.empty() )
+    {
+        return;
+    }
+
+    create_directory( "include" );
+
+    while( !dirs.empty() )
+    {
+        std::string dir = dirs.begin()->first;
+        link_directory( dir, dirs );
+    }
+
+    touch_file( "include/.updated" );
+
+    if( fs_exists( "include/boost" ) )
+    {
+        link_directory( "boost", "include/boost" );
+    }
+}

src/cmd_headers.hpp

@@ -0,0 +1,15 @@
+#ifndef CMD_HEADERS_HPP_INCLUDED
+#define CMD_HEADERS_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+void cmd_headers( char const * argv[] );
+void cmd_headers();
+
+#endif // #ifndef CMD_HEADERS_HPP_INCLUDED

src/cmd_index.cpp

@@ -0,0 +1,433 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "cmd_index.hpp"
+#include "options.hpp"
+#include "message.hpp"
+#include "error.hpp"
+#include "json.hpp"
+#include "fs.hpp"
+#include <map>
+#include <string>
+#include <vector>
+#include <fstream>
+#include <stdexcept>
+#include <errno.h>
+
+static void handle_option( std::string const & opt )
+{
+    if( opt == "-v" )
+    {
+        increase_message_level();
+    }
+    else if( opt == "-q" )
+    {
+        decrease_message_level();
+    }
+    else
+    {
+        throw std::runtime_error( "invalid index option: '" + opt + "'" );
+    }
+}
+
+void cmd_index( char const * argv[] )
+{
+    parse_options( argv, handle_option );
+
+    if( *argv )
+    {
+        throw std::runtime_error( std::string( "unexpected argument '" ) + *argv + "'" );
+    }
+
+    cmd_index();
+}
+
+typedef std::map< std::string, std::vector< std::string > > library;
+
+static std::string to_string( std::vector< std::string > const & v )
+{
+    std::string r;
+
+    for( std::size_t i = 0, n = v.size(); i < n; ++i )
+    {
+        if( i != 0 )
+        {
+            r += ", ";
+        }
+
+        r += v[ i ];
+    }
+
+    return r;
+}
+
+static std::string get_field( library const & lib, std::string const & name )
+{
+    library::const_iterator i = lib.find( name );
+
+    if( i == lib.end() )
+    {
+        return std::string();
+    }
+    else
+    {
+        return to_string( i->second );
+    }
+}
+
+static void set_field( library & lib, std::string const & name, std::string const & value )
+{
+    lib[ name ].resize( 1 );
+    lib[ name ].front() = value;
+}
+
+static void add_library( std::string const & path, std::map< std::string, library > & libraries, std::map< std::string, std::vector< std::string > > & categories )
+{
+    std::string name = path + "/meta/libraries.json";
+
+    msg_printf( 2, "reading '%s'", name.c_str() );
+
+    std::vector< library > v;
+
+    try
+    {
+        read_libraries_json( name, v );
+    }
+    catch( std::exception const & x )
+    {
+        msg_printf( -2, "%s", x.what() );
+    }
+
+    for( std::vector< library >::iterator i = v.begin(); i != v.end(); ++i )
+    {
+        library & lib = *i;
+
+        std::string key = get_field( lib, "key" );
+
+        if( key.empty() )
+        {
+            key = path;
+
+            int j = i - v.begin();
+
+            if( j != 0 )
+            {
+                char buffer[ 32 ];
+                sprintf( buffer, ".%d", j + 1 );
+
+                key += buffer;
+            }
+
+            msg_printf( -1, "'%s': library has no key, assuming '%s'", name.c_str(), key.c_str() );
+
+            set_field( lib, "key", key );
+        }
+
+        std::string name = get_field( lib, "name" );
+
+        if( name.empty() )
+        {
+            msg_printf( -1, "'%s': library '%s' has no name", name.c_str(), key.c_str() );
+        }
+
+        set_field( lib, "path", path );
+
+        std::vector< std::string > const & cv = lib[ "category" ];
+
+        libraries[ key ] = lib;
+
+        for( std::vector< std::string >::const_iterator j = cv.begin(); j != cv.end(); ++j )
+        {
+            categories[ *j ].push_back( key );
+        }
+    }
+}
+
+static void build_library_map( std::string const & path, std::map< std::string, library > & libraries, std::map< std::string, std::vector< std::string > > & categories )
+{
+    // enumerate modules in 'path'
+
+    std::vector< std::string > entries;
+    int r = fs_readdir( path, entries );
+
+    if( r != 0 )
+    {
+        throw_errno_error( path, "read error", errno );
+    }
+
+    for( std::vector< std::string >::const_iterator i = entries.begin(); i != entries.end(); ++i )
+    {
+        if( *i == "." || *i == ".." ) continue;
+
+        std::string p2 = path + "/" + *i;
+
+        if( fs_is_dir( p2 ) && fs_exists( p2 + "/meta/libraries.json" ) )
+        {
+            add_library( p2, libraries, categories );
+        }
+
+        if( fs_exists( p2 + "/sublibs" ) )
+        {
+            // enumerate submodules
+            build_library_map( p2, libraries, categories );
+        }
+    }
+}
+
+static void write_index( std::string const & name, std::map< std::string, library > const & libraries, std::map< std::string, std::vector< std::string > > const & categories );
+
+void cmd_index()
+{
+    msg_printf( 0, "recreating index" );
+
+    std::map< std::string, library > libraries;
+    std::map< std::string, std::vector< std::string > > categories;
+
+    build_library_map( "libs", libraries, categories );
+
+    write_index( "index.html", libraries, categories );
+}
+
+static char const * file_header =
+
+"<!DOCTYPE HTML>\n"
+"<html>\n"
+"\n"
+"<head>\n"
+"\n"
+"<title>Installed Boost C++ Libraries</title>\n"
+"<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />\n"
+"\n"
+"<style type=\"text/css\">\n"
+"\n"
+"A { color: #06C; text-decoration: none; }\n"
+"A:hover { text-decoration: underline; }\n"
+"\n"
+".main { margin-left: 4em; margin-right: 4em; color: #4A6484; }\n"
+"\n"
+".logo { font-family: sans-serif; font-style: italic; }\n"
+".logo .upper { font-size: 48pt; font-weight: 800; }\n"
+".logo .lower { font-size: 17pt; }\n"
+"\n"
+".header { margin-top: 2em; }\n"
+".section { margin-top: 2em; }\n"
+"\n"
+".category-container { margin-left: 1em; padding-left: 1em; border-left: 1px dotted; }\n"
+"\n"
+"dt { float: left; clear: left; width: 6em; }\n"
+"dd { margin-left: 7em; }\n"
+"\n"
+"#alphabetically p { margin-left: 2em; }\n"
+"#alphabetically dl { margin-left: 2em; }\n"
+"\n"
+"</style>\n"
+"\n"
+"</head>\n"
+"\n"
+"<body>\n"
+"<div class=\"main\">\n"
+"\n"
+"<div class=\"logo\">\n"
+"<div class=\"upper\">boost</div>\n"
+"<div class=\"lower\">C++ LIBRARIES</div>\n"
+"</div>\n\n";
+
+static void write_category_link( std::ostream & os, std::string const & name, std::string const & id )
+{
+    os << "<div><a href=\"#cat:" + id + "\">" + name + "</a></div>\n";
+}
+
+static void write_header( std::ostream & os, std::map< std::string, std::pair< std::string, std::vector< std::string > > > const & categories )
+{
+    os << "<div class=\"header\">\n"
+          "<div><a href=\"#alphabetically\">Libraries Listed Alphabetically</a></div>\n"
+          "<div><a href=\"#by_category\">Libraries Listed by Category</a></div>\n"
+          "<div>&nbsp;</div>\n";
+
+    for( std::map< std::string, std::pair< std::string, std::vector< std::string > > >::const_iterator i = categories.begin(); i != categories.end(); ++i )
+    {
+        write_category_link( os, i->first, i->second.first );
+    }
+
+    os << "</div>\n\n";
+}
+
+static std::string category_name( std::string const & cat );
+
+static void write_alpha_library( std::ostream & os, library const & lib )
+{
+    os << "<h4><a href=\"http://www.boost.org/" << get_field( lib, "path" ) << "/" << get_field( lib, "documentation" ) << "\">" << get_field( lib, "name" ) << "</a></h4>\n\n"
+
+          "<p class=\"description\">" << get_field( lib, "description" ) << "</p>\n\n"
+
+          "<dl>\n";
+
+    std::string authors = get_field( lib, "authors" );
+
+    if( !authors.empty() )
+    {
+        os << "<dt>Author(s):</dt><dd>" << authors << "</dd>\n";
+    }
+
+    {
+        library::const_iterator i = lib.find( "category" );
+
+        if( i != lib.end() )
+        {
+            os << "<dt>Category:</dt><dd>";
+
+            int k = 0;
+
+            for( std::vector< std::string>::const_iterator j = i->second.begin(); j != i->second.end(); ++j, ++k )
+            {
+                if( k != 0 )
+                {
+                    os << " &bull; ";
+                }
+
+                os << "<a href=\"#cat:" << *j << "\">" << category_name( *j ) << "</a>";
+            }
+
+            os << "</dd>\n";
+        }
+    }
+
+    os << "</dl>\n\n";
+}
+
+static void write_alpha_section( std::ostream & os, std::map< std::string, library > const & libraries )
+{
+    os << "<div class=\"section\" id=\"alphabetically\">\n"
+          "\n"
+          "<h2>Libraries Listed Alphabetically</h2>\n\n";
+
+    // re-sort by name
+    std::map< std::string, library > lib2;
+
+    for( std::map< std::string, library >::const_iterator i = libraries.begin(); i != libraries.end(); ++i )
+    {
+        std::string name = get_field( i->second, "name" );
+
+        if( name.empty() ) continue;
+
+        lib2[ name ] = i->second;
+    }
+
+    for( std::map< std::string, library >::const_iterator i = lib2.begin(); i != lib2.end(); ++i )
+    {
+        write_alpha_library( os, i->second );
+    }
+
+    os << "</div>\n\n";
+}
+
+static void write_cat_library( std::ostream & os, library const & lib )
+{
+    os << "<h4><a href=\"http://www.boost.org/" << get_field( lib, "path" ) << "/" << get_field( lib, "documentation" ) << "\">" << get_field( lib, "name" ) << "</a></h4>\n\n"
+
+          "<p class=\"description\">" << get_field( lib, "description" ) << "</p>\n\n";
+}
+
+static void write_category( std::ostream & os, std::map< std::string, library > const & libraries, std::string const & name, std::string const & id, std::vector< std::string > const & libs )
+{
+    os << "<h3 class=\"category\" id=\"cat:" << id << "\">" << name << "</h3>\n\n"
+          "<div class=\"category-container\">\n\n";
+
+    for( std::vector< std::string >::const_iterator i = libs.begin(); i != libs.end(); ++i )
+    {
+        std::map< std::string, library >::const_iterator j = libraries.find( *i );
+
+        if( j != libraries.end() )
+        {
+            write_cat_library( os, j->second );
+        }
+    }
+
+    os << "</div>\n\n";
+}
+
+static void write_cat_section( std::ostream & os, std::map< std::string, library > const & libraries, std::map< std::string, std::pair< std::string, std::vector< std::string > > > const & categories )
+{
+    os << "<div class=\"section\" id=\"by_category\">\n"
+          "\n"
+          "<h2>Libraries Listed by Category</h2>\n\n";
+
+    for( std::map< std::string, std::pair< std::string, std::vector< std::string > > >::const_iterator i = categories.begin(); i != categories.end(); ++i )
+    {
+        write_category( os, libraries, i->first, i->second.first, i->second.second );
+    }
+
+    os << "</div>\n\n";
+}
+
+static char const * file_footer =
+
+"</div>\n"
+"</body>\n"
+"</html>\n";
+
+static std::string category_name( std::string const & cat )
+{
+    static char const * table[][ 2 ] =
+    {
+        { "String", "String and Text Processing" },
+        { "Function-objects", "Function Objects and Higher-Order Programming" },
+        { "Generic", "Generic Programming" },
+        { "Metaprogramming", "Template Metaprogramming" },
+        { "Preprocessor", "Preprocessor Metaprogramming" },
+        { "Concurrent", "Concurrent Programming" },
+        { "Math", "Math and Numerics" },
+        { "Correctness", "Correctness and Testing" },
+        { "Data", "Data Structures" },
+        { "Domain", "Domain Specific" },
+        { "Image-processing", "Image Processing" },
+        { "IO", "Input/Output" },
+        { "Inter-language", "Inter-Language Support" },
+        { "Emulation", "Emulation of Language Features" },
+        { "Patterns", "Patterns and Idioms" },
+        { "Programming", "Programming Interfaces" },
+        { "State", "State Machines" },
+        { "workarounds", "Broken Compiler Workarounds" },
+    };
+
+    for( int i = 0, n = sizeof( table ) / sizeof( table[0] ); i < n; ++i )
+    {
+        if( cat == table[ i ][ 0 ] ) return table[ i ][ 1 ];
+    }
+
+    return cat;
+}
+
+static void write_index( std::string const & name, std::map< std::string, library > const & libraries, std::map< std::string, std::vector< std::string > > const & categories )
+{
+    msg_printf( 2, "writing '%s'", name.c_str() );
+
+    std::ofstream os( name.c_str() );
+
+    if( !os )
+    {
+        throw_errno_error( name, "open error", errno );
+    }
+
+    os << file_header;
+
+    // re-sort categories by name
+
+    std::map< std::string, std::pair< std::string, std::vector< std::string > > > cats;
+
+    for( std::map< std::string, std::vector< std::string > >::const_iterator i = categories.begin(); i != categories.end(); ++i )
+    {
+        cats[ category_name( i->first ) ] = *i;
+    }
+
+    write_header( os, cats );
+    write_alpha_section( os, libraries );
+    write_cat_section( os, libraries, cats );
+
+    os << file_footer;
+}

src/cmd_index.hpp

@@ -0,0 +1,15 @@
+#ifndef CMD_INDEX_HPP_INCLUDED
+#define CMD_INDEX_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+void cmd_index( char const * argv[] );
+void cmd_index();
+
+#endif // #ifndef CMD_INDEX_HPP_INCLUDED

src/cmd_install.cpp

@@ -0,0 +1,373 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "options.hpp"
+#include "dependencies.hpp"
+#include "message.hpp"
+#include "package_path.hpp"
+#include "cmd_headers.hpp"
+#include "cmd_index.hpp"
+
+#include "lzma_reader.hpp"
+#include "http_reader.hpp"
+#include "tar.hpp"
+
+#include "error.hpp"
+#include "fs.hpp"
+
+#include <algorithm>
+#include <stdexcept>
+#include <cstdio>
+#include <cstddef>
+#include <cstring>
+#include <errno.h>
+
+
+static bool s_opt_n = false;
+static bool s_opt_d = true;
+static bool s_opt_k = false;
+static bool s_opt_a = false;
+static bool s_opt_i = false;
+static bool s_opt_p = false;
+
+static void handle_option( std::string const & opt )
+{
+    if( opt == "-n" )
+    {
+        s_opt_n = true;
+    }
+    else if( opt == "+d" )
+    {
+        s_opt_d = false;
+    }
+    else if( opt == "-d" )
+    {
+        s_opt_d = true;
+    }
+    else if( opt == "-k" )
+    {
+        s_opt_k = true;
+    }
+    else if( opt == "-a" )
+    {
+        s_opt_a = true;
+    }
+    else if( opt == "-i" )
+    {
+        s_opt_i = true;
+    }
+    else if( opt == "-p" )
+    {
+        s_opt_p = true;
+    }
+    else if( opt == "-v" )
+    {
+        increase_message_level();
+    }
+    else if( opt == "-q" )
+    {
+        decrease_message_level();
+    }
+    else
+    {
+        throw std::runtime_error( "invalid install option: '" + opt + "'" );
+    }
+}
+
+static void removing( std::string const & path )
+{
+    msg_printf( 2, "removing '%s'", path.c_str() );
+}
+
+static void rmerror( std::string const & path, int err )
+{
+    msg_printf( 1, "'%s': remove error: %s", path.c_str(), std::strerror( err ) );
+}
+
+static void remove_partial_installation( std::string const & module, std::string const & path, std::set< std::string > const & files )
+{
+    if( fs_exists( path ) )
+    {
+        msg_printf( 1, "removing partial installation of module '%s'", module.c_str() );
+        fs_remove_all( path, removing, rmerror );
+    }
+
+    for( std::set< std::string >::const_iterator i = files.begin(); i != files.end(); ++i )
+    {
+        if( fs_exists( *i ) )
+        {
+            removing( *i );
+
+            int r = std::remove( i->c_str() );
+
+            if( r != 0 )
+            {
+                rmerror( *i, errno );
+            }
+        }
+    }
+}
+
+static void touch_file( std::string const & path )
+{
+    int fd = fs_creat( path, 0644 );
+
+    if( fd >= 0 )
+    {
+        fs_close( fd );
+    }
+}
+
+static std::string module_package( std::string const & module )
+{
+    std::size_t i = module.find( '~' );
+
+    std::string package = module.substr( 0, i );
+
+    return package;
+}
+
+static void install_module( std::string const & package_path, std::string const & module, std::set< std::string > & installed, std::time_t & mtime )
+{
+    std::string package = module_package( module );
+
+    std::string path = "libs/" + package;
+
+    std::set< std::string > whitelist;
+
+    if( module == "build" )
+    {
+        if( !fs_exists( "tools" ) )
+        {
+            fs_mkdir( "tools/", 0755 );
+        }
+
+        path = "tools/" + module;
+
+        whitelist.insert( "b2.exe" );
+        whitelist.insert( "boost-build.jam" );
+        whitelist.insert( "boostcpp.jam" );
+        whitelist.insert( "Jamroot" );
+        whitelist.insert( "libs/Jamfile.v2" );
+    }
+
+    std::string marker = path + "/.installed";
+
+    if( !fs_exists( marker ) )
+    {
+        if( s_opt_n )
+        {
+            msg_printf( 0, "would have installed module '%s'", module.c_str() );
+        }
+        else
+        {
+            remove_partial_installation( module, path, whitelist );
+
+            msg_printf( 0, "installing module '%s'", module.c_str() );
+
+            std::string tar_path = package_path + package + ".tar.lzma";
+
+            http_reader r1( tar_path );
+            lzma_reader r2( &r1 );
+
+            try
+            {
+                tar_extract( &r2, path + '/', whitelist );
+                touch_file( marker );
+            }
+            catch( std::exception const & )
+            {
+                if( !s_opt_k )
+                {
+                    remove_partial_installation( module, path, whitelist );
+                }
+
+                throw;
+            }
+        }
+
+        installed.insert( module );
+    }
+    else
+    {
+        msg_printf( 1, "module '%s' is already installed", module.c_str() );
+    }
+
+    mtime = std::max( mtime, fs_mtime( marker ) );
+}
+
+static void install_boost_build( std::string const & package_path, std::set< std::string > & installed )
+{
+    std::time_t mtime = 0;
+    install_module( package_path, "build", installed, mtime );
+}
+
+void cmd_install( char const * argv[] )
+{
+    parse_options( argv, handle_option );
+
+    if( s_opt_a + s_opt_i + s_opt_p > 1 )
+    {
+        throw std::runtime_error( "install options -a, -i and -p are mutually exclusive" );
+    }
+
+    if( ( s_opt_a || s_opt_i || s_opt_p ) && *argv != 0 )
+    {
+        throw std::runtime_error( "install options -a, -i and -p are incompatible with a module list" );
+    }
+
+    std::string package_path = get_package_path();
+
+    std::map< std::string, std::vector< std::string > > deps;
+    std::set< std::string > buildable;
+
+    retrieve_dependencies( deps, buildable );
+
+    if( !fs_exists( "libs" ) )
+    {
+        fs_mkdir( "libs/", 0755 );
+    }
+
+    std::vector< std::string > modules;
+
+    if( s_opt_a )
+    {
+        for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i )
+        {
+            modules.push_back( i->first );
+        }
+    }
+    else if( s_opt_i || s_opt_p )
+    {
+        for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i )
+        {
+            std::string module = i->first;
+
+            std::string package = module_package( module );
+
+            std::string path( module );
+            std::replace( path.begin(), path.end(), '~', '/' );
+
+            if( fs_exists( "libs/" + path ) && s_opt_p != fs_exists( "libs/" + package + "/.installed" ) )
+            {
+                modules.push_back( i->first );
+            }
+        }
+    }
+    else
+    {
+        while( *argv )
+        {
+            modules.push_back( *argv );
+            ++argv;
+        }
+    }
+
+    std::set< std::string > installed;
+    std::set< std::string > need_build;
+
+    std::time_t mtime = 0;
+
+    {
+        std::size_t i = 0;
+
+        while( i < modules.size() )
+        {
+            std::string module = modules[ i++ ];
+
+            if( deps.count( module ) == 0 )
+            {
+                msg_printf( -1, "module '%s' does not exist", module.c_str() );
+            }
+            else
+            {
+                install_module( package_path, module, installed, mtime );
+
+                if( s_opt_d )
+                {
+                    std::vector< std::string > mdeps = deps[ module ];
+
+                    for( std::size_t j = 0, n = mdeps.size(); j < n; ++j )
+                    {
+                        std::string const & m2 = mdeps[ j ];
+
+                        if( std::find( modules.begin(), modules.end(), m2 ) == modules.end() )
+                        {
+                            modules.push_back( m2 );
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    std::set< std::string > installed2;
+
+    {
+        bool need_build = false;
+
+        for( std::vector< std::string >::const_iterator i = modules.begin(); i != modules.end(); ++i )
+        {
+            if( buildable.count( *i ) )
+            {
+                need_build = true;
+            }
+        }
+
+        if( s_opt_d && need_build )
+        {
+            install_boost_build( package_path, installed2 );
+        }
+    }
+
+    if( installed.empty() && installed2.empty() )
+    {
+        msg_printf( 0, "nothing to install, everything is already in place" );
+    }
+
+    if( !s_opt_n )
+    {
+        std::set< std::string > need_build;
+
+        for( std::set< std::string >::const_iterator i = installed.begin(); i != installed.end(); ++i )
+        {
+            if( buildable.count( *i ) )
+            {
+                need_build.insert( *i );
+            }
+        }
+
+        if( !need_build.empty() )
+        {
+            std::string m2;
+
+            for( std::set< std::string >::const_iterator i = need_build.begin(); i != need_build.end(); ++i )
+            {
+                m2 += " ";
+                m2 += *i;
+            }
+
+            msg_printf( 0, "the following libraries need to be built:\n %s", m2.c_str() );
+#if defined( _WIN32 )
+            msg_printf( 0, "(use b2 to build)" );
+#else
+            msg_printf( 0, "(use ./b2 to build)" );
+#endif
+        }
+    }
+
+    if( !s_opt_n && mtime >= fs_mtime( "include/.updated" ) ) // headers out of date
+    {
+        cmd_headers();
+    }
+
+    if( !s_opt_n && mtime >= fs_mtime( "index.html" ) ) // index out of date
+    {
+        cmd_index();
+    }
+}

src/cmd_install.hpp

@@ -0,0 +1,14 @@
+#ifndef CMD_INSTALL_HPP_INCLUDED
+#define CMD_INSTALL_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+void cmd_install( char const * argv[] );
+
+#endif // #ifndef CMD_INSTALL_HPP_INCLUDED

src/cmd_list.cpp

@@ -0,0 +1,137 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "cmd_list.hpp"
+#include "options.hpp"
+#include "dependencies.hpp"
+#include "message.hpp"
+#include "fs.hpp"
+#include <algorithm>
+#include <stdexcept>
+#include <cstdio>
+
+static bool s_opt_a = false;
+static bool s_opt_i = false;
+static bool s_opt_p = false;
+static bool s_opt_b = false;
+
+static void handle_option( std::string const & opt )
+{
+    if( opt == "-a" )
+    {
+        s_opt_a = true;
+    }
+    else if( opt == "-i" )
+    {
+        s_opt_i = true;
+    }
+    else if( opt == "-p" )
+    {
+        s_opt_p = true;
+    }
+    else if( opt == "-b" )
+    {
+        s_opt_b = true;
+    }
+    else if( opt == "-v" )
+    {
+        increase_message_level();
+    }
+    else if( opt == "-q" )
+    {
+        decrease_message_level();
+    }
+    else
+    {
+        throw std::runtime_error( "invalid list option: '" + opt + "'" );
+    }
+}
+
+static std::string module_package( std::string const & module )
+{
+    std::size_t i = module.find( '~' );
+
+    std::string package = module.substr( 0, i );
+
+    return package;
+}
+
+void cmd_list( char const * argv[] )
+{
+    parse_options( argv, handle_option );
+
+    std::string prefix;
+
+    if( *argv )
+    {
+        prefix = *argv++;
+    }
+
+    if( *argv )
+    {
+        throw std::runtime_error( std::string( "unexpected list argument '" ) + *argv + "'" );
+    }
+
+    if( s_opt_a + s_opt_i + s_opt_p > 1 )
+    {
+        throw std::runtime_error( "list options -a, -i and -p are mutually exclusive" );
+    }
+
+    if( s_opt_p && s_opt_b )
+    {
+        throw std::runtime_error( "list options -p and -b are incompatible" );
+    }
+
+    if( !s_opt_a && !s_opt_i && !s_opt_p )
+    {
+        s_opt_a = !s_opt_b;
+        s_opt_i = s_opt_b;
+    }
+
+    std::map< std::string, std::vector< std::string > > deps;
+    std::set< std::string > buildable;
+
+    retrieve_dependencies( deps, buildable );
+
+    if( s_opt_a )
+    {
+        for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i )
+        {
+            std::string module = i->first;
+
+            if( module.substr( 0, prefix.size() ) != prefix ) continue;
+
+            if( !s_opt_b || buildable.count( module ) )
+            {
+                printf( "%s\n", module.c_str() );
+            }
+        }
+    }
+    else
+    {
+        for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i )
+        {
+            std::string module = i->first;
+
+            if( module.substr( 0, prefix.size() ) != prefix ) continue;
+
+            std::string package = module_package( module );
+
+            std::string path( module );
+            std::replace( path.begin(), path.end(), '~', '/' );
+
+            if( fs_exists( "libs/" + path ) && s_opt_p != fs_exists( "libs/" + package + "/.installed" ) )
+            {
+                if( !s_opt_b || buildable.count( module ) )
+                {
+                    printf( "%s\n", module.c_str() );
+                }
+            }
+        }
+    }
+}

src/cmd_list.hpp

@@ -0,0 +1,14 @@
+#ifndef CMD_LIST_HPP_INCLUDED
+#define CMD_LIST_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+void cmd_list( char const * argv[] );
+
+#endif // #ifndef CMD_LIST_HPP_INCLUDED

src/cmd_remove.cpp

@@ -0,0 +1,308 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "cmd_remove.hpp"
+#include "cmd_headers.hpp"
+#include "cmd_index.hpp"
+#include "options.hpp"
+#include "dependencies.hpp"
+#include "message.hpp"
+#include "fs.hpp"
+#include <algorithm>
+#include <stdexcept>
+#include <cstdio>
+#include <cstring>
+#include <errno.h>
+
+static bool s_opt_n = false;
+static bool s_opt_f = false;
+static bool s_opt_d = false;
+static bool s_opt_a = false;
+static bool s_opt_p = false;
+
+static void handle_option( std::string const & opt )
+{
+    if( opt == "-n" )
+    {
+        s_opt_n = true;
+    }
+    else if( opt == "-f" )
+    {
+        s_opt_f = true;
+    }
+    else if( opt == "-d" )
+    {
+        s_opt_d = true;
+    }
+    else if( opt == "-a" )
+    {
+        s_opt_a = true;
+    }
+    else if( opt == "-p" )
+    {
+        s_opt_p = true;
+    }
+    else if( opt == "-v" )
+    {
+        increase_message_level();
+    }
+    else if( opt == "-q" )
+    {
+        decrease_message_level();
+    }
+    else
+    {
+        throw std::runtime_error( "invalid remove option: '" + opt + "'" );
+    }
+}
+
+static std::string module_package( std::string const & module )
+{
+    std::size_t i = module.find( '~' );
+
+    std::string package = module.substr( 0, i );
+
+    return package;
+}
+
+static void get_package_dependents( std::string const & package, std::map< std::string, std::vector< std::string > > const & deps, std::vector< std::string > const & packages, std::set< std::string > & deps2 )
+{
+    deps2.clear();
+
+    if( !fs_exists( "libs/" + package + "/.installed" ) )
+    {
+        // partially installed packages have no dependents
+        return;
+    }
+
+    for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i )
+    {
+        std::string m1 = i->first;
+        std::string p1 = module_package( m1 );
+
+        for( std::vector< std::string >::const_iterator j = i->second.begin(); j != i->second.end(); ++j )
+        {
+            std::string m2 = *j;
+            std::string p2 = module_package( m2 );
+
+            // p1 -> p2
+
+            if( p2 == package && fs_exists( "libs/" + p1 + "/.installed" ) && std::find( packages.begin(), packages.end(), p1 ) == packages.end() )
+            {
+                deps2.insert( p1 );
+            }
+        }
+    }
+}
+
+static void removing( std::string const & path )
+{
+    msg_printf( 2, "removing '%s'", path.c_str() );
+}
+
+static void rmerror( std::string const & path, int err )
+{
+    msg_printf( 1, "'%s': remove error: %s", path.c_str(), std::strerror( err ) );
+}
+
+static void remove_package( std::string const & package, int & removed )
+{
+    std::string path = "libs/" + package;
+
+    std::set< std::string > files;
+
+    if( package == "build" )
+    {
+        path = "tools/" + package;
+
+        files.insert( "b2.exe" );
+        files.insert( "boost-build.jam" );
+        files.insert( "boostcpp.jam" );
+        files.insert( "Jamroot" );
+        files.insert( "libs/Jamfile.v2" );
+    }
+
+    if( !fs_exists( path ) )
+    {
+        msg_printf( 1, "package '%s' has already been removed", package.c_str() );
+        return;
+    }
+
+    if( s_opt_n )
+    {
+        msg_printf( 0, "would have removed package '%s'", package.c_str() );
+        return;
+    }
+
+    msg_printf( 0, "removing package '%s'", package.c_str() );
+
+    std::string marker = path + "/.installed";
+
+    std::remove( marker.c_str() );
+
+    fs_remove_all( path, removing, rmerror );
+
+    ++removed;
+
+    for( std::set< std::string >::const_iterator i = files.begin(); i != files.end(); ++i )
+    {
+        if( fs_exists( *i ) )
+        {
+            removing( *i );
+
+            int r = std::remove( i->c_str() );
+
+            if( r != 0 )
+            {
+                rmerror( *i, errno );
+            }
+        }
+    }
+}
+
+void cmd_remove( char const * argv[] )
+{
+    parse_options( argv, handle_option );
+
+    if( s_opt_a && s_opt_p )
+    {
+        throw std::runtime_error( "remove options -a and -p are mutually exclusive" );
+    }
+
+    if( ( s_opt_a || s_opt_p ) && *argv != 0 )
+    {
+        throw std::runtime_error( "remove options -a and -p are incompatible with a package list" );
+    }
+
+    if( s_opt_d && !s_opt_f && !s_opt_n )
+    {
+        throw std::runtime_error( "remove option -d requires -f" );
+    }
+
+    std::map< std::string, std::vector< std::string > > deps;
+    std::set< std::string > buildable;
+
+    retrieve_dependencies( deps, buildable );
+
+    std::vector< std::string > packages;
+
+    if( s_opt_a )
+    {
+        if( !s_opt_f && !s_opt_n )
+        {
+            throw std::runtime_error( "remove option -a requires -f" );
+        }
+
+        for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i )
+        {
+            std::string module = i->first;
+            std::string package = module_package( module );
+
+            if( packages.empty() || packages.back() != package )
+            {
+                packages.push_back( package );
+            }
+        }
+
+        packages.push_back( "build" );
+    }
+    else if( s_opt_p )
+    {
+        for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i )
+        {
+            std::string module = i->first;
+            std::string package = module_package( module );
+
+            if( fs_exists( "libs/" + package ) && !fs_exists( "libs/" + package + "/.installed" ) )
+            {
+                if( packages.empty() || packages.back() != package )
+                {
+                    packages.push_back( package );
+                }
+            }
+
+            if( fs_exists( "tools/build" ) && !fs_exists( "tools/build/.installed" ) )
+            {
+                packages.push_back( "build" );
+            }
+        }
+    }
+    else
+    {
+        while( *argv )
+        {
+            packages.push_back( *argv );
+            ++argv;
+        }
+    }
+
+    if( !s_opt_f && !( s_opt_n && s_opt_d ) )
+    {
+        for( std::vector< std::string >::const_iterator i = packages.begin(); i != packages.end(); ++i )
+        {
+            std::string package = *i;
+
+            std::set< std::string > deps2;
+
+            get_package_dependents( package, deps, packages, deps2 );
+
+            if( !s_opt_f && !deps2.empty() )
+            {
+                std::string list;
+
+                for( std::set< std::string >::const_iterator i = deps2.begin(); i != deps2.end(); ++i )
+                {
+                    list += " ";
+                    list += *i;
+                }
+
+                msg_printf( -2, "package '%s' cannot be removed due to dependents:\n %s", package.c_str(), list.c_str() );
+
+                return;
+            }
+        }
+    }
+
+    int removed = 0;
+
+    {
+        std::size_t i = 0;
+
+        while( i < packages.size() )
+        {
+            std::string package = packages[ i++ ];
+
+            std::set< std::string > deps2;
+
+            get_package_dependents( package, deps, packages, deps2 );
+
+            remove_package( package, removed );
+
+            if( s_opt_d )
+            {
+                for( std::set< std::string >::const_iterator i = deps2.begin(); i != deps2.end(); ++i )
+                {
+                    if( std::find( packages.begin(), packages.end(), *i ) == packages.end() )
+                    {
+                        packages.push_back( *i );
+                    }
+                }
+            }
+        }
+    }
+
+    if( removed )
+    {
+        cmd_headers();
+    }
+
+    if( removed )
+    {
+        cmd_index();
+    }
+}

src/cmd_remove.hpp

@@ -0,0 +1,14 @@
+#ifndef CMD_REMOVE_HPP_INCLUDED
+#define CMD_REMOVE_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+void cmd_remove( char const * argv[] );
+
+#endif // #ifndef CMD_REMOVE_HPP_INCLUDED

src/config.cpp

@@ -0,0 +1,62 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "config.hpp"
+#include "error.hpp"
+#include "string.hpp"
+#include <map>
+#include <fstream>
+#include <cctype>
+#include <cstddef>
+#include <errno.h>
+
+static std::map< std::string, std::string > s_config;
+
+void config_read_file( std::string const & fn )
+{
+    std::ifstream is( fn.c_str() );
+
+    if( !is )
+    {
+        throw_errno_error( fn, "open error", errno );
+    }
+
+    std::string line;
+
+    while( std::getline( is, line ) )
+    {
+        if( line.empty() ) continue;
+
+        if( !std::isalpha( static_cast< unsigned char >( line[0] ) ) ) continue;
+
+        remove_trailing( line, '\r' );
+
+        std::size_t i = line.find( '=' );
+
+        if( i == 0 || i == std::string::npos ) continue;
+
+        std::string key = line.substr( 0, i );
+        std::string value = line.substr( i + 1 );
+
+        s_config[ key ] = value;
+    }
+}
+
+std::string config_get_option( std::string const & name )
+{
+    std::map< std::string, std::string >::const_iterator i = s_config.find( name );
+
+    if( i == s_config.end() )
+    {
+        return std::string();
+    }
+    else
+    {
+        return i->second;
+    }
+}

src/config.hpp

@@ -0,0 +1,17 @@
+#ifndef CONFIG_HPP_INCLUDED
+#define CONFIG_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include <string>
+
+void config_read_file( std::string const & fn );
+std::string config_get_option( std::string const & name );
+
+#endif // #ifndef CONFIG_HPP_INCLUDED

src/dependencies.cpp

@@ -0,0 +1,104 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "dependencies.hpp"
+#include "package_path.hpp"
+#include "lzma_reader.hpp"
+#include "http_reader.hpp"
+#include "error.hpp"
+#include "string.hpp"
+#include <stdexcept>
+#include <sstream>
+
+static void parse_module_description( std::string const & name, std::string const & line, std::map< std::string, std::vector< std::string > > & deps )
+{
+    std::istringstream is( line );
+
+    std::string module, arrow;
+
+    if( is >> module >> arrow && arrow == "->" )
+    {
+        deps[ module ];
+
+        std::string dep;
+
+        while( is >> dep )
+        {
+            deps[ module ].push_back( dep );
+        }
+    }
+    else
+    {
+        throw_error( name, "invalid line: '" + line + "'" );
+    }
+}
+
+static std::string read_text_file( std::string const & url )
+{
+    http_reader r1( url );
+    lzma_reader r2( &r1 );
+
+    std::string data;
+
+    for( ;; )
+    {
+        int const N = 4096;
+
+        char buffer[ N ];
+
+        std::size_t r = r2.read( buffer, N );
+
+        data.append( buffer, r );
+
+        if( r < N ) break;
+    }
+
+    return data;
+}
+
+static void retrieve_dependencies( std::string const & package_path, std::map< std::string, std::vector< std::string > > & deps )
+{
+    std::string url = package_path + "dependencies.txt.lzma";
+
+    std::string data = read_text_file( url );
+
+    std::istringstream is( data );
+
+    std::string line;
+
+    while( std::getline( is, line ) )
+    {
+        remove_trailing( line, '\r' );
+        parse_module_description( url, line, deps );
+    }
+}
+
+static void retrieve_buildable( std::string const & package_path, std::set< std::string > & buildable )
+{
+    std::string data = read_text_file( package_path + "buildable.txt.lzma" );
+
+    std::istringstream is( data );
+
+    std::string module;
+
+    while( is >> module )
+    {
+        buildable.insert( module );
+    }
+}
+
+void retrieve_dependencies( std::map< std::string, std::vector< std::string > > & deps, std::set< std::string > & buildable )
+{
+    deps.clear();
+    buildable.clear();
+
+    std::string package_path = get_package_path();
+
+    retrieve_dependencies( package_path, deps );
+    retrieve_buildable( package_path, buildable );
+}

src/dependencies.hpp

@@ -0,0 +1,19 @@
+#ifndef DEPENDENCIES_HPP_INCLUDED
+#define DEPENDENCIES_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include <map>
+#include <string>
+#include <vector>
+#include <set>
+
+void retrieve_dependencies( std::map< std::string, std::vector< std::string > > & deps, std::set< std::string > & buildable );
+
+#endif // #ifndef DEPENDENCIES_HPP_INCLUDED

src/error.cpp

@@ -0,0 +1,77 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "error.hpp"
+#include <cstring>
+
+error::error( std::string const & name, std::string const & reason ): name_( name ), reason_( reason ), what_( "'" + name_ + "': " + reason )
+{
+}
+
+error::~error() throw()
+{
+}
+
+std::string error::name() const
+{
+    return name_;
+}
+
+std::string error::reason() const
+{
+    return reason_;
+}
+
+char const * error::what() const throw()
+{
+    return what_.c_str();
+}
+
+void throw_error( std::string const & name, std::string const & reason )
+{
+    throw error( name, reason );
+}
+
+void throw_errno_error( std::string const & name, std::string const & reason, int err )
+{
+    if( err == 0 )
+    {
+        throw_error( name, reason );
+    }
+    else
+    {
+        throw_error( name, reason + ": " + std::strerror( err ) );
+    }
+}
+
+#if defined( _WIN32 )
+
+#include <windows.h>
+
+void throw_socket_error( std::string const & name, std::string const & reason, int err )
+{
+    char buffer[ 1024 ];
+
+    if( err == 0 || FormatMessageA( FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, 0, err, 0, buffer, sizeof( buffer ), 0 ) == 0 )
+    {
+        throw_error( name, reason );
+    }
+    else
+    {
+        throw_error( name, reason + ": " + buffer );
+    }
+}
+
+#else
+
+void throw_socket_error( std::string const & name, std::string const & reason, int err )
+{
+    throw_errno_error( name, reason, err );
+}
+
+#endif

src/error.hpp

@@ -0,0 +1,39 @@
+#ifndef ERROR_HPP_INCLUDED
+#define ERROR_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include <string>
+#include <exception>
+
+class error: public std::exception
+{
+private:
+
+    std::string name_;
+    std::string reason_;
+
+    std::string what_;
+
+public:
+
+    error( std::string const & name, std::string const & reason );
+    ~error() throw();
+
+    std::string name() const;
+    std::string reason() const;
+
+    char const * what() const throw();
+};
+
+void throw_error( std::string const & name, std::string const & reason );
+void throw_errno_error( std::string const & name, std::string const & reason, int err );
+void throw_socket_error( std::string const & name, std::string const & reason, int err );
+
+#endif // #ifndef ERROR_HPP_INCLUDED

src/file_reader.cpp

@@ -0,0 +1,47 @@
+//
+// Copyright 2014 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "file_reader.hpp"
+#include "error.hpp"
+#include <errno.h>
+
+file_reader::file_reader( std::FILE * f, std::string const & name ): f_( f ), name_( name )
+{
+}
+
+file_reader::file_reader( std::string const & fn ): name_( fn )
+{
+    f_ = std::fopen( fn.c_str(), "rb" );
+
+    if( f_ == 0 )
+    {
+        throw_errno_error( fn, "open error", errno );
+    }
+}
+
+file_reader::~file_reader()
+{
+    std::fclose( f_ );
+}
+
+std::string file_reader::name() const
+{
+    return name_;
+}
+
+std::size_t file_reader::read( void * p, std::size_t n )
+{
+    std::size_t r = std::fread( p, 1, n, f_ );
+
+    if( r == 0 && std::ferror( f_ ) )
+    {
+        throw_errno_error( name(), "read error", errno );
+    }
+
+    return r;
+}

src/file_reader.hpp

@@ -0,0 +1,38 @@
+#ifndef FILE_READER_HPP_INCLUDED
+#define FILE_READER_HPP_INCLUDED
+
+//
+// Copyright 2014 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "basic_reader.hpp"
+#include <cstdio>
+
+class file_reader: public basic_reader
+{
+private:
+
+    std::FILE * f_;
+    std::string name_;
+
+private:
+
+    file_reader( file_reader const & );
+    file_reader& operator=( file_reader const & );
+
+public:
+
+    explicit file_reader( std::FILE * f, std::string const & n );
+    explicit file_reader( std::string const & fn );
+
+    ~file_reader();
+
+    virtual std::string name() const;
+    virtual std::size_t read( void * p, std::size_t n );
+};
+
+#endif // #ifndef FILE_READER_HPP_INCLUDED

src/fs.cpp

@@ -0,0 +1,792 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "fs.hpp"
+#include <cstdio>
+#include <cassert>
+#include <errno.h>
+
+std::time_t fs_mtime( std::string const & path )
+{
+    int mode;
+    std::time_t mtime, ctime, atime;
+
+    if( fs_stat( path, mode, mtime, ctime, atime ) == 0 )
+    {
+        return mtime;
+    }
+    else
+    {
+        return -1;
+    }
+}
+
+#if defined( _WIN32 )
+
+#define PATH_SEPARATOR '\\'
+
+static bool is_absolute( std::string const & path )
+{
+    return path[ 0 ] == '\\' || path[ 0 ] == '/' || path.size() >= 2 && path[ 1 ] == ':';
+}
+
+static std::size_t find_next_separator( std::string const & path, std::size_t offset )
+{
+    return path.find_first_of( "/\\", offset );
+}
+
+#else
+
+#define PATH_SEPARATOR '/'
+
+static bool is_absolute( std::string const & path )
+{
+    return path[ 0 ] == '/';
+}
+
+static std::size_t find_next_separator( std::string const & path, std::size_t offset )
+{
+    return path.find( '/', offset );
+}
+
+#endif // _WIN32
+
+static int path_element_depth( std::string const & element )
+{
+    if( element == "" || element == "." )
+    {
+        return 0;
+    }
+    else if( element == ".." )
+    {
+        return -1;
+    }
+    else
+    {
+        return +1;
+    }
+}
+
+static int path_depth( std::string const & path )
+{
+    int l = 0;
+
+    std::size_t i = 0;
+
+    for( ;; )
+    {
+        std::size_t j = find_next_separator( path, i );
+
+        if( j == std::string::npos )
+        {
+            l += path_element_depth( path.substr( i ) );
+            break;
+        }
+
+        l += path_element_depth( path.substr( i, j - i ) );
+
+        i = j + 1;
+    }
+
+    return l;
+}
+
+static bool make_relative( std::string const & link, std::string & target )
+{
+    if( target.empty() )
+    {
+        return false;
+    }
+
+    if( is_absolute( target ) )
+    {
+        // absolute pathname
+        return true;
+    }
+
+    if( link.empty() )
+    {
+        return false;
+    }
+
+    if( is_absolute( link ) )
+    {
+        // absolute link, relative target, not supported
+        return true;
+    }
+
+    int l = path_depth( link );
+
+    if( l <= 0 )
+    {
+        return false;
+    }
+
+    while( --l > 0 )
+    {
+        target = PATH_SEPARATOR + target;
+        target = ".."  + target;
+    }
+
+    return true;
+}
+
+#if defined( _WIN32 )
+
+#define _WIN32_WINNT 0x501
+
+#include <algorithm>
+#include <io.h>
+#include <direct.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/utime.h>
+#include <windows.h>
+
+int fs_creat( std::string const & path, int mode )
+{
+    return _open( path.c_str(), _O_CREAT | _O_TRUNC | _O_WRONLY | _O_BINARY,  mode & 0600 );
+}
+
+int fs_write( int fd, void const * buffer, unsigned n )
+{
+    return _write( fd, buffer, n );
+}
+
+int fs_close( int fd )
+{
+    return _close( fd );
+}
+
+int fs_mkdir( std::string const & path, int /*mode*/ )
+{
+    return _mkdir( path.c_str() );
+}
+
+bool fs_exists( std::string const & path )
+{
+    return _access( path.c_str(), 0 ) == 0;
+}
+
+int fs_stat( std::string const & path, int & mode, std::time_t & mtime, std::time_t & ctime, std::time_t & atime )
+{
+    struct _stat st;
+
+    int r = _stat( path.c_str(), &st );
+
+    if( r == 0 )
+    {
+        mode = st.st_mode;
+
+        mtime = st.st_mtime;
+        ctime = st.st_ctime;
+        atime = st.st_atime;
+    }
+
+    return r;
+}
+
+int fs_utime( std::string const & path, std::time_t mtime, std::time_t atime )
+{
+    _utimbuf ut;
+
+    ut.actime = atime;
+    ut.modtime = mtime;
+
+    return _utime( path.c_str(), &ut );
+}
+
+int fs_rmdir( std::string const & path )
+{
+    return _rmdir( path.c_str() );
+}
+
+void fs_remove_all( std::string const & path, void (*removing)( std::string const & ), void (*error)( std::string const &, int ) )
+{
+    removing( path );
+
+    DWORD dw = GetFileAttributesA( path.c_str() );
+
+    if( dw == INVALID_FILE_ATTRIBUTES )
+    {
+        dw = 0;
+    }
+
+    if( dw & FILE_ATTRIBUTE_DIRECTORY )
+    {
+        if( dw & FILE_ATTRIBUTE_REPARSE_POINT )
+        {
+            // junction or directory symlink
+
+            int r = _rmdir( path.c_str() );
+
+            if( r != 0 )
+            {
+                error( path, errno );
+            }
+
+            return;
+        }
+    }
+    else
+    {
+        // file or file symlink
+
+        int r = std::remove( path.c_str() );
+
+        if( r != 0 )
+        {
+            error( path, errno );
+        }
+
+        return;
+    }
+
+    // ordinary directory, descend
+
+    assert( dw & FILE_ATTRIBUTE_DIRECTORY );
+    assert( ( dw & FILE_ATTRIBUTE_REPARSE_POINT ) == 0 );
+
+    _finddata_t fd;
+
+    intptr_t r = _findfirst( ( path + "/*" ).c_str(), &fd );
+
+    if( r >= 0 )
+    {
+        do
+        {
+            std::string name = fd.name;
+
+            if( name != "." && name != ".." )
+            {
+                fs_remove_all( path + '/' + name, removing, error );
+            }
+        }
+        while( _findnext( r, &fd ) == 0 );
+
+        _findclose( r );
+
+        {
+            int r2 = _rmdir( path.c_str() );
+
+            if( r2 != 0 )
+            {
+                error( path, errno );
+            }
+        }
+    }
+    else
+    {
+        error( path + "/*", errno );
+    }
+}
+
+int fs_readdir( std::string const & path, std::vector< std::string > & entries )
+{
+    entries.resize( 0 );
+
+    _finddata_t fd;
+
+    intptr_t r = _findfirst( ( path + "/*" ).c_str(), &fd );
+
+    if( r >= 0 )
+    {
+        do
+        {
+            entries.push_back( fd.name );
+        }
+        while( _findnext( r, &fd ) == 0 );
+
+        _findclose( r );
+
+        return 0;
+    }
+    else
+    {
+        return -1;
+    }
+}
+
+bool fs_is_dir( std::string const & path )
+{
+    DWORD dw = GetFileAttributesA( path.c_str() );
+
+    if( dw == INVALID_FILE_ATTRIBUTES )
+    {
+        dw = 0;
+    }
+
+    return dw & FILE_ATTRIBUTE_DIRECTORY? true: false;
+}
+
+static int errno_from_last_error( DWORD r )
+{
+    switch( r )
+    {
+    case ERROR_ACCESS_DENIED:           return EACCES;
+    case ERROR_ALREADY_EXISTS:          return EEXIST;
+    case ERROR_BAD_NET_NAME:            return ENOENT;
+    case ERROR_BAD_NETPATH:             return ENOENT;
+    case ERROR_BAD_PATHNAME:            return ENOENT;
+    case ERROR_BAD_UNIT:                return ENODEV;
+    case ERROR_BROKEN_PIPE:             return EPIPE;
+    case ERROR_BUFFER_OVERFLOW:         return ENAMETOOLONG;
+    case ERROR_BUSY:                    return EBUSY;
+    case ERROR_BUSY_DRIVE:              return EBUSY;
+    case ERROR_CANNOT_MAKE:             return EACCES;
+    case ERROR_CANTOPEN:                return EIO;
+    case ERROR_CANTREAD:                return EIO;
+    case ERROR_CANTWRITE:               return EIO;
+    case ERROR_CHILD_NOT_COMPLETE:      return ECHILD;
+    case ERROR_CURRENT_DIRECTORY:       return EBUSY;
+    case ERROR_DEV_NOT_EXIST:           return ENODEV;
+    case ERROR_DEVICE_IN_USE:           return EBUSY;
+    case ERROR_DIR_NOT_EMPTY:           return ENOTEMPTY;
+    case ERROR_DIRECT_ACCESS_HANDLE:    return EPERM;
+    case ERROR_DIRECTORY:               return EINVAL;
+    case ERROR_DISK_FULL:               return ENOSPC;
+    case ERROR_DRIVE_LOCKED:            return EACCES;
+    case ERROR_FAIL_I24:                return EIO;
+    case ERROR_FILE_EXISTS:             return EEXIST;
+    case ERROR_FILE_NOT_FOUND:          return ENOENT;
+    case ERROR_FILENAME_EXCED_RANGE:    return ENAMETOOLONG;
+    case ERROR_HANDLE_DISK_FULL:        return ENOSPC;
+    case ERROR_INVALID_ACCESS:          return EACCES;
+    case ERROR_INVALID_DRIVE:           return ENODEV;
+    case ERROR_INVALID_FUNCTION:        return ENOSYS;
+    case ERROR_INVALID_HANDLE:          return EBADF;
+    case ERROR_INVALID_NAME:            return EINVAL;
+    case ERROR_INVALID_TARGET_HANDLE:   return EBADF;
+    case ERROR_LOCK_FAILED:             return EACCES;
+    case ERROR_LOCK_VIOLATION:          return EACCES;
+    case ERROR_LOCKED:                  return EACCES;
+    case ERROR_MAX_THRDS_REACHED:       return EAGAIN;
+    case ERROR_NEGATIVE_SEEK:           return EINVAL;
+    case ERROR_NETWORK_ACCESS_DENIED:   return EACCES;
+    case ERROR_NOACCESS:                return EFAULT;
+    case ERROR_NO_MORE_FILES:           return ENOENT;
+    case ERROR_NO_PROC_SLOTS:           return EAGAIN;
+    case ERROR_NOT_ENOUGH_MEMORY:       return ENOMEM;
+    case ERROR_NOT_ENOUGH_QUOTA:        return ENOMEM;
+    case ERROR_NOT_LOCKED:              return EACCES;
+    case ERROR_NOT_READY:               return EAGAIN;
+    case ERROR_NOT_SAME_DEVICE:         return EXDEV;
+    case ERROR_OPEN_FAILED:             return EIO;
+    case ERROR_OPEN_FILES:              return EBUSY;
+    case ERROR_OPERATION_ABORTED:       return EINTR; // ECANCELED;
+    case ERROR_OUTOFMEMORY:             return ENOMEM;
+    case ERROR_PATH_NOT_FOUND:          return ENOENT;
+    case ERROR_READ_FAULT:              return EIO;
+    case ERROR_RETRY:                   return EAGAIN;
+    case ERROR_SEEK:                    return EIO;
+    case ERROR_SEEK_ON_DEVICE:          return ESPIPE;
+    case ERROR_SHARING_VIOLATION:       return EACCES;
+    case ERROR_TOO_MANY_OPEN_FILES:     return EMFILE;
+    case ERROR_WAIT_NO_CHILDREN:        return ECHILD;
+    case ERROR_WRITE_FAULT:             return EIO;
+    case ERROR_WRITE_PROTECT:           return EROFS;
+
+    default:                            return EINVAL;
+    }
+}
+
+static void set_errno_from_last_error( DWORD r )
+{
+    errno = errno_from_last_error( r );
+}
+
+typedef BOOLEAN (WINAPI *PtrCreateSymbolicLinkA)(
+    /*__in*/ LPCSTR lpSymlinkFileName,
+    /*__in*/ LPCSTR lpTargetFileName,
+    /*__in*/ DWORD dwFlags
+   );
+
+static PtrCreateSymbolicLinkA CreateSymbolicLinkA = PtrCreateSymbolicLinkA( GetProcAddress( GetModuleHandleA( "kernel32.dll" ), "CreateSymbolicLinkA" ) );
+
+#ifndef SYMBOLIC_LINK_FLAG_DIRECTORY
+#  define SYMBOLIC_LINK_FLAG_DIRECTORY 1
+#endif
+
+int fs_link_file( std::string const & link, std::string const & target )
+{
+    {
+        // CreateSymbolicLink happily accepts '/' separated targets, but the symlink doesn't work
+
+        std::string t2( target );
+        std::replace( t2.begin(), t2.end(), '/', '\\' );
+
+        if( !make_relative( link, t2 ) )
+        {
+            errno = EINVAL;
+            return -1;
+        }
+
+        if( CreateSymbolicLinkA && CreateSymbolicLinkA( link.c_str(), t2.c_str(), 0 ) )
+        {
+            return 0;
+        }
+    }
+
+    if( CreateHardLinkA( link.c_str(), target.c_str(), 0 ) )
+    {
+        return 0;
+    }
+
+    set_errno_from_last_error( GetLastError() );
+    return -1;
+}
+
+// definitions copied from Boost.Filesystem
+
+#if !defined(REPARSE_DATA_BUFFER_HEADER_SIZE)  // mingw winnt.h does provide the defs
+
+#define SYMLINK_FLAG_RELATIVE 1
+
+typedef struct _REPARSE_DATA_BUFFER {
+  ULONG  ReparseTag;
+  USHORT  ReparseDataLength;
+  USHORT  Reserved;
+  union {
+    struct {
+      USHORT  SubstituteNameOffset;
+      USHORT  SubstituteNameLength;
+      USHORT  PrintNameOffset;
+      USHORT  PrintNameLength;
+      ULONG  Flags;
+      WCHAR  PathBuffer[1];
+     } SymbolicLinkReparseBuffer;
+    struct {
+      USHORT  SubstituteNameOffset;
+      USHORT  SubstituteNameLength;
+      USHORT  PrintNameOffset;
+      USHORT  PrintNameLength;
+      WCHAR  PathBuffer[1];
+      } MountPointReparseBuffer;
+    struct {
+      UCHAR  DataBuffer[1];
+    } GenericReparseBuffer;
+  };
+} REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER;
+
+#define REPARSE_DATA_BUFFER_HEADER_SIZE \
+  FIELD_OFFSET(REPARSE_DATA_BUFFER, GenericReparseBuffer)
+
+#endif
+
+#ifndef MAXIMUM_REPARSE_DATA_BUFFER_SIZE
+#  define MAXIMUM_REPARSE_DATA_BUFFER_SIZE  ( 16 * 1024 )
+#endif
+
+#ifndef FSCTL_GET_REPARSE_POINT
+#  define FSCTL_GET_REPARSE_POINT 0x900a8
+#endif
+
+#ifndef FSCTL_SET_REPARSE_POINT
+#  define FSCTL_SET_REPARSE_POINT 0x900a4
+#endif
+
+#ifndef IO_REPARSE_TAG_MOUNT_POINT
+#  define IO_REPARSE_TAG_MOUNT_POINT 0xA0000003L
+#endif
+
+static int create_junction( std::string const & link, std::string const & target )
+{
+    std::wstring target2;
+
+    {
+        wchar_t buffer[ 1024 ] = { 0 };
+
+        int r = MultiByteToWideChar( CP_ACP, 0, target.c_str(), -1, buffer, sizeof( buffer ) / sizeof( buffer[0] ) );
+
+        if( r == 0 )
+        {
+            errno = ENAMETOOLONG;
+            return -1;
+        }
+
+        target2 = buffer;
+
+        std::replace( target2.begin(), target2.end(), L'/', L'\\' );
+
+        if( target2.empty() || *target2.rbegin() != L'\\' )
+        {
+            target2 += L'\\';
+        }
+
+        DWORD dw = GetFullPathNameW( target2.c_str(), sizeof( buffer ) / sizeof( buffer[ 0 ] ), buffer, 0 );
+
+        if( dw >= sizeof( buffer ) / sizeof( buffer[ 0 ] ) )
+        {
+            errno = ENAMETOOLONG;
+            return -1;
+        }
+
+        if( dw == 0 )
+        {
+            set_errno_from_last_error( GetLastError() );
+            return -1;
+        }
+
+        target2 = buffer;
+    }
+
+    {
+        int r = _mkdir( link.c_str() );
+
+        if( r != 0 )
+        {
+            return r;
+        }
+    }
+
+    HANDLE h = ::CreateFileA( link.c_str(), GENERIC_WRITE, 0, NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, NULL );
+
+    if( h == INVALID_HANDLE_VALUE )
+    {
+        DWORD r2 = GetLastError();
+
+        _rmdir( link.c_str() );
+
+        set_errno_from_last_error( r2 );
+        return -1;
+    }
+
+    std::wstring print_name( target2 );
+    size_t print_name_len = print_name.size() * sizeof( wchar_t );
+
+    std::wstring substitute_name = L"\\??\\" + print_name;
+    size_t substitute_name_len = substitute_name.size() * sizeof( wchar_t );
+
+    unsigned char buffer[ MAXIMUM_REPARSE_DATA_BUFFER_SIZE ] = { 0 };
+
+    REPARSE_DATA_BUFFER * pb = reinterpret_cast< REPARSE_DATA_BUFFER * >( buffer );
+
+    pb->ReparseTag = IO_REPARSE_TAG_MOUNT_POINT;
+    pb->Reserved = 0;
+
+    wcscpy( pb->MountPointReparseBuffer.PathBuffer, substitute_name.c_str() );
+    wcscpy( pb->MountPointReparseBuffer.PathBuffer + substitute_name.size() + 1, print_name.c_str() );
+
+    pb->MountPointReparseBuffer.SubstituteNameOffset = 0;
+    pb->MountPointReparseBuffer.SubstituteNameLength = substitute_name_len;
+
+    pb->MountPointReparseBuffer.PrintNameOffset = substitute_name_len + sizeof( wchar_t );
+    pb->MountPointReparseBuffer.PrintNameLength = print_name_len;
+
+    pb->ReparseDataLength = FIELD_OFFSET( REPARSE_DATA_BUFFER, MountPointReparseBuffer.PathBuffer ) - FIELD_OFFSET( REPARSE_DATA_BUFFER, MountPointReparseBuffer ) + substitute_name_len + sizeof( wchar_t ) + print_name_len + sizeof( wchar_t );
+
+    {
+        DWORD dw = 0;
+        BOOL r = ::DeviceIoControl( h, FSCTL_SET_REPARSE_POINT, buffer, pb->ReparseDataLength + REPARSE_DATA_BUFFER_HEADER_SIZE, 0, 0, &dw, 0 );
+
+        DWORD r2 = GetLastError();
+
+        CloseHandle( h );
+
+        if( !r )
+        {
+            _rmdir( link.c_str() );
+
+            set_errno_from_last_error( r2 );
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+int fs_link_dir( std::string const & link, std::string const & target )
+{
+    {
+        // CreateSymbolicLink happily accepts '/' separated targets, but the symlink doesn't work
+
+        std::string t2( target );
+        std::replace( t2.begin(), t2.end(), '/', '\\' );
+
+        if( !make_relative( link, t2 ) )
+        {
+            errno = EINVAL;
+            return -1;
+        }
+
+        if( CreateSymbolicLinkA && CreateSymbolicLinkA( link.c_str(), t2.c_str(), SYMBOLIC_LINK_FLAG_DIRECTORY ) )
+        {
+            return 0;
+        }
+    }
+
+    return create_junction( link, target );
+}
+
+#else
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <utime.h>
+#include <dirent.h>
+
+int fs_creat( std::string const & path, int mode )
+{
+    return creat( path.c_str(), mode );
+}
+
+int fs_write( int fd, void const * buffer, unsigned n )
+{
+    return write( fd, buffer, n );
+}
+
+int fs_close( int fd )
+{
+    return close( fd );
+}
+
+int fs_mkdir( std::string const & path, int mode )
+{
+    return mkdir( path.c_str(), mode );
+}
+
+bool fs_exists( std::string const & path )
+{
+    struct stat st;
+    int r = stat( path.c_str(), &st );
+
+    return r == 0;
+}
+
+int fs_stat( std::string const & path, int & mode, std::time_t & mtime, std::time_t & ctime, std::time_t & atime )
+{
+    struct stat st;
+    int r = stat( path.c_str(), &st );
+
+    if( r == 0 )
+    {
+        mode = st.st_mode;
+
+        mtime = st.st_mtime;
+        ctime = st.st_ctime;
+        atime = st.st_atime;
+    }
+
+    return r;
+}
+
+int fs_utime( std::string const & path, std::time_t mtime, std::time_t atime )
+{
+    utimbuf ut;
+
+    ut.actime = atime;
+    ut.modtime = mtime;
+
+    return utime( path.c_str(), &ut );
+}
+
+int fs_rmdir( std::string const & path )
+{
+    return rmdir( path.c_str() );
+}
+
+void fs_remove_all( std::string const & path, void (*removing)( std::string const & ), void (*error)( std::string const &, int ) )
+{
+    removing( path );
+
+    if( !fs_is_dir( path ) )
+    {
+        // file or symlink
+
+        int r = std::remove( path.c_str() );
+
+        if( r != 0 )
+        {
+            error( path, errno );
+        }
+
+        return;
+    }
+
+    // ordinary directory, descend
+
+    DIR * pd = opendir( path.c_str() );
+
+    if( pd == 0 )
+    {
+        error( path + "/*", errno );
+        return;
+    }
+
+    while( dirent * pe = readdir( pd ) )
+    {
+        std::string name = pe->d_name;
+
+        if( name != "." && name != ".." )
+        {
+            fs_remove_all( path + '/' + name, removing, error );
+        }
+    }
+
+    closedir( pd );
+
+    int r = rmdir( path.c_str() );
+
+    if( r != 0 )
+    {
+        error( path, errno );
+    }
+}
+
+int fs_readdir( std::string const & path, std::vector< std::string > & entries )
+{
+    entries.resize( 0 );
+
+    DIR * pd = opendir( path.c_str() );
+    if( pd == 0 ) return -1;
+
+    while( dirent * pe = readdir( pd ) )
+    {
+        entries.push_back( pe->d_name );
+    }
+
+    closedir( pd );
+    return 0;
+}
+
+bool fs_is_dir( std::string const & path )
+{
+    struct stat st;
+    int r = stat( path.c_str(), &st );
+
+    if( r == 0 )
+    {
+        return S_ISDIR( st.st_mode );
+    }
+    else
+    {
+        return false;
+    }
+}
+
+int fs_link_file( std::string const & link, std::string const & target )
+{
+    std::string t2( target );
+
+    if( make_relative( link, t2 ) == 0 && symlink( t2.c_str(), link.c_str() ) == 0 )
+    {
+        return 0;
+    }
+    else
+    {
+        return -1;
+    }
+}
+
+int fs_link_dir( std::string const & link, std::string const & target )
+{
+    return fs_link_file( link, target );
+}
+
+#endif // defined( _WIN32 )

src/fs.hpp

@@ -0,0 +1,43 @@
+#ifndef FS_HPP_INCLUDED
+#define FS_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include <string>
+#include <vector>
+#include <ctime>
+
+int fs_creat( std::string const & path, int mode );
+int fs_write( int fd, void const * buffer, unsigned n );
+int fs_close( int fd );
+
+int fs_mkdir( std::string const & path, int mode );
+
+bool fs_exists( std::string const & path );
+
+int fs_stat( std::string const & path, int & mode, std::time_t & mtime, std::time_t & ctime, std::time_t & atime );
+
+std::time_t fs_mtime( std::string const & path );
+
+int fs_utime( std::string const & path, std::time_t mtime, std::time_t atime );
+
+int fs_rmdir( std::string const & path );
+
+void fs_remove_all( std::string const & path, void (*removing)( std::string const & ), void (*error)( std::string const &, int ) );
+
+int fs_readdir( std::string const & path, std::vector< std::string > & entries );
+
+bool fs_is_dir( std::string const & path );
+
+// these two functions treat 'target' as a path name, not as a string, as per POSIX symlink
+
+int fs_link_file( std::string const & link, std::string const & target ); // symlink, if fails on Windows, hard link
+int fs_link_dir( std::string const & link, std::string const & target ); // symlink, if fails on Windows, junction
+
+#endif // #ifndef FS_HPP_INCLUDED

src/http_reader.cpp

@@ -0,0 +1,126 @@
+//
+// Copyright 2014 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "http_reader.hpp"
+#include "error.hpp"
+#include <sstream>
+#include <cstdlib>
+
+http_url_parser::http_url_parser( std::string const & url )
+{
+    if( url.substr( 0, 7 ) != "http://" )
+    {
+        throw_error( url, "not an HTTP URL" );
+    }
+
+    std::size_t i = url.find( ':', 7 );
+    std::size_t j = url.find( '/', 7 );
+
+    if( j != std::string::npos )
+    {
+        request_ = url.substr( j );
+    }
+    else
+    {
+        request_ = "/";
+    }
+
+    if( i >= j )
+    {
+        // no port
+
+        host_ = url.substr( 7, j - 7 );
+        port_ = 80;
+    }
+    else
+    {
+        host_ = url.substr( 7, i - 7 );
+        port_ = std::atoi( url.substr( i + 1, j - i - 1 ).c_str() );
+    }
+}
+
+std::string http_url_parser::host() const
+{
+    return host_;
+}
+
+int http_url_parser::port() const
+{
+    return port_;
+}
+
+std::string http_url_parser::request() const
+{
+    return request_;
+}
+
+http_reader::http_reader( std::string const & url ): http_url_parser( url ), tcp_reader( this->host(), this->port() ), name_( url )
+{
+    std::string rq = "GET " + this->request() + " HTTP/1.0\r\nHost: " + this->host() + "\r\nConnection: close\r\n\r\n";
+
+    this->write( rq.data(), rq.size() );
+
+    std::string line = this->read_line(); // HTTP/1.0 (code) (message)
+
+    {
+        std::istringstream is( line );
+
+        std::string http, message;
+        int code;
+
+        if( is >> http >> code >> message && http.substr( 0, 7 ) == "HTTP/1." && code >= 100 )
+        {
+            if( code >= 300 )
+            {
+                throw_error( name_, "HTTP error: " + line );
+            }
+        }
+        else
+        {
+            throw_error( name_, "invalid server response: '" + line + "'" );
+        }
+    }
+
+    // skip rest of header
+    while( !read_line().empty() );
+}
+
+http_reader::~http_reader()
+{
+}
+
+std::string http_reader::name() const
+{
+    return name_;
+}
+
+std::string http_reader::read_line()
+{
+    std::string r;
+
+    for( ;; )
+    {
+        char ch;
+
+        std::size_t r2 = read( &ch, 1 );
+
+        if( r2 != 1 )
+        {
+            throw_error( name_, "unexpected end of data" );
+        }
+
+        if( ch == '\n' ) break;
+
+        if( ch != '\r' )
+        {
+            r += ch;
+        }
+    }
+
+    return r;
+}

src/http_reader.hpp

@@ -0,0 +1,54 @@
+#ifndef HTTP_READER_HPP_INCLUDED
+#define HTTP_READER_HPP_INCLUDED
+
+//
+// Copyright 2014 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "tcp_reader.hpp"
+
+class http_url_parser
+{
+private:
+
+    std::string host_;
+    int port_;
+
+    std::string request_;
+
+public:
+
+    explicit http_url_parser( std::string const & url );
+
+    std::string host() const;
+    int port() const;
+
+    std::string request() const;
+};
+
+class http_reader: private http_url_parser, public tcp_reader
+{
+private:
+
+    std::string name_;
+
+private:
+
+    http_reader( http_reader const & );
+    http_reader& operator=( http_reader const & );
+
+    std::string read_line();
+
+public:
+
+    explicit http_reader( std::string const & url );
+    ~http_reader();
+
+    virtual std::string name() const;
+};
+
+#endif // #ifndef HTTP_READER_HPP_INCLUDED

src/json.cpp

@@ -0,0 +1,286 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "json.hpp"
+#include "error.hpp"
+#include <fstream>
+#include <errno.h>
+
+static void throw_parse_error( std::string const & name, std::string const & reason )
+{
+    throw_error( name, "parse error: " + reason );
+}
+
+static void throw_eof_error( std::string const & name )
+{
+    throw_parse_error( name, "unexpected end of data" );
+}
+
+static void throw_expected_error( std::string const & name, std::string const & expected, std::string const & tk )
+{
+    throw_parse_error( name, "expected " + expected + ", got '" + tk + "'" );
+}
+
+static bool is_structural( char ch )
+{
+    return ch == '{' || ch == '}' || ch == '[' || ch == ']' || ch == ':' || ch == ',' || ch == 0;
+}
+
+static bool is_literal( char ch )
+{
+    return ( ch >= 'a' && ch <= 'z' ) || ( ch >= 'A' && ch <= 'Z' ) || ch == '_' || ( ch >= '0' && ch <= '9' ) || ch == '-' || ch == '+' || ch == '.';
+}
+
+static bool is_whitespace( char ch )
+{
+    return ch == 0x09 || ch == 0x0A || ch == 0x0D || ch == ' ';
+}
+
+static void utf8_encode( std::string & r, unsigned c )
+{
+    if( c < 0x80 )
+    {
+        r.push_back( static_cast<unsigned char>( c ) );
+    }
+    else if( c < 0x800 )
+    {
+        r.push_back( static_cast<unsigned char>( 0xC0 | (c >>   6) ) );
+        r.push_back( static_cast<unsigned char>( 0x80 | (c & 0x3F) ) );
+    }
+    else // if( c < 0x10000 )
+    {
+        r.push_back( static_cast<unsigned char>( 0xE0 | (c >> 12)         ) );
+        r.push_back( static_cast<unsigned char>( 0x80 | ((c >> 6) & 0x3F) ) );
+        r.push_back( static_cast<unsigned char>( 0x80 | (c & 0x3F)        ) );
+    }
+}
+
+static std::string get_token( std::istream & is, std::string const & name )
+{
+    char ch;
+
+    std::string tk;
+
+    if( !( is >> ch ) )
+    {
+        // EOF
+    }
+    else if( is_literal( ch ) )
+    {
+        for( ;; )
+        {
+            tk += ch;
+
+            if( !is.get( ch ) )
+            {
+                throw_eof_error( name );
+            }
+
+            if( is_structural( ch ) || is_whitespace( ch ) )
+            {
+                is.putback( ch );
+                break;
+            }
+        }
+    }
+    else if( ch != '"' )
+    {
+        tk += ch;
+    }
+    else
+    {
+        // string
+
+        tk += ch;
+
+        for( ;; )
+        {
+            if( !is.get( ch ) )
+            {
+                throw_eof_error( name );
+            }
+
+            if( ch == '"' )
+            {
+                break;
+            }
+
+            if( ch != '\\' )
+            {
+                tk += ch;
+                continue;
+            }
+
+            if( !is.get( ch ) )
+            {
+                throw_eof_error( name );
+            }
+
+            switch( ch )
+            {
+            case 'b': tk += '\x08'; break;
+            case 'f': tk += '\x0C'; break;
+            case 'r': tk += '\x0D'; break;
+            case 'n': tk += '\x0A'; break;
+            case 't': tk += '\x09'; break;
+            default:  tk += ch; break;
+
+            case 'u':
+                {
+                    char buffer[ 5 ] = { 0 };
+                    int code = 0;
+
+                    if( !is.read( buffer, 4 ) || sscanf( buffer, "%x", &code ) != 1 || code <= 0 )
+                    {
+                        throw_parse_error( name, std::string( "invalid character code '\\u" ) + buffer + "'" );
+                    }
+                    else
+                    {
+                        utf8_encode( tk, code );
+                    }
+                }
+                break;
+            }
+        }
+    }
+
+    return tk;
+}
+
+static void json_parse( std::istream & /*is*/, std::string const & name, std::string tk, std::string & v );
+template< class T > static void json_parse( std::istream & is, std::string const & name, std::string tk, std::vector< T > & v );
+template< class K, class V > static void json_parse( std::istream & is, std::string const & name, std::string tk, std::map< K, V > & m );
+
+// tk is already the result of get_token()
+static void json_parse( std::istream & /*is*/, std::string const & name, std::string tk, std::string & v )
+{
+    char ch = tk[ 0 ];
+
+    if( is_structural( ch ) )
+    {
+        throw_expected_error( name, "string", tk );
+    }
+
+    if( tk[ 0 ] == '"' )
+    {
+        v = tk.substr( 1 );
+    }
+    else
+    {
+        v = tk; // true, false, null, or number
+    }
+}
+
+// tk is already the result of get_token()
+template< class T > static void json_parse( std::istream & is, std::string const & name, std::string tk, std::vector< T > & v )
+{
+    if( tk != "[" )
+    {
+        // assume single element
+
+        T t;
+        json_parse( is, name, tk, t );
+
+        v.push_back( t );
+
+        return;
+    }
+
+    for( ;; )
+    {
+        tk = get_token( is, name );
+
+        if( tk == "]" )
+        {
+            break;
+        }
+
+        {
+            T t;
+            json_parse( is, name, tk, t );
+
+            v.push_back( t );
+        }
+
+        tk = get_token( is, name );
+
+        if( tk == "]" )
+        {
+            break;
+        }
+
+        if( tk != "," )
+        {
+            throw_expected_error( name, "',' or ']'", tk );
+        }
+    }
+}
+
+// tk is already the result of get_token()
+template< class K, class V > static void json_parse( std::istream & is, std::string const & name, std::string tk, std::map< K, V > & m )
+{
+    if( tk != "{" )
+    {
+        throw_expected_error( name, "'{'", tk );
+    }
+
+    for( ;; )
+    {
+        tk = get_token( is, name );
+
+        if( tk == "}" )
+        {
+            break;
+        }
+
+        K k;
+        json_parse( is, name, tk, k );
+
+        tk = get_token( is, name );
+
+        if( tk != ":" )
+        {
+            throw_expected_error( name, "':'", tk );
+            return;
+        }
+
+        tk = get_token( is, name );
+
+        V v;
+        json_parse( is, name, tk, v );
+
+        m[ k ] = v;
+
+        tk = get_token( is, name );
+
+        if( tk == "}" )
+        {
+            break;
+        }
+
+        if( tk != "," )
+        {
+            throw_expected_error( name, "',' or ']'", tk );
+        }
+    }
+}
+
+void read_libraries_json( std::string const & name, std::vector< std::map< std::string, std::vector< std::string > > > & libraries )
+{
+    std::ifstream is( name.c_str() );
+
+    if( !is )
+    {
+        throw_errno_error( name, "open error", errno );
+    }
+
+    std::string tk;
+    tk = get_token( is, name );
+
+    json_parse( is, name, tk, libraries );
+}

src/json.hpp

@@ -0,0 +1,18 @@
+#ifndef JSON_HPP_INCLUDED
+#define JSON_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include <string>
+#include <vector>
+#include <map>
+
+void read_libraries_json( std::string const & name, std::vector< std::map< std::string, std::vector< std::string > > > & libraries );
+
+#endif // #ifndef JSON_HPP_INCLUDED

src/lzma/7zTypes.h

@@ -0,0 +1,262 @@
+/* 7zTypes.h -- Basic types
+2013-11-12 : Igor Pavlov : Public domain */
+
+// No Copyright - Public Domain
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+
+#ifndef __7Z_TYPES_H
+#define __7Z_TYPES_H
+
+#ifdef _WIN32
+/* #include <windows.h> */
+#endif
+
+#include <stddef.h>
+
+#ifndef EXTERN_C_BEGIN
+#ifdef __cplusplus
+#define EXTERN_C_BEGIN extern "C" {
+#define EXTERN_C_END }
+#else
+#define EXTERN_C_BEGIN
+#define EXTERN_C_END
+#endif
+#endif
+
+EXTERN_C_BEGIN
+
+#define SZ_OK 0
+
+#define SZ_ERROR_DATA 1
+#define SZ_ERROR_MEM 2
+#define SZ_ERROR_CRC 3
+#define SZ_ERROR_UNSUPPORTED 4
+#define SZ_ERROR_PARAM 5
+#define SZ_ERROR_INPUT_EOF 6
+#define SZ_ERROR_OUTPUT_EOF 7
+#define SZ_ERROR_READ 8
+#define SZ_ERROR_WRITE 9
+#define SZ_ERROR_PROGRESS 10
+#define SZ_ERROR_FAIL 11
+#define SZ_ERROR_THREAD 12
+
+#define SZ_ERROR_ARCHIVE 16
+#define SZ_ERROR_NO_ARCHIVE 17
+
+typedef int SRes;
+
+#ifdef _WIN32
+/* typedef DWORD WRes; */
+typedef unsigned WRes;
+#else
+typedef int WRes;
+#endif
+
+#ifndef RINOK
+#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; }
+#endif
+
+typedef unsigned char Byte;
+typedef short Int16;
+typedef unsigned short UInt16;
+
+#ifdef _LZMA_UINT32_IS_ULONG
+typedef long Int32;
+typedef unsigned long UInt32;
+#else
+typedef int Int32;
+typedef unsigned int UInt32;
+#endif
+
+#ifdef _SZ_NO_INT_64
+
+/* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers.
+   NOTES: Some code will work incorrectly in that case! */
+
+typedef long Int64;
+typedef unsigned long UInt64;
+
+#else
+
+#if defined(_MSC_VER) || defined(__BORLANDC__)
+typedef __int64 Int64;
+typedef unsigned __int64 UInt64;
+#define UINT64_CONST(n) n
+#else
+typedef long long int Int64;
+typedef unsigned long long int UInt64;
+#define UINT64_CONST(n) n ## ULL
+#endif
+
+#endif
+
+#ifdef _LZMA_NO_SYSTEM_SIZE_T
+typedef UInt32 SizeT;
+#else
+typedef size_t SizeT;
+#endif
+
+typedef int Bool;
+#define True 1
+#define False 0
+
+
+#ifdef _WIN32
+#define MY_STD_CALL __stdcall
+#else
+#define MY_STD_CALL
+#endif
+
+#ifdef _MSC_VER
+
+#if _MSC_VER >= 1300
+#define MY_NO_INLINE __declspec(noinline)
+#else
+#define MY_NO_INLINE
+#endif
+
+#define MY_CDECL __cdecl
+#define MY_FAST_CALL __fastcall
+
+#else
+
+#define MY_NO_INLINE
+#define MY_CDECL
+#define MY_FAST_CALL
+
+#endif
+
+
+/* The following interfaces use first parameter as pointer to structure */
+
+typedef struct
+{
+  Byte (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */
+} IByteIn;
+
+typedef struct
+{
+  void (*Write)(void *p, Byte b);
+} IByteOut;
+
+typedef struct
+{
+  SRes (*Read)(void *p, void *buf, size_t *size);
+    /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
+       (output(*size) < input(*size)) is allowed */
+} ISeqInStream;
+
+/* it can return SZ_ERROR_INPUT_EOF */
+SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size);
+SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType);
+SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf);
+
+typedef struct
+{
+  size_t (*Write)(void *p, const void *buf, size_t size);
+    /* Returns: result - the number of actually written bytes.
+       (result < size) means error */
+} ISeqOutStream;
+
+typedef enum
+{
+  SZ_SEEK_SET = 0,
+  SZ_SEEK_CUR = 1,
+  SZ_SEEK_END = 2
+} ESzSeek;
+
+typedef struct
+{
+  SRes (*Read)(void *p, void *buf, size_t *size);  /* same as ISeqInStream::Read */
+  SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin);
+} ISeekInStream;
+
+typedef struct
+{
+  SRes (*Look)(void *p, const void **buf, size_t *size);
+    /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream.
+       (output(*size) > input(*size)) is not allowed
+       (output(*size) < input(*size)) is allowed */
+  SRes (*Skip)(void *p, size_t offset);
+    /* offset must be <= output(*size) of Look */
+
+  SRes (*Read)(void *p, void *buf, size_t *size);
+    /* reads directly (without buffer). It's same as ISeqInStream::Read */
+  SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin);
+} ILookInStream;
+
+SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size);
+SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset);
+
+/* reads via ILookInStream::Read */
+SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType);
+SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size);
+
+#define LookToRead_BUF_SIZE (1 << 14)
+
+typedef struct
+{
+  ILookInStream s;
+  ISeekInStream *realStream;
+  size_t pos;
+  size_t size;
+  Byte buf[LookToRead_BUF_SIZE];
+} CLookToRead;
+
+void LookToRead_CreateVTable(CLookToRead *p, int lookahead);
+void LookToRead_Init(CLookToRead *p);
+
+typedef struct
+{
+  ISeqInStream s;
+  ILookInStream *realStream;
+} CSecToLook;
+
+void SecToLook_CreateVTable(CSecToLook *p);
+
+typedef struct
+{
+  ISeqInStream s;
+  ILookInStream *realStream;
+} CSecToRead;
+
+void SecToRead_CreateVTable(CSecToRead *p);
+
+typedef struct
+{
+  SRes (*Progress)(void *p, UInt64 inSize, UInt64 outSize);
+    /* Returns: result. (result != SZ_OK) means break.
+       Value (UInt64)(Int64)-1 for size means unknown value. */
+} ICompressProgress;
+
+typedef struct
+{
+  void *(*Alloc)(void *p, size_t size);
+  void (*Free)(void *p, void *address); /* address can be 0 */
+} ISzAlloc;
+
+#define IAlloc_Alloc(p, size) (p)->Alloc((p), size)
+#define IAlloc_Free(p, a) (p)->Free((p), a)
+
+#ifdef _WIN32
+
+#define CHAR_PATH_SEPARATOR '\\'
+#define WCHAR_PATH_SEPARATOR L'\\'
+#define STRING_PATH_SEPARATOR "\\"
+#define WSTRING_PATH_SEPARATOR L"\\"
+
+#else
+
+#define CHAR_PATH_SEPARATOR '/'
+#define WCHAR_PATH_SEPARATOR L'/'
+#define STRING_PATH_SEPARATOR "/"
+#define WSTRING_PATH_SEPARATOR L"/"
+
+#endif
+
+EXTERN_C_END
+
+#endif

src/lzma/Compiler.h

@@ -0,0 +1,34 @@
+/* Compiler.h -- Compiler ypes
+2013-11-12 : Igor Pavlov : Public domain */
+
+// No Copyright - Public Domain
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+
+#ifndef __7Z_COMPILER_H
+#define __7Z_COMPILER_H
+
+#ifdef _MSC_VER
+
+  #ifdef UNDER_CE
+    #define RPC_NO_WINDOWS_H
+    /* #pragma warning(disable : 4115) // '_RPC_ASYNC_STATE' : named type definition in parentheses */
+    #pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union
+    #pragma warning(disable : 4214) // nonstandard extension used : bit field types other than int
+  #endif
+
+  #if _MSC_VER >= 1300
+    #pragma warning(disable : 4996) // This function or variable may be unsafe
+  #else
+    #pragma warning(disable : 4511) // copy constructor could not be generated
+    #pragma warning(disable : 4512) // assignment operator could not be generated
+    #pragma warning(disable : 4702) // unreachable code
+    #pragma warning(disable : 4710) // not inlined
+    #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information
+  #endif
+
+#endif
+
+#endif

src/lzma/LzmaDec.h

@@ -0,0 +1,233 @@
+/* LzmaDec.h -- LZMA Decoder
+2013-01-18 : Igor Pavlov : Public domain */
+
+// No Copyright - Public Domain
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+
+#ifndef __LZMA_DEC_H
+#define __LZMA_DEC_H
+
+#include "7zTypes.h"
+
+EXTERN_C_BEGIN
+
+/* #define _LZMA_PROB32 */
+/* _LZMA_PROB32 can increase the speed on some CPUs,
+   but memory usage for CLzmaDec::probs will be doubled in that case */
+
+#ifdef _LZMA_PROB32
+#define CLzmaProb UInt32
+#else
+#define CLzmaProb UInt16
+#endif
+
+
+/* ---------- LZMA Properties ---------- */
+
+#define LZMA_PROPS_SIZE 5
+
+typedef struct _CLzmaProps
+{
+  unsigned lc, lp, pb;
+  UInt32 dicSize;
+} CLzmaProps;
+
+/* LzmaProps_Decode - decodes properties
+Returns:
+  SZ_OK
+  SZ_ERROR_UNSUPPORTED - Unsupported properties
+*/
+
+SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size);
+
+
+/* ---------- LZMA Decoder state ---------- */
+
+/* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case.
+   Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */
+
+#define LZMA_REQUIRED_INPUT_MAX 20
+
+typedef struct
+{
+  CLzmaProps prop;
+  CLzmaProb *probs;
+  Byte *dic;
+  const Byte *buf;
+  UInt32 range, code;
+  SizeT dicPos;
+  SizeT dicBufSize;
+  UInt32 processedPos;
+  UInt32 checkDicSize;
+  unsigned state;
+  UInt32 reps[4];
+  unsigned remainLen;
+  int needFlush;
+  int needInitState;
+  UInt32 numProbs;
+  unsigned tempBufSize;
+  Byte tempBuf[LZMA_REQUIRED_INPUT_MAX];
+} CLzmaDec;
+
+#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; }
+
+void LzmaDec_Init(CLzmaDec *p);
+
+/* There are two types of LZMA streams:
+     0) Stream with end mark. That end mark adds about 6 bytes to compressed size.
+     1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */
+
+typedef enum
+{
+  LZMA_FINISH_ANY,   /* finish at any point */
+  LZMA_FINISH_END    /* block must be finished at the end */
+} ELzmaFinishMode;
+
+/* ELzmaFinishMode has meaning only if the decoding reaches output limit !!!
+
+   You must use LZMA_FINISH_END, when you know that current output buffer
+   covers last bytes of block. In other cases you must use LZMA_FINISH_ANY.
+
+   If LZMA decoder sees end marker before reaching output limit, it returns SZ_OK,
+   and output value of destLen will be less than output buffer size limit.
+   You can check status result also.
+
+   You can use multiple checks to test data integrity after full decompression:
+     1) Check Result and "status" variable.
+     2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize.
+     3) Check that output(srcLen) = compressedSize, if you know real compressedSize.
+        You must use correct finish mode in that case. */
+
+typedef enum
+{
+  LZMA_STATUS_NOT_SPECIFIED,               /* use main error code instead */
+  LZMA_STATUS_FINISHED_WITH_MARK,          /* stream was finished with end mark. */
+  LZMA_STATUS_NOT_FINISHED,                /* stream was not finished */
+  LZMA_STATUS_NEEDS_MORE_INPUT,            /* you must provide more input bytes */
+  LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK  /* there is probability that stream was finished without end mark */
+} ELzmaStatus;
+
+/* ELzmaStatus is used only as output value for function call */
+
+
+/* ---------- Interfaces ---------- */
+
+/* There are 3 levels of interfaces:
+     1) Dictionary Interface
+     2) Buffer Interface
+     3) One Call Interface
+   You can select any of these interfaces, but don't mix functions from different
+   groups for same object. */
+
+
+/* There are two variants to allocate state for Dictionary Interface:
+     1) LzmaDec_Allocate / LzmaDec_Free
+     2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs
+   You can use variant 2, if you set dictionary buffer manually.
+   For Buffer Interface you must always use variant 1.
+
+LzmaDec_Allocate* can return:
+  SZ_OK
+  SZ_ERROR_MEM         - Memory allocation error
+  SZ_ERROR_UNSUPPORTED - Unsupported properties
+*/
+   
+SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc);
+void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc);
+
+SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAlloc *alloc);
+void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc);
+
+/* ---------- Dictionary Interface ---------- */
+
+/* You can use it, if you want to eliminate the overhead for data copying from
+   dictionary to some other external buffer.
+   You must work with CLzmaDec variables directly in this interface.
+
+   STEPS:
+     LzmaDec_Constr()
+     LzmaDec_Allocate()
+     for (each new stream)
+     {
+       LzmaDec_Init()
+       while (it needs more decompression)
+       {
+         LzmaDec_DecodeToDic()
+         use data from CLzmaDec::dic and update CLzmaDec::dicPos
+       }
+     }
+     LzmaDec_Free()
+*/
+
+/* LzmaDec_DecodeToDic
+   
+   The decoding to internal dictionary buffer (CLzmaDec::dic).
+   You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!!
+
+finishMode:
+  It has meaning only if the decoding reaches output limit (dicLimit).
+  LZMA_FINISH_ANY - Decode just dicLimit bytes.
+  LZMA_FINISH_END - Stream must be finished after dicLimit.
+
+Returns:
+  SZ_OK
+    status:
+      LZMA_STATUS_FINISHED_WITH_MARK
+      LZMA_STATUS_NOT_FINISHED
+      LZMA_STATUS_NEEDS_MORE_INPUT
+      LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK
+  SZ_ERROR_DATA - Data error
+*/
+
+SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit,
+    const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);
+
+
+/* ---------- Buffer Interface ---------- */
+
+/* It's zlib-like interface.
+   See LzmaDec_DecodeToDic description for information about STEPS and return results,
+   but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need
+   to work with CLzmaDec variables manually.
+
+finishMode:
+  It has meaning only if the decoding reaches output limit (*destLen).
+  LZMA_FINISH_ANY - Decode just destLen bytes.
+  LZMA_FINISH_END - Stream must be finished after (*destLen).
+*/
+
+SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen,
+    const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);
+
+
+/* ---------- One Call Interface ---------- */
+
+/* LzmaDecode
+
+finishMode:
+  It has meaning only if the decoding reaches output limit (*destLen).
+  LZMA_FINISH_ANY - Decode just destLen bytes.
+  LZMA_FINISH_END - Stream must be finished after (*destLen).
+
+Returns:
+  SZ_OK
+    status:
+      LZMA_STATUS_FINISHED_WITH_MARK
+      LZMA_STATUS_NOT_FINISHED
+      LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK
+  SZ_ERROR_DATA - Data error
+  SZ_ERROR_MEM  - Memory allocation error
+  SZ_ERROR_UNSUPPORTED - Unsupported properties
+  SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src).
+*/
+
+SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen,
+    const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode,
+    ELzmaStatus *status, ISzAlloc *alloc);
+
+EXTERN_C_END
+
+#endif

src/lzma/Precomp.h

@@ -0,0 +1,16 @@
+/* Precomp.h -- StdAfx
+2013-11-12 : Igor Pavlov : Public domain */
+
+// No Copyright - Public Domain
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+
+#ifndef __7Z_PRECOMP_H
+#define __7Z_PRECOMP_H
+
+#include "Compiler.h"
+/* #include "7zTypes.h" */
+
+#endif

src/lzma_reader.cpp

@@ -0,0 +1,109 @@
+//
+// Copyright 2014, 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "lzma_reader.hpp"
+#include "error.hpp"
+#include "lzma/LzmaDec.h"
+#include <stdlib.h>
+
+void* SzAlloc( void*, size_t size )
+{
+    return malloc( size );
+}
+
+void SzFree( void*, void * address )
+{
+    free( address );
+}
+
+static ISzAlloc s_alloc = { SzAlloc, SzFree };
+
+lzma_reader::lzma_reader( basic_reader * pr ): pr_( pr ), k_( 0 ), m_( 0 )
+{
+    typedef char assert_large_enough[ sizeof( state_ ) >= sizeof( CLzmaDec )? 1: -1 ];
+
+    {
+        std::size_t r = pr_->read( header_, sizeof( header_ ) );
+
+        if( r < sizeof( header_ ) )
+        {
+            throw_error( pr_->name(), "could not read LZMA header" );
+        }
+    }
+
+    CLzmaDec * st = (CLzmaDec*)state_;
+
+    LzmaDec_Construct( st );
+
+    {
+        SRes r = LzmaDec_Allocate( st, header_, LZMA_PROPS_SIZE, &s_alloc );
+
+        if( r != SZ_OK )
+        {
+            throw_error( pr_->name(), "could not allocate LZMA state" );
+        }
+    }
+
+    LzmaDec_Init( st );
+}
+
+lzma_reader::~lzma_reader()
+{
+    CLzmaDec * st = (CLzmaDec*)state_;
+    LzmaDec_Free( st, &s_alloc );
+}
+
+std::string lzma_reader::name() const
+{
+    return pr_->name();
+}
+
+std::size_t lzma_reader::read( void * p, std::size_t n )
+{
+    CLzmaDec * st = (CLzmaDec*)state_;
+    unsigned char * p2 = static_cast< unsigned char* >( p );
+
+    std::size_t r2 = 0;
+
+    while( n > 0 )
+    {
+        if( m_ == 0 )
+        {
+            std::size_t r = pr_->read( buffer_, N );
+
+            if( r == 0 ) return r2;
+
+            k_ = 0;
+            m_ = r;
+        }
+
+        {
+            std::size_t n2 = n;
+            std::size_t m2 = m_;
+
+            ELzmaStatus status = LZMA_STATUS_NOT_SPECIFIED;
+
+            SRes r = LzmaDec_DecodeToBuf( st, p2, &n2, buffer_ + k_, &m2, LZMA_FINISH_ANY, &status );
+
+            if( r != 0 )
+            {
+                return -( 1000 + r );
+            }
+
+            k_ += m2;
+            m_ -= m2;
+
+            p2 += n2;
+            n -= n2;
+
+            r2 += n2;
+        }
+    }
+
+    return r2;
+}

src/lzma_reader.hpp

@@ -0,0 +1,43 @@
+#ifndef LZMA_READER_HPP_INCLUDED
+#define LZMA_READER_HPP_INCLUDED
+
+//
+// Copyright 2014 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "basic_reader.hpp"
+
+class lzma_reader: public basic_reader
+{
+private:
+
+    basic_reader * pr_;
+
+    unsigned char header_[ 13 ];
+    unsigned char state_[ 128 ];
+
+    static int const N = 4096;
+
+    unsigned char buffer_[ N ];
+    int k_, m_;
+
+private:
+
+    lzma_reader( lzma_reader const & );
+    lzma_reader& operator=( lzma_reader const & );
+
+public:
+
+    explicit lzma_reader( basic_reader * pr );
+
+    ~lzma_reader();
+
+    virtual std::string name() const;
+    virtual std::size_t read( void * p, std::size_t n );
+};
+
+#endif // #ifndef LZMA_READER_HPP_INCLUDED

src/message.cpp

@@ -0,0 +1,50 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "message.hpp"
+#include <stdio.h>
+#include <stdarg.h>
+
+static int s_level;
+
+int get_message_level()
+{
+    return s_level;
+}
+
+void set_message_level( int level )
+{
+    s_level = level;
+}
+
+void increase_message_level()
+{
+    ++s_level;
+}
+
+void decrease_message_level()
+{
+    --s_level;
+}
+
+void msg_printf( int level, char const * format, ... )
+{
+    if( level <= s_level )
+    {
+        fprintf( stderr, "bpm: " );
+
+        va_list args;
+        va_start( args, format );
+
+        vfprintf( stderr, format, args );
+
+        va_end( args );
+
+        fprintf( stderr, "\n" );
+    }
+}

src/message.hpp

@@ -0,0 +1,20 @@
+#ifndef MESSAGE_HPP_INCLUDED
+#define MESSAGE_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+int get_message_level();
+void set_message_level( int level );
+
+void increase_message_level();
+void decrease_message_level();
+
+void msg_printf( int level, char const * format, ... );
+
+#endif // #ifndef MESSAGE_HPP_INCLUDED

src/options.cpp

@@ -0,0 +1,40 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "options.hpp"
+
+void parse_options( char const * * & argv, void (*handle_option)( std::string const & opt ) )
+{
+    while( argv[ 0 ] && ( argv[ 0 ][ 0 ] == '-' || argv[ 0 ][ 0 ] == '+' ) )
+    {
+        if( argv[ 0 ][ 0 ] == '-' && argv[ 0 ][ 1 ] == '-' )
+        {
+            if( argv[ 0 ][ 2 ] == 0 )
+            {
+                // -- turns off option processing
+
+                ++argv;
+                break;
+            }
+            else
+            {
+                handle_option( argv[ 0 ] );
+            }
+        }
+        else
+        {
+            for( char const * p = argv[ 0 ] + 1; *p; ++p )
+            {
+                char opt[ 3 ] = { argv[ 0 ][ 0 ], *p, 0 };
+                handle_option( opt );
+            }
+        }
+
+        ++argv;
+    }
+}

src/options.hpp

@@ -0,0 +1,16 @@
+#ifndef OPTIONS_HPP_INCLUDED
+#define OPTIONS_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include <string>
+
+void parse_options( char const * * & argv, void (*handle_option)( std::string const & opt ) );
+
+#endif // #ifndef OPTIONS_HPP_INCLUDED

src/package_path.cpp

@@ -0,0 +1,23 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "package_path.hpp"
+#include "config.hpp"
+#include <stdexcept>
+
+std::string get_package_path()
+{
+    std::string path = config_get_option( "package_path" );
+
+    if( path.substr( 0, 7 ) != "http://" || *path.rbegin() != '/' )
+    {
+        throw std::runtime_error( "invalid package_path '" + path +  "' in bpm.conf" );
+    }
+
+    return path;
+}

src/package_path.hpp

@@ -0,0 +1,16 @@
+#ifndef PACKAGE_PATH_HPP_INCLUDED
+#define PACKAGE_PATH_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include <string>
+
+std::string get_package_path();
+
+#endif // #ifndef PACKAGE_PATH_HPP_INCLUDED

src/string.cpp

@@ -0,0 +1,19 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "string.hpp"
+
+void remove_trailing( std::string & s, char ch )
+{
+    std::string::iterator last = s.end();
+
+    if( last != s.begin() && ( --last, *last == ch ) )
+    {
+        s.erase( last );
+    }
+}

src/string.hpp

@@ -0,0 +1,16 @@
+#ifndef STRING_HPP_INCLUDED
+#define STRING_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include <string>
+
+void remove_trailing( std::string & s, char ch );
+
+#endif // #ifndef STRING_HPP_INCLUDED

src/tar.cpp

@@ -0,0 +1,300 @@
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "tar.hpp"
+#include "error.hpp"
+#include "fs.hpp"
+#include "message.hpp"
+#include <cstdio>
+#include <cstddef>
+#include <cassert>
+#include <ctime>
+#include <cstring>
+#include <errno.h>
+
+static void throw_eof_error( std::string const & name )
+{
+    throw_error( name, "unexpected end of file" );
+}
+
+static bool contains_dotdot( std::string const & fn )
+{
+    std::size_t i = 0;
+
+    for( ;; )
+    {
+        std::size_t j = fn.find( '/', i );
+
+        if( j == std::string::npos )
+        {
+            return fn.substr( i ) == "..";
+        }
+
+        if( fn.substr( i, j - i ) == ".." )
+        {
+            return true;
+        }
+
+        i = j + 1;
+    }
+}
+
+int const N = 512;
+
+static bool read_header( basic_reader * pr, char (&header)[ N ], std::string & fn, char & type, long long & size, int & mode, long long & mtime )
+{
+    {
+        std::size_t r = pr->read( header, N );
+
+        if( r == 0 ) return true; // EOF
+
+        if( r < N )
+        {
+            throw_eof_error( pr->name() );
+        }
+    }
+
+    if( header[0] == 0 )
+    {
+        // empty block
+
+        unsigned m = 0;
+
+        for( int i = 0; i < N; ++i )
+        {
+            m |= header[ i ];
+        }
+
+        if( m != 0 )
+        {
+            throw_error( pr->name(), "bad block" );
+        }
+
+        return false;
+    }
+
+    // check header checksum
+
+    int checksum = 0;
+    std::sscanf( header + 148, "%8o", &checksum );
+
+    {
+        int s = 0;
+
+        std::memset( header + 148, ' ', 8 );
+
+        for( int i = 0; i < N; ++i )
+        {
+            s += static_cast< unsigned char >( header[ i ] );
+        }
+
+        if( s != checksum )
+        {
+            throw_error( pr->name(), "header checksum mismatch" );
+        }
+    }
+
+    header[ 99 ] = 0;
+    fn = header;
+
+    header[ 257 + 5 ] = 0;
+    std::string magic( header + 257 );
+
+    if( magic == "ustar" )
+    {
+        header[ 345 + 130 ] = 0;
+        std::string prefix( header + 345 );
+
+        fn = prefix + fn;
+    }
+
+    type = header[ 156 ];
+
+    size = 0;
+    std::sscanf( header + 124, "%12llo", &size );
+
+    mode = 0;
+    std::sscanf( header + 100, "%8o", &mode );
+
+    mtime = 0;
+    std::sscanf( header + 136, "%12llo", &mtime );
+
+    return false;
+}
+
+static void read_long_name( basic_reader * pr, long long size, std::string & fn )
+{
+    fn.clear();
+
+    long long k = 0;
+
+    while( k < size )
+    {
+        char data[ N ];
+
+        {
+            std::size_t r = pr->read( data, N );
+
+            if( r < N )
+            {
+                throw_eof_error( pr->name() );
+            }
+        }
+
+        unsigned m;
+
+        if( k + N <= size )
+        {
+            m = N;
+        }
+        else
+        {
+            assert( size - k <= UINT_MAX );
+            assert( size - k <= N );
+
+            m = static_cast< unsigned >( size - k );
+        }
+
+        fn.append( data, m );
+
+        k += N;
+    }
+}
+
+void tar_extract( basic_reader * pr, std::string const & prefix, std::set< std::string > const & whitelist )
+{
+    msg_printf( 1, "extracting from '%s'", pr->name().c_str() );
+
+    for( ;; )
+    {
+        char header[ N ];
+
+        std::string fn;
+
+        char type = 0;
+        long long size = 0;
+        int mode = 0;
+        long long mtime = 0;
+
+        if( read_header( pr, header, fn, type, size, mode, mtime ) )
+        {
+            break; // EOF
+        }
+
+        if( header[ 0 ] == 0 )
+        {
+            continue;
+        }
+
+        if( type == 'L' )
+        {
+            // GNU long name extension
+
+            read_long_name( pr, size, fn );
+
+            std::string fn2;
+
+            if( read_header( pr, header, fn2, type, size, mode, mtime ) )
+            {
+                throw_eof_error( pr->name() );
+            }
+        }
+
+        if( ( fn.substr( 0, prefix.size() ) != prefix && whitelist.count( fn ) == 0 ) || contains_dotdot( fn ) )
+        {
+            throw_error( pr->name(), "disallowed file name: '" + fn + "'" );
+        }
+
+        if( type != 0 && type != '0' && type != '5' )
+        {
+            throw_error( pr->name(), "disallowed file type" );
+        }
+
+        msg_printf( 2, "extracting '%s'", fn.c_str() );
+
+        if( type == '5' ) // directory
+        {
+            if( size != 0 )
+            {
+                throw_error( pr->name(), "directory with nonzero size: '" + fn + "'" );
+            }
+
+            int r = fs_mkdir( fn, 0755 );
+
+            if( r < 0 )
+            {
+                throw_errno_error( fn, "create error", errno );
+            }
+
+            if( fn != prefix )
+            {
+                fs_utime( fn, mtime, std::time( 0 ) );
+            }
+        }
+        else // regular file
+        {
+            int fd = fs_creat( fn, 0644 );
+
+            if( fd < 0 )
+            {
+                throw_errno_error( fn, "create error", errno );
+            }
+
+            long long k = 0;
+
+            while( k < size )
+            {
+                char data[ N ];
+
+                {
+                    std::size_t r = pr->read( data, N );
+
+                    if( r < N )
+                    {
+                        throw_eof_error( pr->name() );
+                    }
+                }
+
+                unsigned m;
+
+                if( k + N <= size )
+                {
+                    m = N;
+                }
+                else
+                {
+                    assert( size - k <= UINT_MAX );
+                    assert( size - k <= N );
+
+                    m = static_cast< unsigned >( size - k );
+                }
+
+                int r = fs_write( fd, data, m );
+
+                if( r < 0 )
+                {
+                    int r2 = errno;
+
+                    fs_close( fd );
+                    throw_errno_error( fn, "write error", r2 );
+                }
+
+                if( r != m )
+                {
+                    fs_close( fd );
+                    throw_errno_error( fn, "write error", ENOSPC );
+                }
+
+                k += N;
+            }
+
+            fs_close( fd );
+            fs_utime( fn, mtime, std::time( 0 ) );
+        }
+    }
+}

src/tar.hpp

@@ -0,0 +1,18 @@
+#ifndef TAR_HPP_INCLUDED
+#define TAR_HPP_INCLUDED
+
+//
+// Copyright 2015 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "basic_reader.hpp"
+#include <string>
+#include <set>
+
+void tar_extract( basic_reader * pr, std::string const & prefix, std::set< std::string > const & whitelist );
+
+#endif // #ifndef TAR_HPP_INCLUDED

src/tcp_reader.cpp

@@ -0,0 +1,161 @@
+//
+// Copyright 2014 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "tcp_reader.hpp"
+#include "error.hpp"
+#include <cassert>
+#include <cstdio>
+
+#ifdef _WIN32
+# include <winsock.h>
+#else
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/ioctl.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <errno.h>
+#include <unistd.h>
+#include <netdb.h>
+
+typedef int SOCKET;
+
+int const INVALID_SOCKET = -1;
+
+inline int closesocket( SOCKET s )
+{
+    return close( s );
+}
+
+inline int WSAGetLastError()
+{
+    return errno;
+}
+
+#endif
+
+static bool is_dotted_numeric( char const * p )
+{
+    for( ; *p; ++p )
+    {
+        if( !( *p >= '0' && *p <= '9' ) && *p != '.' )
+        {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+tcp_reader::tcp_reader( std::string const & host, int port )
+{
+    {
+        char buffer[ 32 ];
+        std::sprintf( buffer, "%d", port );
+
+        name_ = host + ':' + buffer;
+    }
+
+    if( port <= 0 || port >= 65536 )
+    {
+        throw_error( name_, "invalid port number" );
+    }
+
+    unsigned long a;
+
+    char const * p = host.c_str();
+
+    if( is_dotted_numeric( p ) )
+    {
+        a = inet_addr( p );
+
+        if( a == INADDR_NONE || a == 0 )
+        {
+            throw_error( host, "invalid host address" );
+        }
+    }
+    else
+    {
+        hostent const * q = gethostbyname( p );
+
+        if( q == 0 || q->h_length != 4 )
+        {
+            throw_error( host, "unable to resolve host name" );
+        }
+
+        a = *reinterpret_cast<unsigned long const *>( q->h_addr_list[0] );
+    }
+
+    sk_ = ::socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
+
+    if( sk_ == INVALID_SOCKET )
+    {
+        throw_socket_error( name_, "socket create error", WSAGetLastError() );
+    }
+
+    sockaddr_in addr = { AF_INET };
+
+    addr.sin_addr.s_addr = a;
+    addr.sin_port = htons( static_cast<u_short>( port ) );
+
+    int r = ::connect( sk_, (sockaddr const *)&addr, sizeof(addr) );
+
+    if( r != 0 )
+    {
+        int r2 = WSAGetLastError();
+
+        closesocket( sk_ );
+
+        throw_socket_error( name_, "TCP connect error", r2 );
+    }
+}
+
+tcp_reader::~tcp_reader()
+{
+    shutdown( sk_, 2 );
+    closesocket( sk_ );
+}
+
+std::string tcp_reader::name() const
+{
+    return name_;
+}
+
+std::size_t tcp_reader::read( void * p, std::size_t n )
+{
+    assert( n <= INT_MAX );
+
+    int r = ::recv( sk_, static_cast< char* >( p ), n, 0 );
+
+    if( r < 0 )
+    {
+        throw_socket_error( name(), "TCP receive error", WSAGetLastError() );
+    }
+
+    return r;
+}
+
+void tcp_reader::write( void const * p, std::size_t n )
+{
+    assert( n <= INT_MAX );
+
+    int r = ::send( sk_, static_cast< char const* >( p ), n, 0 );
+
+    if( r < 0 || r < n )
+    {
+        throw_socket_error( name(), "TCP send error", WSAGetLastError() );
+    }
+}
+
+#if defined( _WIN32 )
+
+static WSADATA s_wd;
+static int s_wsa_startup = WSAStartup( MAKEWORD( 1, 1 ), &s_wd );
+
+#endif // defined( _WIN32 )

src/tcp_reader.hpp

@@ -0,0 +1,40 @@
+#ifndef TCP_READER_HPP_INCLUDED
+#define TCP_READER_HPP_INCLUDED
+
+//
+// Copyright 2014 Peter Dimov
+//
+// Distributed under the Boost Software License, Version 1.0.
+// See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt
+//
+
+#include "basic_reader.hpp"
+
+class tcp_reader: public basic_reader
+{
+private:
+
+    std::ptrdiff_t sk_;
+    std::string name_;
+
+private:
+
+    tcp_reader( tcp_reader const & );
+    tcp_reader& operator=( tcp_reader const & );
+
+public:
+
+    explicit tcp_reader( std::string const & host, int port );
+
+    ~tcp_reader();
+
+    virtual std::string name() const;
+    virtual std::size_t read( void * p, std::size_t n );
+
+protected:
+
+    void write( void const * p, std::size_t n );
+};
+
+#endif // #ifndef TCP_READER_HPP_INCLUDED