diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3e84d7c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,96 @@ +* text=auto !eol svneol=native#text/plain +*.gitattributes text svneol=native#text/plain + +# Scriptish formats +*.bat text svneol=native#text/plain +*.bsh text svneol=native#text/x-beanshell +*.cgi text svneol=native#text/plain +*.cmd text svneol=native#text/plain +*.js text svneol=native#text/javascript +*.php text svneol=native#text/x-php +*.pl text svneol=native#text/x-perl +*.pm text svneol=native#text/x-perl +*.py text svneol=native#text/x-python +*.sh eol=lf svneol=LF#text/x-sh +configure eol=lf svneol=LF#text/x-sh + +# Image formats +*.bmp binary svneol=unset#image/bmp +*.gif binary svneol=unset#image/gif +*.ico binary svneol=unset#image/ico +*.jpeg binary svneol=unset#image/jpeg +*.jpg binary svneol=unset#image/jpeg +*.png binary svneol=unset#image/png +*.tif binary svneol=unset#image/tiff +*.tiff binary svneol=unset#image/tiff +*.svg text svneol=native#image/svg%2Bxml + +# Data formats +*.pdf binary svneol=unset#application/pdf +*.avi binary svneol=unset#video/avi +*.doc binary svneol=unset#application/msword +*.dsp text svneol=crlf#text/plain +*.dsw text svneol=crlf#text/plain +*.eps binary svneol=unset#application/postscript +*.gz binary svneol=unset#application/gzip +*.mov binary svneol=unset#video/quicktime +*.mp3 binary svneol=unset#audio/mpeg +*.ppt binary svneol=unset#application/vnd.ms-powerpoint +*.ps binary svneol=unset#application/postscript +*.psd binary svneol=unset#application/photoshop +*.rdf binary svneol=unset#text/rdf +*.rss text svneol=unset#text/xml +*.rtf binary svneol=unset#text/rtf +*.sln text svneol=native#text/plain +*.swf binary svneol=unset#application/x-shockwave-flash +*.tgz binary svneol=unset#application/gzip +*.vcproj text svneol=native#text/xml +*.vcxproj text svneol=native#text/xml +*.vsprops text svneol=native#text/xml +*.wav binary svneol=unset#audio/wav +*.xls binary svneol=unset#application/vnd.ms-excel +*.zip binary svneol=unset#application/zip + +# Text formats +.htaccess text svneol=native#text/plain +*.bbk text svneol=native#text/xml +*.cmake text svneol=native#text/plain +*.css text svneol=native#text/css +*.dtd text svneol=native#text/xml +*.htm text svneol=native#text/html +*.html text svneol=native#text/html +*.ini text svneol=native#text/plain +*.log text svneol=native#text/plain +*.mak text svneol=native#text/plain +*.qbk text svneol=native#text/plain +*.rst text svneol=native#text/plain +*.sql text svneol=native#text/x-sql +*.txt text svneol=native#text/plain +*.xhtml text svneol=native#text/xhtml%2Bxml +*.xml text svneol=native#text/xml +*.xsd text svneol=native#text/xml +*.xsl text svneol=native#text/xml +*.xslt text svneol=native#text/xml +*.xul text svneol=native#text/xul +*.yml text svneol=native#text/plain +boost-no-inspect text svneol=native#text/plain +CHANGES text svneol=native#text/plain +COPYING text svneol=native#text/plain +INSTALL text svneol=native#text/plain +Jamfile text svneol=native#text/plain +Jamroot text svneol=native#text/plain +Jamfile.v2 text svneol=native#text/plain +Jamrules text svneol=native#text/plain +Makefile* text svneol=native#text/plain +README text svneol=native#text/plain +TODO text svneol=native#text/plain + +# Code formats +*.c text svneol=native#text/plain +*.cpp text svneol=native#text/plain +*.h text svneol=native#text/plain +*.hpp text svneol=native#text/plain +*.ipp text svneol=native#text/plain +*.tpp text svneol=native#text/plain +*.jam text svneol=native#text/plain +*.java text svneol=native#text/plain diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 new file mode 100644 index 0000000..dd8622b --- /dev/null +++ b/build/Jamfile.v2 @@ -0,0 +1,30 @@ +# Copyright 2014, 2015 Peter Dimov +# +# Distributed under the Boost Software License, Version 1.0. +# See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt + +project : default-build release ; + +local SOURCES = + + bpm.cpp cmd_headers.cpp cmd_index.cpp cmd_install.cpp + cmd_list.cpp cmd_remove.cpp config.cpp dependencies.cpp + error.cpp file_reader.cpp fs.cpp http_reader.cpp json.cpp + lzma_reader.cpp message.cpp options.cpp package_path.cpp + string.cpp tar.cpp tcp_reader.cpp lzma/LzmaDec.c ; + +lib ws2_32 ; + +exe bpm : ../src/$(SOURCES) : + + NT:ws2_32 + + msvc:static + msvc:/wd4996 ; + +install dist-bin : bpm : ../../../dist/bin ; +explicit dist-bin ; + +alias install : dist-bin ; +explicit install ; diff --git a/scripts/package.bat b/scripts/package.bat new file mode 100644 index 0000000..97daf8e --- /dev/null +++ b/scripts/package.bat @@ -0,0 +1,28 @@ +@REM This cmd.exe batch script generates +@REM a package-based Boost distribution +@REM for use with bpm. +@REM +@REM It needs to be run from the Boost root. +@REM +@REM Copyright 2014, 2015 Peter Dimov +@REM +@REM Distributed under the Boost Software License, Version 1.0. +@REM See accompanying file LICENSE_1_0.txt or copy at +@REM http://www.boost.org/LICENSE_1_0.txt + +FOR /f %%i IN ('git rev-parse HEAD') DO @SET REV=%%i + +FOR /f %%i IN ('git rev-parse --short HEAD') DO @SET SHREV=%%i + +FOR /f %%i IN ('git rev-parse --abbrev-ref HEAD') DO @SET BRANCH=%%i + +SET OUTDIR=..\pkg-%BRANCH%-%SHREV% + +mkdir %OUTDIR% + +dist\bin\boostdep.exe --track-sources --list-dependencies | xz --format=lzma > %OUTDIR%\dependencies.txt.lzma +dist\bin\boostdep.exe --list-buildable | xz --format=lzma > %OUTDIR%\buildable.txt.lzma + +FOR /d %%i IN (libs/*) DO tar cf %OUTDIR%\%%i.tar.lzma --lzma libs/%%i/ + +tar cf %OUTDIR%\build.tar.lzma --lzma b2.exe boost-build.jam boostcpp.jam Jamroot libs/Jamfile.v2 tools/build/ diff --git a/src/basic_reader.hpp b/src/basic_reader.hpp new file mode 100644 index 0000000..17a91b1 --- /dev/null +++ b/src/basic_reader.hpp @@ -0,0 +1,34 @@ +#ifndef BASIC_READER_HPP_INCLUDED +#define BASIC_READER_HPP_INCLUDED + +// +// Copyright 2014 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include +#include + +class basic_reader +{ +protected: + + ~basic_reader() + { + } + +public: + + virtual std::string name() const = 0; + + // read throws on error, and returns number of bytes read + // which can be less than n when the end of the stream has + // been reached + + virtual std::size_t read( void * p, std::size_t n ) = 0; +}; + +#endif // #ifndef BASIC_READER_HPP_INCLUDED diff --git a/src/bpm.cpp b/src/bpm.cpp new file mode 100644 index 0000000..b8852f4 --- /dev/null +++ b/src/bpm.cpp @@ -0,0 +1,155 @@ +// +// bpm - a tool to install Boost modules +// +// Copyright 2014, 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "message.hpp" +#include "error.hpp" +#include "config.hpp" +#include "options.hpp" +#include "cmd_install.hpp" +#include "cmd_headers.hpp" +#include "cmd_index.hpp" +#include "cmd_remove.hpp" +#include "cmd_list.hpp" +#include +#include +#include +#include +#include + +static void handle_option( std::string const & opt ) +{ + if( opt == "-v" ) + { + increase_message_level(); + } + else if( opt == "-q" ) + { + decrease_message_level(); + } + else + { + throw std::runtime_error( "invalid option: '" + opt + "'" ); + } +} + +static void usage() +{ + printf( "%s", + + "Usage: bpm [-v] [-q] command [options] [modules]\n\n" + + " -v: Be verbose\n" + " -vv: Be more verbose\n" + " -q: Be quiet\n\n" + + " bpm install [-n] [+d] [-k] [-a] [-i] [-p] ...\n\n" + + " Installs the specified modules and their dependencies into\n" + " the current directory.\n\n" + + " -n: Only output what would be installed\n" + " +d: Do not install dependencies\n" + " -k: Do not remove partial installations on error\n" + " -a: All modules (use instead of a module list)\n" + " -i: Installed modules\n" + " -p: Partially installed modules\n\n" + + " bpm remove [-n] [-f] [-d] [-a] [-p] ...\n\n" + + " Removes the specified packages.\n\n" + + " (A package, like 'numeric', can contain more than one module.)\n\n" + + " -n: Only output what would be removed\n" + " -f: Force removal even when dependents exist\n" + " -d: Remove dependents as well. Requires -f\n" + " -a: Remove all packages. Requires -f\n" + " -p: Remove partially installed packages\n\n" + + " bpm list [-a] [-i] [-p] [-b] [prefix]\n\n" + + " Lists modules matching [prefix].\n\n" + + " -a: All modules (default when no -b)\n" + " -i: Installed modules (default when -b)\n" + " -p: Partially installed modules\n" + " -b: Modules that require building\n\n" + + " bpm headers\n\n" + + " Recreates the header links in the include/ subdirectory of\n" + " the current directory.\n\n" + + " bpm index\n\n" + + " Recreates the file index.html, which lists the installed\n" + " modules, in the current directory.\n" + + ); +} + +int main( int argc, char const * argv[] ) +{ + if( argc < 2 || std::string( argv[ 1 ] ) == "--help" ) + { + usage(); + return 0; + } + + try + { + ++argv; + + config_read_file( "bpm.conf" ); + + parse_options( argv, handle_option ); + + if( *argv == 0 ) + { + throw std::runtime_error( "missing command" ); + } + + std::string command( *argv++ ); + + if( command == "install" ) + { + cmd_install( argv ); + } + else if( command == "remove" ) + { + cmd_remove( argv ); + } + else if( command == "headers" ) + { + cmd_headers( argv ); + } + else if( command == "index" ) + { + cmd_index( argv ); + } + else if( command == "list" ) + { + cmd_list( argv ); + } + else if( command == "help" ) + { + usage(); + } + else + { + throw std::runtime_error( "invalid command: '" + command + "'" ); + } + } + catch( std::exception const & x ) + { + fprintf( stderr, "bpm: %s\n", x.what() ); + return EXIT_FAILURE; + } +} diff --git a/src/cmd_headers.cpp b/src/cmd_headers.cpp new file mode 100644 index 0000000..ffa4948 --- /dev/null +++ b/src/cmd_headers.cpp @@ -0,0 +1,274 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "cmd_headers.hpp" +#include "options.hpp" +#include "message.hpp" +#include "error.hpp" +#include "fs.hpp" +#include +#include +#include +#include + +static void handle_option( std::string const & opt ) +{ + if( opt == "-v" ) + { + increase_message_level(); + } + else if( opt == "-q" ) + { + decrease_message_level(); + } + else + { + throw std::runtime_error( "invalid headers option: '" + opt + "'" ); + } +} + +void cmd_headers( char const * argv[] ) +{ + parse_options( argv, handle_option ); + + if( *argv ) + { + throw std::runtime_error( std::string( "unexpected argument '" ) + *argv + "'" ); + } + + cmd_headers(); +} + +static void removing( std::string const & path ) +{ + msg_printf( 2, "removing '%s'", path.c_str() ); +} + +static void rmerror( std::string const & path, int err ) +{ + throw_errno_error( path, "remove error", err ); +} + +static void build_dir_map2( std::string const & path, std::string const & suffix, std::map< std::string, std::vector< std::string > > & dirs ) +{ + // enumerate header directories in 'path' + 'suffix' + + std::vector< std::string > entries; + int r = fs_readdir( path + suffix, entries ); + + if( r != 0 ) + { + throw_errno_error( path + suffix, "read error", errno ); + } + + for( std::vector< std::string >::const_iterator i = entries.begin(); i != entries.end(); ++i ) + { + if( *i == "." || *i == ".." ) continue; + + std::string sx2 = suffix + "/" + *i; + + std::string p2 = path + sx2; + + if( !fs_is_dir( p2 ) ) continue; + + dirs[ sx2 ].push_back( p2 ); + + build_dir_map2( path, sx2, dirs ); + } +} + +static void build_dir_map( std::string const & path, std::map< std::string, std::vector< std::string > > & dirs ) +{ + // enumerate modules in 'path' + + std::vector< std::string > entries; + int r = fs_readdir( path, entries ); + + if( r != 0 ) + { + throw_errno_error( path, "read error", errno ); + } + + for( std::vector< std::string >::const_iterator i = entries.begin(); i != entries.end(); ++i ) + { + if( *i == "." || *i == ".." ) continue; + + std::string p2 = path + "/" + *i; + + if( fs_is_dir( p2 ) && fs_is_dir( p2 + "/include" ) ) + { + build_dir_map2( p2 + "/", "include", dirs ); + } + + if( fs_exists( p2 + "/sublibs" ) ) + { + // enumerate submodules + build_dir_map( p2, dirs ); + } + } +} + +static void link_directory( std::string const & dir, std::map< std::string, std::vector< std::string > > & dirs ); + +static void link_files( std::string const & path, std::string const & target, std::map< std::string, std::vector< std::string > > & dirs ) +{ + // msg_printf( 1, "linking files from '%s' into '%s'", path.c_str(), target.c_str() ); + + std::vector< std::string > entries; + int r = fs_readdir( path, entries ); + + if( r != 0 ) + { + throw_errno_error( path, "read error", errno ); + } + + for( std::vector< std::string >::const_iterator i = entries.begin(); i != entries.end(); ++i ) + { + if( *i == "." || *i == ".." ) continue; + + std::string p2 = path + "/" + *i; + std::string t2 = target + "/" + *i; + + if( fs_is_dir( p2 ) ) + { + link_directory( t2, dirs ); + } + else + { + // file + msg_printf( 1, "linking '%s' to '%s'", t2.c_str(), p2.c_str() ); + + if( fs_link_file( t2, p2 ) != 0 ) + { + throw_errno_error( t2, "link create error", errno ); + } + } + } +} + +static void remove_directory_from_list( std::string const & dir, std::map< std::string, std::vector< std::string > > & dirs ) +{ + for( std::map< std::string, std::vector< std::string > >::iterator i = dirs.find( dir ); i != dirs.end(); ) + { + std::string d2 = i->first; + + if( d2 == dir || d2.substr( 0, 1 + dir.size() ) == dir + "/" ) + { + dirs.erase( i++ ); + } + else + { + break; + } + } +} + +static void link_directory( std::string const & dir, std::string const & target ) +{ + msg_printf( 1, "linking '%s' to '%s'", dir.c_str(), target.c_str() ); + + if( fs_link_dir( dir, target ) != 0 ) + { + throw_errno_error( dir, "link create error", errno ); + } +} + +static void create_directory( std::string const & dir ) +{ + msg_printf( 1, "creating '%s'", dir.c_str() ); + + if( fs_mkdir( dir, 0755 ) != 0 ) + { + throw_errno_error( dir, "create error", errno ); + } +} + +static void link_directory( std::string const & dir, std::map< std::string, std::vector< std::string > > & dirs ) +{ + if( dirs.count( dir ) == 0 ) + { + // already processed + return; + } + + std::vector< std::string > const & d2 = dirs[ dir ]; + + assert( !d2.empty() ); + + if( d2.size() == 1 ) + { + link_directory( dir, d2.front() ); + } + else + { + create_directory( dir ); + + for( std::vector< std::string >::const_iterator i = d2.begin(); i != d2.end(); ++i ) + { + link_files( *i, dir, dirs ); + } + } + + remove_directory_from_list( dir, dirs ); +} + +static void remove_directory( std::string const & dir ) +{ + if( fs_exists( dir ) ) + { + fs_remove_all( dir, removing, rmerror ); + } +} + +static void touch_file( std::string const & path ) +{ + int fd = fs_creat( path, 0644 ); + + if( fd >= 0 ) + { + fs_close( fd ); + } +} + +void cmd_headers() +{ + msg_printf( 0, "recreating header links" ); + + msg_printf( 1, "removing old header links" ); + + remove_directory( "include" ); + remove_directory( "boost" ); + + if( !fs_exists( "libs" ) ) + { + return; + } + + std::map< std::string, std::vector< std::string > > dirs; + build_dir_map( "libs", dirs ); + + if( dirs.empty() ) + { + return; + } + + create_directory( "include" ); + + while( !dirs.empty() ) + { + std::string dir = dirs.begin()->first; + link_directory( dir, dirs ); + } + + touch_file( "include/.updated" ); + + if( fs_exists( "include/boost" ) ) + { + link_directory( "boost", "include/boost" ); + } +} diff --git a/src/cmd_headers.hpp b/src/cmd_headers.hpp new file mode 100644 index 0000000..95d3056 --- /dev/null +++ b/src/cmd_headers.hpp @@ -0,0 +1,15 @@ +#ifndef CMD_HEADERS_HPP_INCLUDED +#define CMD_HEADERS_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +void cmd_headers( char const * argv[] ); +void cmd_headers(); + +#endif // #ifndef CMD_HEADERS_HPP_INCLUDED diff --git a/src/cmd_index.cpp b/src/cmd_index.cpp new file mode 100644 index 0000000..2cc374e --- /dev/null +++ b/src/cmd_index.cpp @@ -0,0 +1,433 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "cmd_index.hpp" +#include "options.hpp" +#include "message.hpp" +#include "error.hpp" +#include "json.hpp" +#include "fs.hpp" +#include +#include +#include +#include +#include +#include + +static void handle_option( std::string const & opt ) +{ + if( opt == "-v" ) + { + increase_message_level(); + } + else if( opt == "-q" ) + { + decrease_message_level(); + } + else + { + throw std::runtime_error( "invalid index option: '" + opt + "'" ); + } +} + +void cmd_index( char const * argv[] ) +{ + parse_options( argv, handle_option ); + + if( *argv ) + { + throw std::runtime_error( std::string( "unexpected argument '" ) + *argv + "'" ); + } + + cmd_index(); +} + +typedef std::map< std::string, std::vector< std::string > > library; + +static std::string to_string( std::vector< std::string > const & v ) +{ + std::string r; + + for( std::size_t i = 0, n = v.size(); i < n; ++i ) + { + if( i != 0 ) + { + r += ", "; + } + + r += v[ i ]; + } + + return r; +} + +static std::string get_field( library const & lib, std::string const & name ) +{ + library::const_iterator i = lib.find( name ); + + if( i == lib.end() ) + { + return std::string(); + } + else + { + return to_string( i->second ); + } +} + +static void set_field( library & lib, std::string const & name, std::string const & value ) +{ + lib[ name ].resize( 1 ); + lib[ name ].front() = value; +} + +static void add_library( std::string const & path, std::map< std::string, library > & libraries, std::map< std::string, std::vector< std::string > > & categories ) +{ + std::string name = path + "/meta/libraries.json"; + + msg_printf( 2, "reading '%s'", name.c_str() ); + + std::vector< library > v; + + try + { + read_libraries_json( name, v ); + } + catch( std::exception const & x ) + { + msg_printf( -2, "%s", x.what() ); + } + + for( std::vector< library >::iterator i = v.begin(); i != v.end(); ++i ) + { + library & lib = *i; + + std::string key = get_field( lib, "key" ); + + if( key.empty() ) + { + key = path; + + int j = i - v.begin(); + + if( j != 0 ) + { + char buffer[ 32 ]; + sprintf( buffer, ".%d", j + 1 ); + + key += buffer; + } + + msg_printf( -1, "'%s': library has no key, assuming '%s'", name.c_str(), key.c_str() ); + + set_field( lib, "key", key ); + } + + std::string name = get_field( lib, "name" ); + + if( name.empty() ) + { + msg_printf( -1, "'%s': library '%s' has no name", name.c_str(), key.c_str() ); + } + + set_field( lib, "path", path ); + + std::vector< std::string > const & cv = lib[ "category" ]; + + libraries[ key ] = lib; + + for( std::vector< std::string >::const_iterator j = cv.begin(); j != cv.end(); ++j ) + { + categories[ *j ].push_back( key ); + } + } +} + +static void build_library_map( std::string const & path, std::map< std::string, library > & libraries, std::map< std::string, std::vector< std::string > > & categories ) +{ + // enumerate modules in 'path' + + std::vector< std::string > entries; + int r = fs_readdir( path, entries ); + + if( r != 0 ) + { + throw_errno_error( path, "read error", errno ); + } + + for( std::vector< std::string >::const_iterator i = entries.begin(); i != entries.end(); ++i ) + { + if( *i == "." || *i == ".." ) continue; + + std::string p2 = path + "/" + *i; + + if( fs_is_dir( p2 ) && fs_exists( p2 + "/meta/libraries.json" ) ) + { + add_library( p2, libraries, categories ); + } + + if( fs_exists( p2 + "/sublibs" ) ) + { + // enumerate submodules + build_library_map( p2, libraries, categories ); + } + } +} + +static void write_index( std::string const & name, std::map< std::string, library > const & libraries, std::map< std::string, std::vector< std::string > > const & categories ); + +void cmd_index() +{ + msg_printf( 0, "recreating index" ); + + std::map< std::string, library > libraries; + std::map< std::string, std::vector< std::string > > categories; + + build_library_map( "libs", libraries, categories ); + + write_index( "index.html", libraries, categories ); +} + +static char const * file_header = + +"\n" +"\n" +"\n" +"\n" +"\n" +"Installed Boost C++ Libraries\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"\n" +"
\n" +"\n" +"
\n" +"
boost
\n" +"
C++ LIBRARIES
\n" +"
\n\n"; + +static void write_category_link( std::ostream & os, std::string const & name, std::string const & id ) +{ + os << "\n"; +} + +static void write_header( std::ostream & os, std::map< std::string, std::pair< std::string, std::vector< std::string > > > const & categories ) +{ + os << "
\n" + "\n" + "\n" + "
 
\n"; + + for( std::map< std::string, std::pair< std::string, std::vector< std::string > > >::const_iterator i = categories.begin(); i != categories.end(); ++i ) + { + write_category_link( os, i->first, i->second.first ); + } + + os << "
\n\n"; +} + +static std::string category_name( std::string const & cat ); + +static void write_alpha_library( std::ostream & os, library const & lib ) +{ + os << "

" << get_field( lib, "name" ) << "

\n\n" + + "

" << get_field( lib, "description" ) << "

\n\n" + + "
\n"; + + std::string authors = get_field( lib, "authors" ); + + if( !authors.empty() ) + { + os << "
Author(s):
" << authors << "
\n"; + } + + { + library::const_iterator i = lib.find( "category" ); + + if( i != lib.end() ) + { + os << "
Category:
"; + + int k = 0; + + for( std::vector< std::string>::const_iterator j = i->second.begin(); j != i->second.end(); ++j, ++k ) + { + if( k != 0 ) + { + os << " • "; + } + + os << "" << category_name( *j ) << ""; + } + + os << "
\n"; + } + } + + os << "
\n\n"; +} + +static void write_alpha_section( std::ostream & os, std::map< std::string, library > const & libraries ) +{ + os << "
\n" + "\n" + "

Libraries Listed Alphabetically

\n\n"; + + // re-sort by name + std::map< std::string, library > lib2; + + for( std::map< std::string, library >::const_iterator i = libraries.begin(); i != libraries.end(); ++i ) + { + std::string name = get_field( i->second, "name" ); + + if( name.empty() ) continue; + + lib2[ name ] = i->second; + } + + for( std::map< std::string, library >::const_iterator i = lib2.begin(); i != lib2.end(); ++i ) + { + write_alpha_library( os, i->second ); + } + + os << "
\n\n"; +} + +static void write_cat_library( std::ostream & os, library const & lib ) +{ + os << "

" << get_field( lib, "name" ) << "

\n\n" + + "

" << get_field( lib, "description" ) << "

\n\n"; +} + +static void write_category( std::ostream & os, std::map< std::string, library > const & libraries, std::string const & name, std::string const & id, std::vector< std::string > const & libs ) +{ + os << "

" << name << "

\n\n" + "
\n\n"; + + for( std::vector< std::string >::const_iterator i = libs.begin(); i != libs.end(); ++i ) + { + std::map< std::string, library >::const_iterator j = libraries.find( *i ); + + if( j != libraries.end() ) + { + write_cat_library( os, j->second ); + } + } + + os << "
\n\n"; +} + +static void write_cat_section( std::ostream & os, std::map< std::string, library > const & libraries, std::map< std::string, std::pair< std::string, std::vector< std::string > > > const & categories ) +{ + os << "
\n" + "\n" + "

Libraries Listed by Category

\n\n"; + + for( std::map< std::string, std::pair< std::string, std::vector< std::string > > >::const_iterator i = categories.begin(); i != categories.end(); ++i ) + { + write_category( os, libraries, i->first, i->second.first, i->second.second ); + } + + os << "
\n\n"; +} + +static char const * file_footer = + +"
\n" +"\n" +"\n"; + +static std::string category_name( std::string const & cat ) +{ + static char const * table[][ 2 ] = + { + { "String", "String and Text Processing" }, + { "Function-objects", "Function Objects and Higher-Order Programming" }, + { "Generic", "Generic Programming" }, + { "Metaprogramming", "Template Metaprogramming" }, + { "Preprocessor", "Preprocessor Metaprogramming" }, + { "Concurrent", "Concurrent Programming" }, + { "Math", "Math and Numerics" }, + { "Correctness", "Correctness and Testing" }, + { "Data", "Data Structures" }, + { "Domain", "Domain Specific" }, + { "Image-processing", "Image Processing" }, + { "IO", "Input/Output" }, + { "Inter-language", "Inter-Language Support" }, + { "Emulation", "Emulation of Language Features" }, + { "Patterns", "Patterns and Idioms" }, + { "Programming", "Programming Interfaces" }, + { "State", "State Machines" }, + { "workarounds", "Broken Compiler Workarounds" }, + }; + + for( int i = 0, n = sizeof( table ) / sizeof( table[0] ); i < n; ++i ) + { + if( cat == table[ i ][ 0 ] ) return table[ i ][ 1 ]; + } + + return cat; +} + +static void write_index( std::string const & name, std::map< std::string, library > const & libraries, std::map< std::string, std::vector< std::string > > const & categories ) +{ + msg_printf( 2, "writing '%s'", name.c_str() ); + + std::ofstream os( name.c_str() ); + + if( !os ) + { + throw_errno_error( name, "open error", errno ); + } + + os << file_header; + + // re-sort categories by name + + std::map< std::string, std::pair< std::string, std::vector< std::string > > > cats; + + for( std::map< std::string, std::vector< std::string > >::const_iterator i = categories.begin(); i != categories.end(); ++i ) + { + cats[ category_name( i->first ) ] = *i; + } + + write_header( os, cats ); + write_alpha_section( os, libraries ); + write_cat_section( os, libraries, cats ); + + os << file_footer; +} diff --git a/src/cmd_index.hpp b/src/cmd_index.hpp new file mode 100644 index 0000000..2895c31 --- /dev/null +++ b/src/cmd_index.hpp @@ -0,0 +1,15 @@ +#ifndef CMD_INDEX_HPP_INCLUDED +#define CMD_INDEX_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +void cmd_index( char const * argv[] ); +void cmd_index(); + +#endif // #ifndef CMD_INDEX_HPP_INCLUDED diff --git a/src/cmd_install.cpp b/src/cmd_install.cpp new file mode 100644 index 0000000..5b6a789 --- /dev/null +++ b/src/cmd_install.cpp @@ -0,0 +1,373 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "options.hpp" +#include "dependencies.hpp" +#include "message.hpp" +#include "package_path.hpp" +#include "cmd_headers.hpp" +#include "cmd_index.hpp" + +#include "lzma_reader.hpp" +#include "http_reader.hpp" +#include "tar.hpp" + +#include "error.hpp" +#include "fs.hpp" + +#include +#include +#include +#include +#include +#include + + +static bool s_opt_n = false; +static bool s_opt_d = true; +static bool s_opt_k = false; +static bool s_opt_a = false; +static bool s_opt_i = false; +static bool s_opt_p = false; + +static void handle_option( std::string const & opt ) +{ + if( opt == "-n" ) + { + s_opt_n = true; + } + else if( opt == "+d" ) + { + s_opt_d = false; + } + else if( opt == "-d" ) + { + s_opt_d = true; + } + else if( opt == "-k" ) + { + s_opt_k = true; + } + else if( opt == "-a" ) + { + s_opt_a = true; + } + else if( opt == "-i" ) + { + s_opt_i = true; + } + else if( opt == "-p" ) + { + s_opt_p = true; + } + else if( opt == "-v" ) + { + increase_message_level(); + } + else if( opt == "-q" ) + { + decrease_message_level(); + } + else + { + throw std::runtime_error( "invalid install option: '" + opt + "'" ); + } +} + +static void removing( std::string const & path ) +{ + msg_printf( 2, "removing '%s'", path.c_str() ); +} + +static void rmerror( std::string const & path, int err ) +{ + msg_printf( 1, "'%s': remove error: %s", path.c_str(), std::strerror( err ) ); +} + +static void remove_partial_installation( std::string const & module, std::string const & path, std::set< std::string > const & files ) +{ + if( fs_exists( path ) ) + { + msg_printf( 1, "removing partial installation of module '%s'", module.c_str() ); + fs_remove_all( path, removing, rmerror ); + } + + for( std::set< std::string >::const_iterator i = files.begin(); i != files.end(); ++i ) + { + if( fs_exists( *i ) ) + { + removing( *i ); + + int r = std::remove( i->c_str() ); + + if( r != 0 ) + { + rmerror( *i, errno ); + } + } + } +} + +static void touch_file( std::string const & path ) +{ + int fd = fs_creat( path, 0644 ); + + if( fd >= 0 ) + { + fs_close( fd ); + } +} + +static std::string module_package( std::string const & module ) +{ + std::size_t i = module.find( '~' ); + + std::string package = module.substr( 0, i ); + + return package; +} + +static void install_module( std::string const & package_path, std::string const & module, std::set< std::string > & installed, std::time_t & mtime ) +{ + std::string package = module_package( module ); + + std::string path = "libs/" + package; + + std::set< std::string > whitelist; + + if( module == "build" ) + { + if( !fs_exists( "tools" ) ) + { + fs_mkdir( "tools/", 0755 ); + } + + path = "tools/" + module; + + whitelist.insert( "b2.exe" ); + whitelist.insert( "boost-build.jam" ); + whitelist.insert( "boostcpp.jam" ); + whitelist.insert( "Jamroot" ); + whitelist.insert( "libs/Jamfile.v2" ); + } + + std::string marker = path + "/.installed"; + + if( !fs_exists( marker ) ) + { + if( s_opt_n ) + { + msg_printf( 0, "would have installed module '%s'", module.c_str() ); + } + else + { + remove_partial_installation( module, path, whitelist ); + + msg_printf( 0, "installing module '%s'", module.c_str() ); + + std::string tar_path = package_path + package + ".tar.lzma"; + + http_reader r1( tar_path ); + lzma_reader r2( &r1 ); + + try + { + tar_extract( &r2, path + '/', whitelist ); + touch_file( marker ); + } + catch( std::exception const & ) + { + if( !s_opt_k ) + { + remove_partial_installation( module, path, whitelist ); + } + + throw; + } + } + + installed.insert( module ); + } + else + { + msg_printf( 1, "module '%s' is already installed", module.c_str() ); + } + + mtime = std::max( mtime, fs_mtime( marker ) ); +} + +static void install_boost_build( std::string const & package_path, std::set< std::string > & installed ) +{ + std::time_t mtime = 0; + install_module( package_path, "build", installed, mtime ); +} + +void cmd_install( char const * argv[] ) +{ + parse_options( argv, handle_option ); + + if( s_opt_a + s_opt_i + s_opt_p > 1 ) + { + throw std::runtime_error( "install options -a, -i and -p are mutually exclusive" ); + } + + if( ( s_opt_a || s_opt_i || s_opt_p ) && *argv != 0 ) + { + throw std::runtime_error( "install options -a, -i and -p are incompatible with a module list" ); + } + + std::string package_path = get_package_path(); + + std::map< std::string, std::vector< std::string > > deps; + std::set< std::string > buildable; + + retrieve_dependencies( deps, buildable ); + + if( !fs_exists( "libs" ) ) + { + fs_mkdir( "libs/", 0755 ); + } + + std::vector< std::string > modules; + + if( s_opt_a ) + { + for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i ) + { + modules.push_back( i->first ); + } + } + else if( s_opt_i || s_opt_p ) + { + for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i ) + { + std::string module = i->first; + + std::string package = module_package( module ); + + std::string path( module ); + std::replace( path.begin(), path.end(), '~', '/' ); + + if( fs_exists( "libs/" + path ) && s_opt_p != fs_exists( "libs/" + package + "/.installed" ) ) + { + modules.push_back( i->first ); + } + } + } + else + { + while( *argv ) + { + modules.push_back( *argv ); + ++argv; + } + } + + std::set< std::string > installed; + std::set< std::string > need_build; + + std::time_t mtime = 0; + + { + std::size_t i = 0; + + while( i < modules.size() ) + { + std::string module = modules[ i++ ]; + + if( deps.count( module ) == 0 ) + { + msg_printf( -1, "module '%s' does not exist", module.c_str() ); + } + else + { + install_module( package_path, module, installed, mtime ); + + if( s_opt_d ) + { + std::vector< std::string > mdeps = deps[ module ]; + + for( std::size_t j = 0, n = mdeps.size(); j < n; ++j ) + { + std::string const & m2 = mdeps[ j ]; + + if( std::find( modules.begin(), modules.end(), m2 ) == modules.end() ) + { + modules.push_back( m2 ); + } + } + } + } + } + } + + std::set< std::string > installed2; + + { + bool need_build = false; + + for( std::vector< std::string >::const_iterator i = modules.begin(); i != modules.end(); ++i ) + { + if( buildable.count( *i ) ) + { + need_build = true; + } + } + + if( s_opt_d && need_build ) + { + install_boost_build( package_path, installed2 ); + } + } + + if( installed.empty() && installed2.empty() ) + { + msg_printf( 0, "nothing to install, everything is already in place" ); + } + + if( !s_opt_n ) + { + std::set< std::string > need_build; + + for( std::set< std::string >::const_iterator i = installed.begin(); i != installed.end(); ++i ) + { + if( buildable.count( *i ) ) + { + need_build.insert( *i ); + } + } + + if( !need_build.empty() ) + { + std::string m2; + + for( std::set< std::string >::const_iterator i = need_build.begin(); i != need_build.end(); ++i ) + { + m2 += " "; + m2 += *i; + } + + msg_printf( 0, "the following libraries need to be built:\n %s", m2.c_str() ); +#if defined( _WIN32 ) + msg_printf( 0, "(use b2 to build)" ); +#else + msg_printf( 0, "(use ./b2 to build)" ); +#endif + } + } + + if( !s_opt_n && mtime >= fs_mtime( "include/.updated" ) ) // headers out of date + { + cmd_headers(); + } + + if( !s_opt_n && mtime >= fs_mtime( "index.html" ) ) // index out of date + { + cmd_index(); + } +} diff --git a/src/cmd_install.hpp b/src/cmd_install.hpp new file mode 100644 index 0000000..a312e8c --- /dev/null +++ b/src/cmd_install.hpp @@ -0,0 +1,14 @@ +#ifndef CMD_INSTALL_HPP_INCLUDED +#define CMD_INSTALL_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +void cmd_install( char const * argv[] ); + +#endif // #ifndef CMD_INSTALL_HPP_INCLUDED diff --git a/src/cmd_list.cpp b/src/cmd_list.cpp new file mode 100644 index 0000000..15dd2af --- /dev/null +++ b/src/cmd_list.cpp @@ -0,0 +1,137 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "cmd_list.hpp" +#include "options.hpp" +#include "dependencies.hpp" +#include "message.hpp" +#include "fs.hpp" +#include +#include +#include + +static bool s_opt_a = false; +static bool s_opt_i = false; +static bool s_opt_p = false; +static bool s_opt_b = false; + +static void handle_option( std::string const & opt ) +{ + if( opt == "-a" ) + { + s_opt_a = true; + } + else if( opt == "-i" ) + { + s_opt_i = true; + } + else if( opt == "-p" ) + { + s_opt_p = true; + } + else if( opt == "-b" ) + { + s_opt_b = true; + } + else if( opt == "-v" ) + { + increase_message_level(); + } + else if( opt == "-q" ) + { + decrease_message_level(); + } + else + { + throw std::runtime_error( "invalid list option: '" + opt + "'" ); + } +} + +static std::string module_package( std::string const & module ) +{ + std::size_t i = module.find( '~' ); + + std::string package = module.substr( 0, i ); + + return package; +} + +void cmd_list( char const * argv[] ) +{ + parse_options( argv, handle_option ); + + std::string prefix; + + if( *argv ) + { + prefix = *argv++; + } + + if( *argv ) + { + throw std::runtime_error( std::string( "unexpected list argument '" ) + *argv + "'" ); + } + + if( s_opt_a + s_opt_i + s_opt_p > 1 ) + { + throw std::runtime_error( "list options -a, -i and -p are mutually exclusive" ); + } + + if( s_opt_p && s_opt_b ) + { + throw std::runtime_error( "list options -p and -b are incompatible" ); + } + + if( !s_opt_a && !s_opt_i && !s_opt_p ) + { + s_opt_a = !s_opt_b; + s_opt_i = s_opt_b; + } + + std::map< std::string, std::vector< std::string > > deps; + std::set< std::string > buildable; + + retrieve_dependencies( deps, buildable ); + + if( s_opt_a ) + { + for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i ) + { + std::string module = i->first; + + if( module.substr( 0, prefix.size() ) != prefix ) continue; + + if( !s_opt_b || buildable.count( module ) ) + { + printf( "%s\n", module.c_str() ); + } + } + } + else + { + for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i ) + { + std::string module = i->first; + + if( module.substr( 0, prefix.size() ) != prefix ) continue; + + std::string package = module_package( module ); + + std::string path( module ); + std::replace( path.begin(), path.end(), '~', '/' ); + + if( fs_exists( "libs/" + path ) && s_opt_p != fs_exists( "libs/" + package + "/.installed" ) ) + { + if( !s_opt_b || buildable.count( module ) ) + { + printf( "%s\n", module.c_str() ); + } + } + } + } +} diff --git a/src/cmd_list.hpp b/src/cmd_list.hpp new file mode 100644 index 0000000..afa2b4f --- /dev/null +++ b/src/cmd_list.hpp @@ -0,0 +1,14 @@ +#ifndef CMD_LIST_HPP_INCLUDED +#define CMD_LIST_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +void cmd_list( char const * argv[] ); + +#endif // #ifndef CMD_LIST_HPP_INCLUDED diff --git a/src/cmd_remove.cpp b/src/cmd_remove.cpp new file mode 100644 index 0000000..4e62e18 --- /dev/null +++ b/src/cmd_remove.cpp @@ -0,0 +1,308 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "cmd_remove.hpp" +#include "cmd_headers.hpp" +#include "cmd_index.hpp" +#include "options.hpp" +#include "dependencies.hpp" +#include "message.hpp" +#include "fs.hpp" +#include +#include +#include +#include +#include + +static bool s_opt_n = false; +static bool s_opt_f = false; +static bool s_opt_d = false; +static bool s_opt_a = false; +static bool s_opt_p = false; + +static void handle_option( std::string const & opt ) +{ + if( opt == "-n" ) + { + s_opt_n = true; + } + else if( opt == "-f" ) + { + s_opt_f = true; + } + else if( opt == "-d" ) + { + s_opt_d = true; + } + else if( opt == "-a" ) + { + s_opt_a = true; + } + else if( opt == "-p" ) + { + s_opt_p = true; + } + else if( opt == "-v" ) + { + increase_message_level(); + } + else if( opt == "-q" ) + { + decrease_message_level(); + } + else + { + throw std::runtime_error( "invalid remove option: '" + opt + "'" ); + } +} + +static std::string module_package( std::string const & module ) +{ + std::size_t i = module.find( '~' ); + + std::string package = module.substr( 0, i ); + + return package; +} + +static void get_package_dependents( std::string const & package, std::map< std::string, std::vector< std::string > > const & deps, std::vector< std::string > const & packages, std::set< std::string > & deps2 ) +{ + deps2.clear(); + + if( !fs_exists( "libs/" + package + "/.installed" ) ) + { + // partially installed packages have no dependents + return; + } + + for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i ) + { + std::string m1 = i->first; + std::string p1 = module_package( m1 ); + + for( std::vector< std::string >::const_iterator j = i->second.begin(); j != i->second.end(); ++j ) + { + std::string m2 = *j; + std::string p2 = module_package( m2 ); + + // p1 -> p2 + + if( p2 == package && fs_exists( "libs/" + p1 + "/.installed" ) && std::find( packages.begin(), packages.end(), p1 ) == packages.end() ) + { + deps2.insert( p1 ); + } + } + } +} + +static void removing( std::string const & path ) +{ + msg_printf( 2, "removing '%s'", path.c_str() ); +} + +static void rmerror( std::string const & path, int err ) +{ + msg_printf( 1, "'%s': remove error: %s", path.c_str(), std::strerror( err ) ); +} + +static void remove_package( std::string const & package, int & removed ) +{ + std::string path = "libs/" + package; + + std::set< std::string > files; + + if( package == "build" ) + { + path = "tools/" + package; + + files.insert( "b2.exe" ); + files.insert( "boost-build.jam" ); + files.insert( "boostcpp.jam" ); + files.insert( "Jamroot" ); + files.insert( "libs/Jamfile.v2" ); + } + + if( !fs_exists( path ) ) + { + msg_printf( 1, "package '%s' has already been removed", package.c_str() ); + return; + } + + if( s_opt_n ) + { + msg_printf( 0, "would have removed package '%s'", package.c_str() ); + return; + } + + msg_printf( 0, "removing package '%s'", package.c_str() ); + + std::string marker = path + "/.installed"; + + std::remove( marker.c_str() ); + + fs_remove_all( path, removing, rmerror ); + + ++removed; + + for( std::set< std::string >::const_iterator i = files.begin(); i != files.end(); ++i ) + { + if( fs_exists( *i ) ) + { + removing( *i ); + + int r = std::remove( i->c_str() ); + + if( r != 0 ) + { + rmerror( *i, errno ); + } + } + } +} + +void cmd_remove( char const * argv[] ) +{ + parse_options( argv, handle_option ); + + if( s_opt_a && s_opt_p ) + { + throw std::runtime_error( "remove options -a and -p are mutually exclusive" ); + } + + if( ( s_opt_a || s_opt_p ) && *argv != 0 ) + { + throw std::runtime_error( "remove options -a and -p are incompatible with a package list" ); + } + + if( s_opt_d && !s_opt_f && !s_opt_n ) + { + throw std::runtime_error( "remove option -d requires -f" ); + } + + std::map< std::string, std::vector< std::string > > deps; + std::set< std::string > buildable; + + retrieve_dependencies( deps, buildable ); + + std::vector< std::string > packages; + + if( s_opt_a ) + { + if( !s_opt_f && !s_opt_n ) + { + throw std::runtime_error( "remove option -a requires -f" ); + } + + for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i ) + { + std::string module = i->first; + std::string package = module_package( module ); + + if( packages.empty() || packages.back() != package ) + { + packages.push_back( package ); + } + } + + packages.push_back( "build" ); + } + else if( s_opt_p ) + { + for( std::map< std::string, std::vector< std::string > >::const_iterator i = deps.begin(); i != deps.end(); ++i ) + { + std::string module = i->first; + std::string package = module_package( module ); + + if( fs_exists( "libs/" + package ) && !fs_exists( "libs/" + package + "/.installed" ) ) + { + if( packages.empty() || packages.back() != package ) + { + packages.push_back( package ); + } + } + + if( fs_exists( "tools/build" ) && !fs_exists( "tools/build/.installed" ) ) + { + packages.push_back( "build" ); + } + } + } + else + { + while( *argv ) + { + packages.push_back( *argv ); + ++argv; + } + } + + if( !s_opt_f && !( s_opt_n && s_opt_d ) ) + { + for( std::vector< std::string >::const_iterator i = packages.begin(); i != packages.end(); ++i ) + { + std::string package = *i; + + std::set< std::string > deps2; + + get_package_dependents( package, deps, packages, deps2 ); + + if( !s_opt_f && !deps2.empty() ) + { + std::string list; + + for( std::set< std::string >::const_iterator i = deps2.begin(); i != deps2.end(); ++i ) + { + list += " "; + list += *i; + } + + msg_printf( -2, "package '%s' cannot be removed due to dependents:\n %s", package.c_str(), list.c_str() ); + + return; + } + } + } + + int removed = 0; + + { + std::size_t i = 0; + + while( i < packages.size() ) + { + std::string package = packages[ i++ ]; + + std::set< std::string > deps2; + + get_package_dependents( package, deps, packages, deps2 ); + + remove_package( package, removed ); + + if( s_opt_d ) + { + for( std::set< std::string >::const_iterator i = deps2.begin(); i != deps2.end(); ++i ) + { + if( std::find( packages.begin(), packages.end(), *i ) == packages.end() ) + { + packages.push_back( *i ); + } + } + } + } + } + + if( removed ) + { + cmd_headers(); + } + + if( removed ) + { + cmd_index(); + } +} diff --git a/src/cmd_remove.hpp b/src/cmd_remove.hpp new file mode 100644 index 0000000..2c2a796 --- /dev/null +++ b/src/cmd_remove.hpp @@ -0,0 +1,14 @@ +#ifndef CMD_REMOVE_HPP_INCLUDED +#define CMD_REMOVE_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +void cmd_remove( char const * argv[] ); + +#endif // #ifndef CMD_REMOVE_HPP_INCLUDED diff --git a/src/config.cpp b/src/config.cpp new file mode 100644 index 0000000..255d3f9 --- /dev/null +++ b/src/config.cpp @@ -0,0 +1,62 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "config.hpp" +#include "error.hpp" +#include "string.hpp" +#include +#include +#include +#include +#include + +static std::map< std::string, std::string > s_config; + +void config_read_file( std::string const & fn ) +{ + std::ifstream is( fn.c_str() ); + + if( !is ) + { + throw_errno_error( fn, "open error", errno ); + } + + std::string line; + + while( std::getline( is, line ) ) + { + if( line.empty() ) continue; + + if( !std::isalpha( static_cast< unsigned char >( line[0] ) ) ) continue; + + remove_trailing( line, '\r' ); + + std::size_t i = line.find( '=' ); + + if( i == 0 || i == std::string::npos ) continue; + + std::string key = line.substr( 0, i ); + std::string value = line.substr( i + 1 ); + + s_config[ key ] = value; + } +} + +std::string config_get_option( std::string const & name ) +{ + std::map< std::string, std::string >::const_iterator i = s_config.find( name ); + + if( i == s_config.end() ) + { + return std::string(); + } + else + { + return i->second; + } +} diff --git a/src/config.hpp b/src/config.hpp new file mode 100644 index 0000000..0f8ee67 --- /dev/null +++ b/src/config.hpp @@ -0,0 +1,17 @@ +#ifndef CONFIG_HPP_INCLUDED +#define CONFIG_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include + +void config_read_file( std::string const & fn ); +std::string config_get_option( std::string const & name ); + +#endif // #ifndef CONFIG_HPP_INCLUDED diff --git a/src/dependencies.cpp b/src/dependencies.cpp new file mode 100644 index 0000000..029f636 --- /dev/null +++ b/src/dependencies.cpp @@ -0,0 +1,104 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "dependencies.hpp" +#include "package_path.hpp" +#include "lzma_reader.hpp" +#include "http_reader.hpp" +#include "error.hpp" +#include "string.hpp" +#include +#include + +static void parse_module_description( std::string const & name, std::string const & line, std::map< std::string, std::vector< std::string > > & deps ) +{ + std::istringstream is( line ); + + std::string module, arrow; + + if( is >> module >> arrow && arrow == "->" ) + { + deps[ module ]; + + std::string dep; + + while( is >> dep ) + { + deps[ module ].push_back( dep ); + } + } + else + { + throw_error( name, "invalid line: '" + line + "'" ); + } +} + +static std::string read_text_file( std::string const & url ) +{ + http_reader r1( url ); + lzma_reader r2( &r1 ); + + std::string data; + + for( ;; ) + { + int const N = 4096; + + char buffer[ N ]; + + std::size_t r = r2.read( buffer, N ); + + data.append( buffer, r ); + + if( r < N ) break; + } + + return data; +} + +static void retrieve_dependencies( std::string const & package_path, std::map< std::string, std::vector< std::string > > & deps ) +{ + std::string url = package_path + "dependencies.txt.lzma"; + + std::string data = read_text_file( url ); + + std::istringstream is( data ); + + std::string line; + + while( std::getline( is, line ) ) + { + remove_trailing( line, '\r' ); + parse_module_description( url, line, deps ); + } +} + +static void retrieve_buildable( std::string const & package_path, std::set< std::string > & buildable ) +{ + std::string data = read_text_file( package_path + "buildable.txt.lzma" ); + + std::istringstream is( data ); + + std::string module; + + while( is >> module ) + { + buildable.insert( module ); + } +} + +void retrieve_dependencies( std::map< std::string, std::vector< std::string > > & deps, std::set< std::string > & buildable ) +{ + deps.clear(); + buildable.clear(); + + std::string package_path = get_package_path(); + + retrieve_dependencies( package_path, deps ); + retrieve_buildable( package_path, buildable ); +} diff --git a/src/dependencies.hpp b/src/dependencies.hpp new file mode 100644 index 0000000..ef2a90d --- /dev/null +++ b/src/dependencies.hpp @@ -0,0 +1,19 @@ +#ifndef DEPENDENCIES_HPP_INCLUDED +#define DEPENDENCIES_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include +#include +#include +#include + +void retrieve_dependencies( std::map< std::string, std::vector< std::string > > & deps, std::set< std::string > & buildable ); + +#endif // #ifndef DEPENDENCIES_HPP_INCLUDED diff --git a/src/error.cpp b/src/error.cpp new file mode 100644 index 0000000..0d49d45 --- /dev/null +++ b/src/error.cpp @@ -0,0 +1,77 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "error.hpp" +#include + +error::error( std::string const & name, std::string const & reason ): name_( name ), reason_( reason ), what_( "'" + name_ + "': " + reason ) +{ +} + +error::~error() throw() +{ +} + +std::string error::name() const +{ + return name_; +} + +std::string error::reason() const +{ + return reason_; +} + +char const * error::what() const throw() +{ + return what_.c_str(); +} + +void throw_error( std::string const & name, std::string const & reason ) +{ + throw error( name, reason ); +} + +void throw_errno_error( std::string const & name, std::string const & reason, int err ) +{ + if( err == 0 ) + { + throw_error( name, reason ); + } + else + { + throw_error( name, reason + ": " + std::strerror( err ) ); + } +} + +#if defined( _WIN32 ) + +#include + +void throw_socket_error( std::string const & name, std::string const & reason, int err ) +{ + char buffer[ 1024 ]; + + if( err == 0 || FormatMessageA( FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, 0, err, 0, buffer, sizeof( buffer ), 0 ) == 0 ) + { + throw_error( name, reason ); + } + else + { + throw_error( name, reason + ": " + buffer ); + } +} + +#else + +void throw_socket_error( std::string const & name, std::string const & reason, int err ) +{ + throw_errno_error( name, reason, err ); +} + +#endif diff --git a/src/error.hpp b/src/error.hpp new file mode 100644 index 0000000..a52dc5f --- /dev/null +++ b/src/error.hpp @@ -0,0 +1,39 @@ +#ifndef ERROR_HPP_INCLUDED +#define ERROR_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include +#include + +class error: public std::exception +{ +private: + + std::string name_; + std::string reason_; + + std::string what_; + +public: + + error( std::string const & name, std::string const & reason ); + ~error() throw(); + + std::string name() const; + std::string reason() const; + + char const * what() const throw(); +}; + +void throw_error( std::string const & name, std::string const & reason ); +void throw_errno_error( std::string const & name, std::string const & reason, int err ); +void throw_socket_error( std::string const & name, std::string const & reason, int err ); + +#endif // #ifndef ERROR_HPP_INCLUDED diff --git a/src/file_reader.cpp b/src/file_reader.cpp new file mode 100644 index 0000000..f3a6733 --- /dev/null +++ b/src/file_reader.cpp @@ -0,0 +1,47 @@ +// +// Copyright 2014 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "file_reader.hpp" +#include "error.hpp" +#include + +file_reader::file_reader( std::FILE * f, std::string const & name ): f_( f ), name_( name ) +{ +} + +file_reader::file_reader( std::string const & fn ): name_( fn ) +{ + f_ = std::fopen( fn.c_str(), "rb" ); + + if( f_ == 0 ) + { + throw_errno_error( fn, "open error", errno ); + } +} + +file_reader::~file_reader() +{ + std::fclose( f_ ); +} + +std::string file_reader::name() const +{ + return name_; +} + +std::size_t file_reader::read( void * p, std::size_t n ) +{ + std::size_t r = std::fread( p, 1, n, f_ ); + + if( r == 0 && std::ferror( f_ ) ) + { + throw_errno_error( name(), "read error", errno ); + } + + return r; +} diff --git a/src/file_reader.hpp b/src/file_reader.hpp new file mode 100644 index 0000000..52b4857 --- /dev/null +++ b/src/file_reader.hpp @@ -0,0 +1,38 @@ +#ifndef FILE_READER_HPP_INCLUDED +#define FILE_READER_HPP_INCLUDED + +// +// Copyright 2014 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "basic_reader.hpp" +#include + +class file_reader: public basic_reader +{ +private: + + std::FILE * f_; + std::string name_; + +private: + + file_reader( file_reader const & ); + file_reader& operator=( file_reader const & ); + +public: + + explicit file_reader( std::FILE * f, std::string const & n ); + explicit file_reader( std::string const & fn ); + + ~file_reader(); + + virtual std::string name() const; + virtual std::size_t read( void * p, std::size_t n ); +}; + +#endif // #ifndef FILE_READER_HPP_INCLUDED diff --git a/src/fs.cpp b/src/fs.cpp new file mode 100644 index 0000000..567e035 --- /dev/null +++ b/src/fs.cpp @@ -0,0 +1,792 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "fs.hpp" +#include +#include +#include + +std::time_t fs_mtime( std::string const & path ) +{ + int mode; + std::time_t mtime, ctime, atime; + + if( fs_stat( path, mode, mtime, ctime, atime ) == 0 ) + { + return mtime; + } + else + { + return -1; + } +} + +#if defined( _WIN32 ) + +#define PATH_SEPARATOR '\\' + +static bool is_absolute( std::string const & path ) +{ + return path[ 0 ] == '\\' || path[ 0 ] == '/' || path.size() >= 2 && path[ 1 ] == ':'; +} + +static std::size_t find_next_separator( std::string const & path, std::size_t offset ) +{ + return path.find_first_of( "/\\", offset ); +} + +#else + +#define PATH_SEPARATOR '/' + +static bool is_absolute( std::string const & path ) +{ + return path[ 0 ] == '/'; +} + +static std::size_t find_next_separator( std::string const & path, std::size_t offset ) +{ + return path.find( '/', offset ); +} + +#endif // _WIN32 + +static int path_element_depth( std::string const & element ) +{ + if( element == "" || element == "." ) + { + return 0; + } + else if( element == ".." ) + { + return -1; + } + else + { + return +1; + } +} + +static int path_depth( std::string const & path ) +{ + int l = 0; + + std::size_t i = 0; + + for( ;; ) + { + std::size_t j = find_next_separator( path, i ); + + if( j == std::string::npos ) + { + l += path_element_depth( path.substr( i ) ); + break; + } + + l += path_element_depth( path.substr( i, j - i ) ); + + i = j + 1; + } + + return l; +} + +static bool make_relative( std::string const & link, std::string & target ) +{ + if( target.empty() ) + { + return false; + } + + if( is_absolute( target ) ) + { + // absolute pathname + return true; + } + + if( link.empty() ) + { + return false; + } + + if( is_absolute( link ) ) + { + // absolute link, relative target, not supported + return true; + } + + int l = path_depth( link ); + + if( l <= 0 ) + { + return false; + } + + while( --l > 0 ) + { + target = PATH_SEPARATOR + target; + target = ".." + target; + } + + return true; +} + +#if defined( _WIN32 ) + +#define _WIN32_WINNT 0x501 + +#include +#include +#include +#include +#include +#include +#include + +int fs_creat( std::string const & path, int mode ) +{ + return _open( path.c_str(), _O_CREAT | _O_TRUNC | _O_WRONLY | _O_BINARY, mode & 0600 ); +} + +int fs_write( int fd, void const * buffer, unsigned n ) +{ + return _write( fd, buffer, n ); +} + +int fs_close( int fd ) +{ + return _close( fd ); +} + +int fs_mkdir( std::string const & path, int /*mode*/ ) +{ + return _mkdir( path.c_str() ); +} + +bool fs_exists( std::string const & path ) +{ + return _access( path.c_str(), 0 ) == 0; +} + +int fs_stat( std::string const & path, int & mode, std::time_t & mtime, std::time_t & ctime, std::time_t & atime ) +{ + struct _stat st; + + int r = _stat( path.c_str(), &st ); + + if( r == 0 ) + { + mode = st.st_mode; + + mtime = st.st_mtime; + ctime = st.st_ctime; + atime = st.st_atime; + } + + return r; +} + +int fs_utime( std::string const & path, std::time_t mtime, std::time_t atime ) +{ + _utimbuf ut; + + ut.actime = atime; + ut.modtime = mtime; + + return _utime( path.c_str(), &ut ); +} + +int fs_rmdir( std::string const & path ) +{ + return _rmdir( path.c_str() ); +} + +void fs_remove_all( std::string const & path, void (*removing)( std::string const & ), void (*error)( std::string const &, int ) ) +{ + removing( path ); + + DWORD dw = GetFileAttributesA( path.c_str() ); + + if( dw == INVALID_FILE_ATTRIBUTES ) + { + dw = 0; + } + + if( dw & FILE_ATTRIBUTE_DIRECTORY ) + { + if( dw & FILE_ATTRIBUTE_REPARSE_POINT ) + { + // junction or directory symlink + + int r = _rmdir( path.c_str() ); + + if( r != 0 ) + { + error( path, errno ); + } + + return; + } + } + else + { + // file or file symlink + + int r = std::remove( path.c_str() ); + + if( r != 0 ) + { + error( path, errno ); + } + + return; + } + + // ordinary directory, descend + + assert( dw & FILE_ATTRIBUTE_DIRECTORY ); + assert( ( dw & FILE_ATTRIBUTE_REPARSE_POINT ) == 0 ); + + _finddata_t fd; + + intptr_t r = _findfirst( ( path + "/*" ).c_str(), &fd ); + + if( r >= 0 ) + { + do + { + std::string name = fd.name; + + if( name != "." && name != ".." ) + { + fs_remove_all( path + '/' + name, removing, error ); + } + } + while( _findnext( r, &fd ) == 0 ); + + _findclose( r ); + + { + int r2 = _rmdir( path.c_str() ); + + if( r2 != 0 ) + { + error( path, errno ); + } + } + } + else + { + error( path + "/*", errno ); + } +} + +int fs_readdir( std::string const & path, std::vector< std::string > & entries ) +{ + entries.resize( 0 ); + + _finddata_t fd; + + intptr_t r = _findfirst( ( path + "/*" ).c_str(), &fd ); + + if( r >= 0 ) + { + do + { + entries.push_back( fd.name ); + } + while( _findnext( r, &fd ) == 0 ); + + _findclose( r ); + + return 0; + } + else + { + return -1; + } +} + +bool fs_is_dir( std::string const & path ) +{ + DWORD dw = GetFileAttributesA( path.c_str() ); + + if( dw == INVALID_FILE_ATTRIBUTES ) + { + dw = 0; + } + + return dw & FILE_ATTRIBUTE_DIRECTORY? true: false; +} + +static int errno_from_last_error( DWORD r ) +{ + switch( r ) + { + case ERROR_ACCESS_DENIED: return EACCES; + case ERROR_ALREADY_EXISTS: return EEXIST; + case ERROR_BAD_NET_NAME: return ENOENT; + case ERROR_BAD_NETPATH: return ENOENT; + case ERROR_BAD_PATHNAME: return ENOENT; + case ERROR_BAD_UNIT: return ENODEV; + case ERROR_BROKEN_PIPE: return EPIPE; + case ERROR_BUFFER_OVERFLOW: return ENAMETOOLONG; + case ERROR_BUSY: return EBUSY; + case ERROR_BUSY_DRIVE: return EBUSY; + case ERROR_CANNOT_MAKE: return EACCES; + case ERROR_CANTOPEN: return EIO; + case ERROR_CANTREAD: return EIO; + case ERROR_CANTWRITE: return EIO; + case ERROR_CHILD_NOT_COMPLETE: return ECHILD; + case ERROR_CURRENT_DIRECTORY: return EBUSY; + case ERROR_DEV_NOT_EXIST: return ENODEV; + case ERROR_DEVICE_IN_USE: return EBUSY; + case ERROR_DIR_NOT_EMPTY: return ENOTEMPTY; + case ERROR_DIRECT_ACCESS_HANDLE: return EPERM; + case ERROR_DIRECTORY: return EINVAL; + case ERROR_DISK_FULL: return ENOSPC; + case ERROR_DRIVE_LOCKED: return EACCES; + case ERROR_FAIL_I24: return EIO; + case ERROR_FILE_EXISTS: return EEXIST; + case ERROR_FILE_NOT_FOUND: return ENOENT; + case ERROR_FILENAME_EXCED_RANGE: return ENAMETOOLONG; + case ERROR_HANDLE_DISK_FULL: return ENOSPC; + case ERROR_INVALID_ACCESS: return EACCES; + case ERROR_INVALID_DRIVE: return ENODEV; + case ERROR_INVALID_FUNCTION: return ENOSYS; + case ERROR_INVALID_HANDLE: return EBADF; + case ERROR_INVALID_NAME: return EINVAL; + case ERROR_INVALID_TARGET_HANDLE: return EBADF; + case ERROR_LOCK_FAILED: return EACCES; + case ERROR_LOCK_VIOLATION: return EACCES; + case ERROR_LOCKED: return EACCES; + case ERROR_MAX_THRDS_REACHED: return EAGAIN; + case ERROR_NEGATIVE_SEEK: return EINVAL; + case ERROR_NETWORK_ACCESS_DENIED: return EACCES; + case ERROR_NOACCESS: return EFAULT; + case ERROR_NO_MORE_FILES: return ENOENT; + case ERROR_NO_PROC_SLOTS: return EAGAIN; + case ERROR_NOT_ENOUGH_MEMORY: return ENOMEM; + case ERROR_NOT_ENOUGH_QUOTA: return ENOMEM; + case ERROR_NOT_LOCKED: return EACCES; + case ERROR_NOT_READY: return EAGAIN; + case ERROR_NOT_SAME_DEVICE: return EXDEV; + case ERROR_OPEN_FAILED: return EIO; + case ERROR_OPEN_FILES: return EBUSY; + case ERROR_OPERATION_ABORTED: return EINTR; // ECANCELED; + case ERROR_OUTOFMEMORY: return ENOMEM; + case ERROR_PATH_NOT_FOUND: return ENOENT; + case ERROR_READ_FAULT: return EIO; + case ERROR_RETRY: return EAGAIN; + case ERROR_SEEK: return EIO; + case ERROR_SEEK_ON_DEVICE: return ESPIPE; + case ERROR_SHARING_VIOLATION: return EACCES; + case ERROR_TOO_MANY_OPEN_FILES: return EMFILE; + case ERROR_WAIT_NO_CHILDREN: return ECHILD; + case ERROR_WRITE_FAULT: return EIO; + case ERROR_WRITE_PROTECT: return EROFS; + + default: return EINVAL; + } +} + +static void set_errno_from_last_error( DWORD r ) +{ + errno = errno_from_last_error( r ); +} + +typedef BOOLEAN (WINAPI *PtrCreateSymbolicLinkA)( + /*__in*/ LPCSTR lpSymlinkFileName, + /*__in*/ LPCSTR lpTargetFileName, + /*__in*/ DWORD dwFlags + ); + +static PtrCreateSymbolicLinkA CreateSymbolicLinkA = PtrCreateSymbolicLinkA( GetProcAddress( GetModuleHandleA( "kernel32.dll" ), "CreateSymbolicLinkA" ) ); + +#ifndef SYMBOLIC_LINK_FLAG_DIRECTORY +# define SYMBOLIC_LINK_FLAG_DIRECTORY 1 +#endif + +int fs_link_file( std::string const & link, std::string const & target ) +{ + { + // CreateSymbolicLink happily accepts '/' separated targets, but the symlink doesn't work + + std::string t2( target ); + std::replace( t2.begin(), t2.end(), '/', '\\' ); + + if( !make_relative( link, t2 ) ) + { + errno = EINVAL; + return -1; + } + + if( CreateSymbolicLinkA && CreateSymbolicLinkA( link.c_str(), t2.c_str(), 0 ) ) + { + return 0; + } + } + + if( CreateHardLinkA( link.c_str(), target.c_str(), 0 ) ) + { + return 0; + } + + set_errno_from_last_error( GetLastError() ); + return -1; +} + +// definitions copied from Boost.Filesystem + +#if !defined(REPARSE_DATA_BUFFER_HEADER_SIZE) // mingw winnt.h does provide the defs + +#define SYMLINK_FLAG_RELATIVE 1 + +typedef struct _REPARSE_DATA_BUFFER { + ULONG ReparseTag; + USHORT ReparseDataLength; + USHORT Reserved; + union { + struct { + USHORT SubstituteNameOffset; + USHORT SubstituteNameLength; + USHORT PrintNameOffset; + USHORT PrintNameLength; + ULONG Flags; + WCHAR PathBuffer[1]; + } SymbolicLinkReparseBuffer; + struct { + USHORT SubstituteNameOffset; + USHORT SubstituteNameLength; + USHORT PrintNameOffset; + USHORT PrintNameLength; + WCHAR PathBuffer[1]; + } MountPointReparseBuffer; + struct { + UCHAR DataBuffer[1]; + } GenericReparseBuffer; + }; +} REPARSE_DATA_BUFFER, *PREPARSE_DATA_BUFFER; + +#define REPARSE_DATA_BUFFER_HEADER_SIZE \ + FIELD_OFFSET(REPARSE_DATA_BUFFER, GenericReparseBuffer) + +#endif + +#ifndef MAXIMUM_REPARSE_DATA_BUFFER_SIZE +# define MAXIMUM_REPARSE_DATA_BUFFER_SIZE ( 16 * 1024 ) +#endif + +#ifndef FSCTL_GET_REPARSE_POINT +# define FSCTL_GET_REPARSE_POINT 0x900a8 +#endif + +#ifndef FSCTL_SET_REPARSE_POINT +# define FSCTL_SET_REPARSE_POINT 0x900a4 +#endif + +#ifndef IO_REPARSE_TAG_MOUNT_POINT +# define IO_REPARSE_TAG_MOUNT_POINT 0xA0000003L +#endif + +static int create_junction( std::string const & link, std::string const & target ) +{ + std::wstring target2; + + { + wchar_t buffer[ 1024 ] = { 0 }; + + int r = MultiByteToWideChar( CP_ACP, 0, target.c_str(), -1, buffer, sizeof( buffer ) / sizeof( buffer[0] ) ); + + if( r == 0 ) + { + errno = ENAMETOOLONG; + return -1; + } + + target2 = buffer; + + std::replace( target2.begin(), target2.end(), L'/', L'\\' ); + + if( target2.empty() || *target2.rbegin() != L'\\' ) + { + target2 += L'\\'; + } + + DWORD dw = GetFullPathNameW( target2.c_str(), sizeof( buffer ) / sizeof( buffer[ 0 ] ), buffer, 0 ); + + if( dw >= sizeof( buffer ) / sizeof( buffer[ 0 ] ) ) + { + errno = ENAMETOOLONG; + return -1; + } + + if( dw == 0 ) + { + set_errno_from_last_error( GetLastError() ); + return -1; + } + + target2 = buffer; + } + + { + int r = _mkdir( link.c_str() ); + + if( r != 0 ) + { + return r; + } + } + + HANDLE h = ::CreateFileA( link.c_str(), GENERIC_WRITE, 0, NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT | FILE_FLAG_BACKUP_SEMANTICS, NULL ); + + if( h == INVALID_HANDLE_VALUE ) + { + DWORD r2 = GetLastError(); + + _rmdir( link.c_str() ); + + set_errno_from_last_error( r2 ); + return -1; + } + + std::wstring print_name( target2 ); + size_t print_name_len = print_name.size() * sizeof( wchar_t ); + + std::wstring substitute_name = L"\\??\\" + print_name; + size_t substitute_name_len = substitute_name.size() * sizeof( wchar_t ); + + unsigned char buffer[ MAXIMUM_REPARSE_DATA_BUFFER_SIZE ] = { 0 }; + + REPARSE_DATA_BUFFER * pb = reinterpret_cast< REPARSE_DATA_BUFFER * >( buffer ); + + pb->ReparseTag = IO_REPARSE_TAG_MOUNT_POINT; + pb->Reserved = 0; + + wcscpy( pb->MountPointReparseBuffer.PathBuffer, substitute_name.c_str() ); + wcscpy( pb->MountPointReparseBuffer.PathBuffer + substitute_name.size() + 1, print_name.c_str() ); + + pb->MountPointReparseBuffer.SubstituteNameOffset = 0; + pb->MountPointReparseBuffer.SubstituteNameLength = substitute_name_len; + + pb->MountPointReparseBuffer.PrintNameOffset = substitute_name_len + sizeof( wchar_t ); + pb->MountPointReparseBuffer.PrintNameLength = print_name_len; + + pb->ReparseDataLength = FIELD_OFFSET( REPARSE_DATA_BUFFER, MountPointReparseBuffer.PathBuffer ) - FIELD_OFFSET( REPARSE_DATA_BUFFER, MountPointReparseBuffer ) + substitute_name_len + sizeof( wchar_t ) + print_name_len + sizeof( wchar_t ); + + { + DWORD dw = 0; + BOOL r = ::DeviceIoControl( h, FSCTL_SET_REPARSE_POINT, buffer, pb->ReparseDataLength + REPARSE_DATA_BUFFER_HEADER_SIZE, 0, 0, &dw, 0 ); + + DWORD r2 = GetLastError(); + + CloseHandle( h ); + + if( !r ) + { + _rmdir( link.c_str() ); + + set_errno_from_last_error( r2 ); + return -1; + } + } + + return 0; +} + +int fs_link_dir( std::string const & link, std::string const & target ) +{ + { + // CreateSymbolicLink happily accepts '/' separated targets, but the symlink doesn't work + + std::string t2( target ); + std::replace( t2.begin(), t2.end(), '/', '\\' ); + + if( !make_relative( link, t2 ) ) + { + errno = EINVAL; + return -1; + } + + if( CreateSymbolicLinkA && CreateSymbolicLinkA( link.c_str(), t2.c_str(), SYMBOLIC_LINK_FLAG_DIRECTORY ) ) + { + return 0; + } + } + + return create_junction( link, target ); +} + +#else + +#include +#include +#include +#include +#include + +int fs_creat( std::string const & path, int mode ) +{ + return creat( path.c_str(), mode ); +} + +int fs_write( int fd, void const * buffer, unsigned n ) +{ + return write( fd, buffer, n ); +} + +int fs_close( int fd ) +{ + return close( fd ); +} + +int fs_mkdir( std::string const & path, int mode ) +{ + return mkdir( path.c_str(), mode ); +} + +bool fs_exists( std::string const & path ) +{ + struct stat st; + int r = stat( path.c_str(), &st ); + + return r == 0; +} + +int fs_stat( std::string const & path, int & mode, std::time_t & mtime, std::time_t & ctime, std::time_t & atime ) +{ + struct stat st; + int r = stat( path.c_str(), &st ); + + if( r == 0 ) + { + mode = st.st_mode; + + mtime = st.st_mtime; + ctime = st.st_ctime; + atime = st.st_atime; + } + + return r; +} + +int fs_utime( std::string const & path, std::time_t mtime, std::time_t atime ) +{ + utimbuf ut; + + ut.actime = atime; + ut.modtime = mtime; + + return utime( path.c_str(), &ut ); +} + +int fs_rmdir( std::string const & path ) +{ + return rmdir( path.c_str() ); +} + +void fs_remove_all( std::string const & path, void (*removing)( std::string const & ), void (*error)( std::string const &, int ) ) +{ + removing( path ); + + if( !fs_is_dir( path ) ) + { + // file or symlink + + int r = std::remove( path.c_str() ); + + if( r != 0 ) + { + error( path, errno ); + } + + return; + } + + // ordinary directory, descend + + DIR * pd = opendir( path.c_str() ); + + if( pd == 0 ) + { + error( path + "/*", errno ); + return; + } + + while( dirent * pe = readdir( pd ) ) + { + std::string name = pe->d_name; + + if( name != "." && name != ".." ) + { + fs_remove_all( path + '/' + name, removing, error ); + } + } + + closedir( pd ); + + int r = rmdir( path.c_str() ); + + if( r != 0 ) + { + error( path, errno ); + } +} + +int fs_readdir( std::string const & path, std::vector< std::string > & entries ) +{ + entries.resize( 0 ); + + DIR * pd = opendir( path.c_str() ); + if( pd == 0 ) return -1; + + while( dirent * pe = readdir( pd ) ) + { + entries.push_back( pe->d_name ); + } + + closedir( pd ); + return 0; +} + +bool fs_is_dir( std::string const & path ) +{ + struct stat st; + int r = stat( path.c_str(), &st ); + + if( r == 0 ) + { + return S_ISDIR( st.st_mode ); + } + else + { + return false; + } +} + +int fs_link_file( std::string const & link, std::string const & target ) +{ + std::string t2( target ); + + if( make_relative( link, t2 ) == 0 && symlink( t2.c_str(), link.c_str() ) == 0 ) + { + return 0; + } + else + { + return -1; + } +} + +int fs_link_dir( std::string const & link, std::string const & target ) +{ + return fs_link_file( link, target ); +} + +#endif // defined( _WIN32 ) diff --git a/src/fs.hpp b/src/fs.hpp new file mode 100644 index 0000000..1c26304 --- /dev/null +++ b/src/fs.hpp @@ -0,0 +1,43 @@ +#ifndef FS_HPP_INCLUDED +#define FS_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include +#include +#include + +int fs_creat( std::string const & path, int mode ); +int fs_write( int fd, void const * buffer, unsigned n ); +int fs_close( int fd ); + +int fs_mkdir( std::string const & path, int mode ); + +bool fs_exists( std::string const & path ); + +int fs_stat( std::string const & path, int & mode, std::time_t & mtime, std::time_t & ctime, std::time_t & atime ); + +std::time_t fs_mtime( std::string const & path ); + +int fs_utime( std::string const & path, std::time_t mtime, std::time_t atime ); + +int fs_rmdir( std::string const & path ); + +void fs_remove_all( std::string const & path, void (*removing)( std::string const & ), void (*error)( std::string const &, int ) ); + +int fs_readdir( std::string const & path, std::vector< std::string > & entries ); + +bool fs_is_dir( std::string const & path ); + +// these two functions treat 'target' as a path name, not as a string, as per POSIX symlink + +int fs_link_file( std::string const & link, std::string const & target ); // symlink, if fails on Windows, hard link +int fs_link_dir( std::string const & link, std::string const & target ); // symlink, if fails on Windows, junction + +#endif // #ifndef FS_HPP_INCLUDED diff --git a/src/http_reader.cpp b/src/http_reader.cpp new file mode 100644 index 0000000..057ca5b --- /dev/null +++ b/src/http_reader.cpp @@ -0,0 +1,126 @@ +// +// Copyright 2014 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "http_reader.hpp" +#include "error.hpp" +#include +#include + +http_url_parser::http_url_parser( std::string const & url ) +{ + if( url.substr( 0, 7 ) != "http://" ) + { + throw_error( url, "not an HTTP URL" ); + } + + std::size_t i = url.find( ':', 7 ); + std::size_t j = url.find( '/', 7 ); + + if( j != std::string::npos ) + { + request_ = url.substr( j ); + } + else + { + request_ = "/"; + } + + if( i >= j ) + { + // no port + + host_ = url.substr( 7, j - 7 ); + port_ = 80; + } + else + { + host_ = url.substr( 7, i - 7 ); + port_ = std::atoi( url.substr( i + 1, j - i - 1 ).c_str() ); + } +} + +std::string http_url_parser::host() const +{ + return host_; +} + +int http_url_parser::port() const +{ + return port_; +} + +std::string http_url_parser::request() const +{ + return request_; +} + +http_reader::http_reader( std::string const & url ): http_url_parser( url ), tcp_reader( this->host(), this->port() ), name_( url ) +{ + std::string rq = "GET " + this->request() + " HTTP/1.0\r\nHost: " + this->host() + "\r\nConnection: close\r\n\r\n"; + + this->write( rq.data(), rq.size() ); + + std::string line = this->read_line(); // HTTP/1.0 (code) (message) + + { + std::istringstream is( line ); + + std::string http, message; + int code; + + if( is >> http >> code >> message && http.substr( 0, 7 ) == "HTTP/1." && code >= 100 ) + { + if( code >= 300 ) + { + throw_error( name_, "HTTP error: " + line ); + } + } + else + { + throw_error( name_, "invalid server response: '" + line + "'" ); + } + } + + // skip rest of header + while( !read_line().empty() ); +} + +http_reader::~http_reader() +{ +} + +std::string http_reader::name() const +{ + return name_; +} + +std::string http_reader::read_line() +{ + std::string r; + + for( ;; ) + { + char ch; + + std::size_t r2 = read( &ch, 1 ); + + if( r2 != 1 ) + { + throw_error( name_, "unexpected end of data" ); + } + + if( ch == '\n' ) break; + + if( ch != '\r' ) + { + r += ch; + } + } + + return r; +} diff --git a/src/http_reader.hpp b/src/http_reader.hpp new file mode 100644 index 0000000..2609c37 --- /dev/null +++ b/src/http_reader.hpp @@ -0,0 +1,54 @@ +#ifndef HTTP_READER_HPP_INCLUDED +#define HTTP_READER_HPP_INCLUDED + +// +// Copyright 2014 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "tcp_reader.hpp" + +class http_url_parser +{ +private: + + std::string host_; + int port_; + + std::string request_; + +public: + + explicit http_url_parser( std::string const & url ); + + std::string host() const; + int port() const; + + std::string request() const; +}; + +class http_reader: private http_url_parser, public tcp_reader +{ +private: + + std::string name_; + +private: + + http_reader( http_reader const & ); + http_reader& operator=( http_reader const & ); + + std::string read_line(); + +public: + + explicit http_reader( std::string const & url ); + ~http_reader(); + + virtual std::string name() const; +}; + +#endif // #ifndef HTTP_READER_HPP_INCLUDED diff --git a/src/json.cpp b/src/json.cpp new file mode 100644 index 0000000..795b7bb --- /dev/null +++ b/src/json.cpp @@ -0,0 +1,286 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "json.hpp" +#include "error.hpp" +#include +#include + +static void throw_parse_error( std::string const & name, std::string const & reason ) +{ + throw_error( name, "parse error: " + reason ); +} + +static void throw_eof_error( std::string const & name ) +{ + throw_parse_error( name, "unexpected end of data" ); +} + +static void throw_expected_error( std::string const & name, std::string const & expected, std::string const & tk ) +{ + throw_parse_error( name, "expected " + expected + ", got '" + tk + "'" ); +} + +static bool is_structural( char ch ) +{ + return ch == '{' || ch == '}' || ch == '[' || ch == ']' || ch == ':' || ch == ',' || ch == 0; +} + +static bool is_literal( char ch ) +{ + return ( ch >= 'a' && ch <= 'z' ) || ( ch >= 'A' && ch <= 'Z' ) || ch == '_' || ( ch >= '0' && ch <= '9' ) || ch == '-' || ch == '+' || ch == '.'; +} + +static bool is_whitespace( char ch ) +{ + return ch == 0x09 || ch == 0x0A || ch == 0x0D || ch == ' '; +} + +static void utf8_encode( std::string & r, unsigned c ) +{ + if( c < 0x80 ) + { + r.push_back( static_cast( c ) ); + } + else if( c < 0x800 ) + { + r.push_back( static_cast( 0xC0 | (c >> 6) ) ); + r.push_back( static_cast( 0x80 | (c & 0x3F) ) ); + } + else // if( c < 0x10000 ) + { + r.push_back( static_cast( 0xE0 | (c >> 12) ) ); + r.push_back( static_cast( 0x80 | ((c >> 6) & 0x3F) ) ); + r.push_back( static_cast( 0x80 | (c & 0x3F) ) ); + } +} + +static std::string get_token( std::istream & is, std::string const & name ) +{ + char ch; + + std::string tk; + + if( !( is >> ch ) ) + { + // EOF + } + else if( is_literal( ch ) ) + { + for( ;; ) + { + tk += ch; + + if( !is.get( ch ) ) + { + throw_eof_error( name ); + } + + if( is_structural( ch ) || is_whitespace( ch ) ) + { + is.putback( ch ); + break; + } + } + } + else if( ch != '"' ) + { + tk += ch; + } + else + { + // string + + tk += ch; + + for( ;; ) + { + if( !is.get( ch ) ) + { + throw_eof_error( name ); + } + + if( ch == '"' ) + { + break; + } + + if( ch != '\\' ) + { + tk += ch; + continue; + } + + if( !is.get( ch ) ) + { + throw_eof_error( name ); + } + + switch( ch ) + { + case 'b': tk += '\x08'; break; + case 'f': tk += '\x0C'; break; + case 'r': tk += '\x0D'; break; + case 'n': tk += '\x0A'; break; + case 't': tk += '\x09'; break; + default: tk += ch; break; + + case 'u': + { + char buffer[ 5 ] = { 0 }; + int code = 0; + + if( !is.read( buffer, 4 ) || sscanf( buffer, "%x", &code ) != 1 || code <= 0 ) + { + throw_parse_error( name, std::string( "invalid character code '\\u" ) + buffer + "'" ); + } + else + { + utf8_encode( tk, code ); + } + } + break; + } + } + } + + return tk; +} + +static void json_parse( std::istream & /*is*/, std::string const & name, std::string tk, std::string & v ); +template< class T > static void json_parse( std::istream & is, std::string const & name, std::string tk, std::vector< T > & v ); +template< class K, class V > static void json_parse( std::istream & is, std::string const & name, std::string tk, std::map< K, V > & m ); + +// tk is already the result of get_token() +static void json_parse( std::istream & /*is*/, std::string const & name, std::string tk, std::string & v ) +{ + char ch = tk[ 0 ]; + + if( is_structural( ch ) ) + { + throw_expected_error( name, "string", tk ); + } + + if( tk[ 0 ] == '"' ) + { + v = tk.substr( 1 ); + } + else + { + v = tk; // true, false, null, or number + } +} + +// tk is already the result of get_token() +template< class T > static void json_parse( std::istream & is, std::string const & name, std::string tk, std::vector< T > & v ) +{ + if( tk != "[" ) + { + // assume single element + + T t; + json_parse( is, name, tk, t ); + + v.push_back( t ); + + return; + } + + for( ;; ) + { + tk = get_token( is, name ); + + if( tk == "]" ) + { + break; + } + + { + T t; + json_parse( is, name, tk, t ); + + v.push_back( t ); + } + + tk = get_token( is, name ); + + if( tk == "]" ) + { + break; + } + + if( tk != "," ) + { + throw_expected_error( name, "',' or ']'", tk ); + } + } +} + +// tk is already the result of get_token() +template< class K, class V > static void json_parse( std::istream & is, std::string const & name, std::string tk, std::map< K, V > & m ) +{ + if( tk != "{" ) + { + throw_expected_error( name, "'{'", tk ); + } + + for( ;; ) + { + tk = get_token( is, name ); + + if( tk == "}" ) + { + break; + } + + K k; + json_parse( is, name, tk, k ); + + tk = get_token( is, name ); + + if( tk != ":" ) + { + throw_expected_error( name, "':'", tk ); + return; + } + + tk = get_token( is, name ); + + V v; + json_parse( is, name, tk, v ); + + m[ k ] = v; + + tk = get_token( is, name ); + + if( tk == "}" ) + { + break; + } + + if( tk != "," ) + { + throw_expected_error( name, "',' or ']'", tk ); + } + } +} + +void read_libraries_json( std::string const & name, std::vector< std::map< std::string, std::vector< std::string > > > & libraries ) +{ + std::ifstream is( name.c_str() ); + + if( !is ) + { + throw_errno_error( name, "open error", errno ); + } + + std::string tk; + tk = get_token( is, name ); + + json_parse( is, name, tk, libraries ); +} diff --git a/src/json.hpp b/src/json.hpp new file mode 100644 index 0000000..9f4fd4b --- /dev/null +++ b/src/json.hpp @@ -0,0 +1,18 @@ +#ifndef JSON_HPP_INCLUDED +#define JSON_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include +#include +#include + +void read_libraries_json( std::string const & name, std::vector< std::map< std::string, std::vector< std::string > > > & libraries ); + +#endif // #ifndef JSON_HPP_INCLUDED diff --git a/src/lzma/7zTypes.h b/src/lzma/7zTypes.h new file mode 100644 index 0000000..4af64e1 --- /dev/null +++ b/src/lzma/7zTypes.h @@ -0,0 +1,262 @@ +/* 7zTypes.h -- Basic types +2013-11-12 : Igor Pavlov : Public domain */ + +// No Copyright - Public Domain +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt + +#ifndef __7Z_TYPES_H +#define __7Z_TYPES_H + +#ifdef _WIN32 +/* #include */ +#endif + +#include + +#ifndef EXTERN_C_BEGIN +#ifdef __cplusplus +#define EXTERN_C_BEGIN extern "C" { +#define EXTERN_C_END } +#else +#define EXTERN_C_BEGIN +#define EXTERN_C_END +#endif +#endif + +EXTERN_C_BEGIN + +#define SZ_OK 0 + +#define SZ_ERROR_DATA 1 +#define SZ_ERROR_MEM 2 +#define SZ_ERROR_CRC 3 +#define SZ_ERROR_UNSUPPORTED 4 +#define SZ_ERROR_PARAM 5 +#define SZ_ERROR_INPUT_EOF 6 +#define SZ_ERROR_OUTPUT_EOF 7 +#define SZ_ERROR_READ 8 +#define SZ_ERROR_WRITE 9 +#define SZ_ERROR_PROGRESS 10 +#define SZ_ERROR_FAIL 11 +#define SZ_ERROR_THREAD 12 + +#define SZ_ERROR_ARCHIVE 16 +#define SZ_ERROR_NO_ARCHIVE 17 + +typedef int SRes; + +#ifdef _WIN32 +/* typedef DWORD WRes; */ +typedef unsigned WRes; +#else +typedef int WRes; +#endif + +#ifndef RINOK +#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; } +#endif + +typedef unsigned char Byte; +typedef short Int16; +typedef unsigned short UInt16; + +#ifdef _LZMA_UINT32_IS_ULONG +typedef long Int32; +typedef unsigned long UInt32; +#else +typedef int Int32; +typedef unsigned int UInt32; +#endif + +#ifdef _SZ_NO_INT_64 + +/* define _SZ_NO_INT_64, if your compiler doesn't support 64-bit integers. + NOTES: Some code will work incorrectly in that case! */ + +typedef long Int64; +typedef unsigned long UInt64; + +#else + +#if defined(_MSC_VER) || defined(__BORLANDC__) +typedef __int64 Int64; +typedef unsigned __int64 UInt64; +#define UINT64_CONST(n) n +#else +typedef long long int Int64; +typedef unsigned long long int UInt64; +#define UINT64_CONST(n) n ## ULL +#endif + +#endif + +#ifdef _LZMA_NO_SYSTEM_SIZE_T +typedef UInt32 SizeT; +#else +typedef size_t SizeT; +#endif + +typedef int Bool; +#define True 1 +#define False 0 + + +#ifdef _WIN32 +#define MY_STD_CALL __stdcall +#else +#define MY_STD_CALL +#endif + +#ifdef _MSC_VER + +#if _MSC_VER >= 1300 +#define MY_NO_INLINE __declspec(noinline) +#else +#define MY_NO_INLINE +#endif + +#define MY_CDECL __cdecl +#define MY_FAST_CALL __fastcall + +#else + +#define MY_NO_INLINE +#define MY_CDECL +#define MY_FAST_CALL + +#endif + + +/* The following interfaces use first parameter as pointer to structure */ + +typedef struct +{ + Byte (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */ +} IByteIn; + +typedef struct +{ + void (*Write)(void *p, Byte b); +} IByteOut; + +typedef struct +{ + SRes (*Read)(void *p, void *buf, size_t *size); + /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. + (output(*size) < input(*size)) is allowed */ +} ISeqInStream; + +/* it can return SZ_ERROR_INPUT_EOF */ +SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size); +SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType); +SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf); + +typedef struct +{ + size_t (*Write)(void *p, const void *buf, size_t size); + /* Returns: result - the number of actually written bytes. + (result < size) means error */ +} ISeqOutStream; + +typedef enum +{ + SZ_SEEK_SET = 0, + SZ_SEEK_CUR = 1, + SZ_SEEK_END = 2 +} ESzSeek; + +typedef struct +{ + SRes (*Read)(void *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ + SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); +} ISeekInStream; + +typedef struct +{ + SRes (*Look)(void *p, const void **buf, size_t *size); + /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. + (output(*size) > input(*size)) is not allowed + (output(*size) < input(*size)) is allowed */ + SRes (*Skip)(void *p, size_t offset); + /* offset must be <= output(*size) of Look */ + + SRes (*Read)(void *p, void *buf, size_t *size); + /* reads directly (without buffer). It's same as ISeqInStream::Read */ + SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); +} ILookInStream; + +SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size); +SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset); + +/* reads via ILookInStream::Read */ +SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType); +SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size); + +#define LookToRead_BUF_SIZE (1 << 14) + +typedef struct +{ + ILookInStream s; + ISeekInStream *realStream; + size_t pos; + size_t size; + Byte buf[LookToRead_BUF_SIZE]; +} CLookToRead; + +void LookToRead_CreateVTable(CLookToRead *p, int lookahead); +void LookToRead_Init(CLookToRead *p); + +typedef struct +{ + ISeqInStream s; + ILookInStream *realStream; +} CSecToLook; + +void SecToLook_CreateVTable(CSecToLook *p); + +typedef struct +{ + ISeqInStream s; + ILookInStream *realStream; +} CSecToRead; + +void SecToRead_CreateVTable(CSecToRead *p); + +typedef struct +{ + SRes (*Progress)(void *p, UInt64 inSize, UInt64 outSize); + /* Returns: result. (result != SZ_OK) means break. + Value (UInt64)(Int64)-1 for size means unknown value. */ +} ICompressProgress; + +typedef struct +{ + void *(*Alloc)(void *p, size_t size); + void (*Free)(void *p, void *address); /* address can be 0 */ +} ISzAlloc; + +#define IAlloc_Alloc(p, size) (p)->Alloc((p), size) +#define IAlloc_Free(p, a) (p)->Free((p), a) + +#ifdef _WIN32 + +#define CHAR_PATH_SEPARATOR '\\' +#define WCHAR_PATH_SEPARATOR L'\\' +#define STRING_PATH_SEPARATOR "\\" +#define WSTRING_PATH_SEPARATOR L"\\" + +#else + +#define CHAR_PATH_SEPARATOR '/' +#define WCHAR_PATH_SEPARATOR L'/' +#define STRING_PATH_SEPARATOR "/" +#define WSTRING_PATH_SEPARATOR L"/" + +#endif + +EXTERN_C_END + +#endif diff --git a/src/lzma/Compiler.h b/src/lzma/Compiler.h new file mode 100644 index 0000000..d56997a --- /dev/null +++ b/src/lzma/Compiler.h @@ -0,0 +1,34 @@ +/* Compiler.h -- Compiler ypes +2013-11-12 : Igor Pavlov : Public domain */ + +// No Copyright - Public Domain +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt + +#ifndef __7Z_COMPILER_H +#define __7Z_COMPILER_H + +#ifdef _MSC_VER + + #ifdef UNDER_CE + #define RPC_NO_WINDOWS_H + /* #pragma warning(disable : 4115) // '_RPC_ASYNC_STATE' : named type definition in parentheses */ + #pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union + #pragma warning(disable : 4214) // nonstandard extension used : bit field types other than int + #endif + + #if _MSC_VER >= 1300 + #pragma warning(disable : 4996) // This function or variable may be unsafe + #else + #pragma warning(disable : 4511) // copy constructor could not be generated + #pragma warning(disable : 4512) // assignment operator could not be generated + #pragma warning(disable : 4702) // unreachable code + #pragma warning(disable : 4710) // not inlined + #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information + #endif + +#endif + +#endif diff --git a/src/lzma/LzmaDec.c b/src/lzma/LzmaDec.c new file mode 100644 index 0000000..2b59945 --- /dev/null +++ b/src/lzma/LzmaDec.c @@ -0,0 +1,1031 @@ +/* LzmaDec.c -- LZMA Decoder +2011-09-03 : Igor Pavlov : Public domain */ + +// No Copyright - Public Domain +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt + +#include "Precomp.h" + +#include "LzmaDec.h" + +#include + +#define kNumTopBits 24 +#define kTopValue ((UInt32)1 << kNumTopBits) + +#define kNumBitModelTotalBits 11 +#define kBitModelTotal (1 << kNumBitModelTotalBits) +#define kNumMoveBits 5 + +#define RC_INIT_SIZE 5 + +#define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } + +#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); +#define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); +#define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ + { UPDATE_0(p); i = (i + i); A0; } else \ + { UPDATE_1(p); i = (i + i) + 1; A1; } +#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) + +#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } +#define TREE_DECODE(probs, limit, i) \ + { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; } + +/* #define _LZMA_SIZE_OPT */ + +#ifdef _LZMA_SIZE_OPT +#define TREE_6_DECODE(probs, i) TREE_DECODE(probs, (1 << 6), i) +#else +#define TREE_6_DECODE(probs, i) \ + { i = 1; \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + TREE_GET_BIT(probs, i); \ + i -= 0x40; } +#endif + +#define NORMAL_LITER_DEC GET_BIT(prob + symbol, symbol) +#define MATCHED_LITER_DEC \ + matchByte <<= 1; \ + bit = (matchByte & offs); \ + probLit = prob + offs + bit + symbol; \ + GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) + +#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } + +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define UPDATE_0_CHECK range = bound; +#define UPDATE_1_CHECK range -= bound; code -= bound; +#define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ + { UPDATE_0_CHECK; i = (i + i); A0; } else \ + { UPDATE_1_CHECK; i = (i + i) + 1; A1; } +#define GET_BIT_CHECK(p, i) GET_BIT2_CHECK(p, i, ; , ;) +#define TREE_DECODE_CHECK(probs, limit, i) \ + { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } + + +#define kNumPosBitsMax 4 +#define kNumPosStatesMax (1 << kNumPosBitsMax) + +#define kLenNumLowBits 3 +#define kLenNumLowSymbols (1 << kLenNumLowBits) +#define kLenNumMidBits 3 +#define kLenNumMidSymbols (1 << kLenNumMidBits) +#define kLenNumHighBits 8 +#define kLenNumHighSymbols (1 << kLenNumHighBits) + +#define LenChoice 0 +#define LenChoice2 (LenChoice + 1) +#define LenLow (LenChoice2 + 1) +#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) +#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define kNumLenProbs (LenHigh + kLenNumHighSymbols) + + +#define kNumStates 12 +#define kNumLitStates 7 + +#define kStartPosModelIndex 4 +#define kEndPosModelIndex 14 +#define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) + +#define kNumPosSlotBits 6 +#define kNumLenToPosStates 4 + +#define kNumAlignBits 4 +#define kAlignTableSize (1 << kNumAlignBits) + +#define kMatchMinLen 2 +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) + +#define IsMatch 0 +#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +#define IsRepG0 (IsRep + kNumStates) +#define IsRepG1 (IsRepG0 + kNumStates) +#define IsRepG2 (IsRepG1 + kNumStates) +#define IsRep0Long (IsRepG2 + kNumStates) +#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) +#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) +#define LenCoder (Align + kAlignTableSize) +#define RepLenCoder (LenCoder + kNumLenProbs) +#define Literal (RepLenCoder + kNumLenProbs) + +#define LZMA_BASE_SIZE 1846 +#define LZMA_LIT_SIZE 768 + +#define LzmaProps_GetNumProbs(p) ((UInt32)LZMA_BASE_SIZE + (LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + +#if Literal != LZMA_BASE_SIZE +StopCompilingDueBUG +#endif + +#define LZMA_DIC_MIN (1 << 12) + +/* First LZMA-symbol is always decoded. +And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization +Out: + Result: + SZ_OK - OK + SZ_ERROR_DATA - Error + p->remainLen: + < kMatchSpecLenStart : normal remain + = kMatchSpecLenStart : finished + = kMatchSpecLenStart + 1 : Flush marker + = kMatchSpecLenStart + 2 : State Init Marker +*/ + +static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + CLzmaProb *probs = p->probs; + + unsigned state = p->state; + UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; + unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; + unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; + unsigned lc = p->prop.lc; + + Byte *dic = p->dic; + SizeT dicBufSize = p->dicBufSize; + SizeT dicPos = p->dicPos; + + UInt32 processedPos = p->processedPos; + UInt32 checkDicSize = p->checkDicSize; + unsigned len = 0; + + const Byte *buf = p->buf; + UInt32 range = p->range; + UInt32 code = p->code; + + do + { + CLzmaProb *prob; + UInt32 bound; + unsigned ttt; + unsigned posState = processedPos & pbMask; + + prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + IF_BIT_0(prob) + { + unsigned symbol; + UPDATE_0(prob); + prob = probs + Literal; + if (checkDicSize != 0 || processedPos != 0) + prob += (LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + + (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); + + if (state < kNumLitStates) + { + state -= (state < 4) ? state : 3; + symbol = 1; + #ifdef _LZMA_SIZE_OPT + do { NORMAL_LITER_DEC } while (symbol < 0x100); + #else + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + NORMAL_LITER_DEC + #endif + } + else + { + unsigned matchByte = p->dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; + unsigned offs = 0x100; + state -= (state < 10) ? 3 : 6; + symbol = 1; + #ifdef _LZMA_SIZE_OPT + do + { + unsigned bit; + CLzmaProb *probLit; + MATCHED_LITER_DEC + } + while (symbol < 0x100); + #else + { + unsigned bit; + CLzmaProb *probLit; + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + MATCHED_LITER_DEC + } + #endif + } + dic[dicPos++] = (Byte)symbol; + processedPos++; + continue; + } + else + { + UPDATE_1(prob); + prob = probs + IsRep + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + state += kNumStates; + prob = probs + LenCoder; + } + else + { + UPDATE_1(prob); + if (checkDicSize == 0 && processedPos == 0) + return SZ_ERROR_DATA; + prob = probs + IsRepG0 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + IF_BIT_0(prob) + { + UPDATE_0(prob); + dic[dicPos] = dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; + dicPos++; + processedPos++; + state = state < kNumLitStates ? 9 : 11; + continue; + } + UPDATE_1(prob); + } + else + { + UInt32 distance; + UPDATE_1(prob); + prob = probs + IsRepG1 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + distance = rep1; + } + else + { + UPDATE_1(prob); + prob = probs + IsRepG2 + state; + IF_BIT_0(prob) + { + UPDATE_0(prob); + distance = rep2; + } + else + { + UPDATE_1(prob); + distance = rep3; + rep3 = rep2; + } + rep2 = rep1; + } + rep1 = rep0; + rep0 = distance; + } + state = state < kNumLitStates ? 8 : 11; + prob = probs + RepLenCoder; + } + { + unsigned limit, offset; + CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenLow + (posState << kLenNumLowBits); + offset = 0; + limit = (1 << kLenNumLowBits); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenChoice2; + IF_BIT_0(probLen) + { + UPDATE_0(probLen); + probLen = prob + LenMid + (posState << kLenNumMidBits); + offset = kLenNumLowSymbols; + limit = (1 << kLenNumMidBits); + } + else + { + UPDATE_1(probLen); + probLen = prob + LenHigh; + offset = kLenNumLowSymbols + kLenNumMidSymbols; + limit = (1 << kLenNumHighBits); + } + } + TREE_DECODE(probLen, limit, len); + len += offset; + } + + if (state >= kNumStates) + { + UInt32 distance; + prob = probs + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); + TREE_6_DECODE(prob, distance); + if (distance >= kStartPosModelIndex) + { + unsigned posSlot = (unsigned)distance; + int numDirectBits = (int)(((distance >> 1) - 1)); + distance = (2 | (distance & 1)); + if (posSlot < kEndPosModelIndex) + { + distance <<= numDirectBits; + prob = probs + SpecPos + distance - posSlot - 1; + { + UInt32 mask = 1; + unsigned i = 1; + do + { + GET_BIT2(prob + i, i, ; , distance |= mask); + mask <<= 1; + } + while (--numDirectBits != 0); + } + } + else + { + numDirectBits -= kNumAlignBits; + do + { + NORMALIZE + range >>= 1; + + { + UInt32 t; + code -= range; + t = (0 - ((UInt32)code >> 31)); /* (UInt32)((Int32)code >> 31) */ + distance = (distance << 1) + (t + 1); + code += range & t; + } + /* + distance <<= 1; + if (code >= range) + { + code -= range; + distance |= 1; + } + */ + } + while (--numDirectBits != 0); + prob = probs + Align; + distance <<= kNumAlignBits; + { + unsigned i = 1; + GET_BIT2(prob + i, i, ; , distance |= 1); + GET_BIT2(prob + i, i, ; , distance |= 2); + GET_BIT2(prob + i, i, ; , distance |= 4); + GET_BIT2(prob + i, i, ; , distance |= 8); + } + if (distance == (UInt32)0xFFFFFFFF) + { + len += kMatchSpecLenStart; + state -= kNumStates; + break; + } + } + } + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance + 1; + if (checkDicSize == 0) + { + if (distance >= processedPos) + return SZ_ERROR_DATA; + } + else if (distance >= checkDicSize) + return SZ_ERROR_DATA; + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + } + + len += kMatchMinLen; + + if (limit == dicPos) + return SZ_ERROR_DATA; + { + SizeT rem = limit - dicPos; + unsigned curLen = ((rem < len) ? (unsigned)rem : len); + SizeT pos = (dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0); + + processedPos += curLen; + + len -= curLen; + if (pos + curLen <= dicBufSize) + { + Byte *dest = dic + dicPos; + ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; + const Byte *lim = dest + curLen; + dicPos += curLen; + do + *(dest) = (Byte)*(dest + src); + while (++dest != lim); + } + else + { + do + { + dic[dicPos++] = dic[pos]; + if (++pos == dicBufSize) + pos = 0; + } + while (--curLen != 0); + } + } + } + } + while (dicPos < limit && buf < bufLimit); + NORMALIZE; + p->buf = buf; + p->range = range; + p->code = code; + p->remainLen = len; + p->dicPos = dicPos; + p->processedPos = processedPos; + p->reps[0] = rep0; + p->reps[1] = rep1; + p->reps[2] = rep2; + p->reps[3] = rep3; + p->state = state; + + return SZ_OK; +} + +static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) +{ + if (p->remainLen != 0 && p->remainLen < kMatchSpecLenStart) + { + Byte *dic = p->dic; + SizeT dicPos = p->dicPos; + SizeT dicBufSize = p->dicBufSize; + unsigned len = p->remainLen; + UInt32 rep0 = p->reps[0]; + if (limit - dicPos < len) + len = (unsigned)(limit - dicPos); + + if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) + p->checkDicSize = p->prop.dicSize; + + p->processedPos += len; + p->remainLen -= len; + while (len != 0) + { + len--; + dic[dicPos] = dic[(dicPos - rep0) + ((dicPos < rep0) ? dicBufSize : 0)]; + dicPos++; + } + p->dicPos = dicPos; + } +} + +static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) +{ + do + { + SizeT limit2 = limit; + if (p->checkDicSize == 0) + { + UInt32 rem = p->prop.dicSize - p->processedPos; + if (limit - p->dicPos > rem) + limit2 = p->dicPos + rem; + } + RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); + if (p->processedPos >= p->prop.dicSize) + p->checkDicSize = p->prop.dicSize; + LzmaDec_WriteRem(p, limit); + } + while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); + + if (p->remainLen > kMatchSpecLenStart) + { + p->remainLen = kMatchSpecLenStart; + } + return 0; +} + +typedef enum +{ + DUMMY_ERROR, /* unexpected end of input stream */ + DUMMY_LIT, + DUMMY_MATCH, + DUMMY_REP +} ELzmaDummy; + +static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inSize) +{ + UInt32 range = p->range; + UInt32 code = p->code; + const Byte *bufLimit = buf + inSize; + CLzmaProb *probs = p->probs; + unsigned state = p->state; + ELzmaDummy res; + + { + CLzmaProb *prob; + UInt32 bound; + unsigned ttt; + unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); + + prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK + + /* if (bufLimit - buf >= 7) return DUMMY_LIT; */ + + prob = probs + Literal; + if (p->checkDicSize != 0 || p->processedPos != 0) + prob += (LZMA_LIT_SIZE * + ((((p->processedPos) & ((1 << (p->prop.lp)) - 1)) << p->prop.lc) + + (p->dic[(p->dicPos == 0 ? p->dicBufSize : p->dicPos) - 1] >> (8 - p->prop.lc)))); + + if (state < kNumLitStates) + { + unsigned symbol = 1; + do { GET_BIT_CHECK(prob + symbol, symbol) } while (symbol < 0x100); + } + else + { + unsigned matchByte = p->dic[p->dicPos - p->reps[0] + + ((p->dicPos < p->reps[0]) ? p->dicBufSize : 0)]; + unsigned offs = 0x100; + unsigned symbol = 1; + do + { + unsigned bit; + CLzmaProb *probLit; + matchByte <<= 1; + bit = (matchByte & offs); + probLit = prob + offs + bit + symbol; + GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) + } + while (symbol < 0x100); + } + res = DUMMY_LIT; + } + else + { + unsigned len; + UPDATE_1_CHECK; + + prob = probs + IsRep + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + state = 0; + prob = probs + LenCoder; + res = DUMMY_MATCH; + } + else + { + UPDATE_1_CHECK; + res = DUMMY_REP; + prob = probs + IsRepG0 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + NORMALIZE_CHECK; + return DUMMY_REP; + } + else + { + UPDATE_1_CHECK; + } + } + else + { + UPDATE_1_CHECK; + prob = probs + IsRepG1 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + } + else + { + UPDATE_1_CHECK; + prob = probs + IsRepG2 + state; + IF_BIT_0_CHECK(prob) + { + UPDATE_0_CHECK; + } + else + { + UPDATE_1_CHECK; + } + } + } + state = kNumStates; + prob = probs + RepLenCoder; + } + { + unsigned limit, offset; + CLzmaProb *probLen = prob + LenChoice; + IF_BIT_0_CHECK(probLen) + { + UPDATE_0_CHECK; + probLen = prob + LenLow + (posState << kLenNumLowBits); + offset = 0; + limit = 1 << kLenNumLowBits; + } + else + { + UPDATE_1_CHECK; + probLen = prob + LenChoice2; + IF_BIT_0_CHECK(probLen) + { + UPDATE_0_CHECK; + probLen = prob + LenMid + (posState << kLenNumMidBits); + offset = kLenNumLowSymbols; + limit = 1 << kLenNumMidBits; + } + else + { + UPDATE_1_CHECK; + probLen = prob + LenHigh; + offset = kLenNumLowSymbols + kLenNumMidSymbols; + limit = 1 << kLenNumHighBits; + } + } + TREE_DECODE_CHECK(probLen, limit, len); + len += offset; + } + + if (state < 4) + { + unsigned posSlot; + prob = probs + PosSlot + + ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + kNumPosSlotBits); + TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); + if (posSlot >= kStartPosModelIndex) + { + int numDirectBits = ((posSlot >> 1) - 1); + + /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */ + + if (posSlot < kEndPosModelIndex) + { + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; + } + else + { + numDirectBits -= kNumAlignBits; + do + { + NORMALIZE_CHECK + range >>= 1; + code -= range & (((code - range) >> 31) - 1); + /* if (code >= range) code -= range; */ + } + while (--numDirectBits != 0); + prob = probs + Align; + numDirectBits = kNumAlignBits; + } + { + unsigned i = 1; + do + { + GET_BIT_CHECK(prob + i, i); + } + while (--numDirectBits != 0); + } + } + } + } + } + NORMALIZE_CHECK; + return res; +} + + +static void LzmaDec_InitRc(CLzmaDec *p, const Byte *data) +{ + p->code = ((UInt32)data[1] << 24) | ((UInt32)data[2] << 16) | ((UInt32)data[3] << 8) | ((UInt32)data[4]); + p->range = 0xFFFFFFFF; + p->needFlush = 0; +} + +void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) +{ + p->needFlush = 1; + p->remainLen = 0; + p->tempBufSize = 0; + + if (initDic) + { + p->processedPos = 0; + p->checkDicSize = 0; + p->needInitState = 1; + } + if (initState) + p->needInitState = 1; +} + +void LzmaDec_Init(CLzmaDec *p) +{ + p->dicPos = 0; + LzmaDec_InitDicAndState(p, True, True); +} + +static void LzmaDec_InitStateReal(CLzmaDec *p) +{ + UInt32 numProbs = Literal + ((UInt32)LZMA_LIT_SIZE << (p->prop.lc + p->prop.lp)); + UInt32 i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + p->needInitState = 0; +} + +SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, + ELzmaFinishMode finishMode, ELzmaStatus *status) +{ + SizeT inSize = *srcLen; + (*srcLen) = 0; + LzmaDec_WriteRem(p, dicLimit); + + *status = LZMA_STATUS_NOT_SPECIFIED; + + while (p->remainLen != kMatchSpecLenStart) + { + int checkEndMarkNow; + + if (p->needFlush != 0) + { + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + if (p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + + LzmaDec_InitRc(p, p->tempBuf); + p->tempBufSize = 0; + } + + checkEndMarkNow = 0; + if (p->dicPos >= dicLimit) + { + if (p->remainLen == 0 && p->code == 0) + { + *status = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK; + return SZ_OK; + } + if (finishMode == LZMA_FINISH_ANY) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_OK; + } + if (p->remainLen != 0) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_ERROR_DATA; + } + checkEndMarkNow = 1; + } + + if (p->needInitState) + LzmaDec_InitStateReal(p); + + if (p->tempBufSize == 0) + { + SizeT processed; + const Byte *bufLimit; + if (inSize < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) + { + int dummyRes = LzmaDec_TryDummy(p, src, inSize); + if (dummyRes == DUMMY_ERROR) + { + memcpy(p->tempBuf, src, inSize); + p->tempBufSize = (unsigned)inSize; + (*srcLen) += inSize; + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_ERROR_DATA; + } + bufLimit = src; + } + else + bufLimit = src + inSize - LZMA_REQUIRED_INPUT_MAX; + p->buf = src; + if (LzmaDec_DecodeReal2(p, dicLimit, bufLimit) != 0) + return SZ_ERROR_DATA; + processed = (SizeT)(p->buf - src); + (*srcLen) += processed; + src += processed; + inSize -= processed; + } + else + { + unsigned rem = p->tempBufSize, lookAhead = 0; + while (rem < LZMA_REQUIRED_INPUT_MAX && lookAhead < inSize) + p->tempBuf[rem++] = src[lookAhead++]; + p->tempBufSize = rem; + if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) + { + int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem); + if (dummyRes == DUMMY_ERROR) + { + (*srcLen) += lookAhead; + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + if (checkEndMarkNow && dummyRes != DUMMY_MATCH) + { + *status = LZMA_STATUS_NOT_FINISHED; + return SZ_ERROR_DATA; + } + } + p->buf = p->tempBuf; + if (LzmaDec_DecodeReal2(p, dicLimit, p->buf) != 0) + return SZ_ERROR_DATA; + lookAhead -= (rem - (unsigned)(p->buf - p->tempBuf)); + (*srcLen) += lookAhead; + src += lookAhead; + inSize -= lookAhead; + p->tempBufSize = 0; + } + } + if (p->code == 0) + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; +} + +SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) +{ + SizeT outSize = *destLen; + SizeT inSize = *srcLen; + *srcLen = *destLen = 0; + for (;;) + { + SizeT inSizeCur = inSize, outSizeCur, dicPos; + ELzmaFinishMode curFinishMode; + SRes res; + if (p->dicPos == p->dicBufSize) + p->dicPos = 0; + dicPos = p->dicPos; + if (outSize > p->dicBufSize - dicPos) + { + outSizeCur = p->dicBufSize; + curFinishMode = LZMA_FINISH_ANY; + } + else + { + outSizeCur = dicPos + outSize; + curFinishMode = finishMode; + } + + res = LzmaDec_DecodeToDic(p, outSizeCur, src, &inSizeCur, curFinishMode, status); + src += inSizeCur; + inSize -= inSizeCur; + *srcLen += inSizeCur; + outSizeCur = p->dicPos - dicPos; + memcpy(dest, p->dic + dicPos, outSizeCur); + dest += outSizeCur; + outSize -= outSizeCur; + *destLen += outSizeCur; + if (res != 0) + return res; + if (outSizeCur == 0 || outSize == 0) + return SZ_OK; + } +} + +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc) +{ + alloc->Free(alloc, p->probs); + p->probs = 0; +} + +static void LzmaDec_FreeDict(CLzmaDec *p, ISzAlloc *alloc) +{ + alloc->Free(alloc, p->dic); + p->dic = 0; +} + +void LzmaDec_Free(CLzmaDec *p, ISzAlloc *alloc) +{ + LzmaDec_FreeProbs(p, alloc); + LzmaDec_FreeDict(p, alloc); +} + +SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size) +{ + UInt32 dicSize; + Byte d; + + if (size < LZMA_PROPS_SIZE) + return SZ_ERROR_UNSUPPORTED; + else + dicSize = data[1] | ((UInt32)data[2] << 8) | ((UInt32)data[3] << 16) | ((UInt32)data[4] << 24); + + if (dicSize < LZMA_DIC_MIN) + dicSize = LZMA_DIC_MIN; + p->dicSize = dicSize; + + d = data[0]; + if (d >= (9 * 5 * 5)) + return SZ_ERROR_UNSUPPORTED; + + p->lc = d % 9; + d /= 9; + p->pb = d / 5; + p->lp = d % 5; + + return SZ_OK; +} + +static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAlloc *alloc) +{ + UInt32 numProbs = LzmaProps_GetNumProbs(propNew); + if (p->probs == 0 || numProbs != p->numProbs) + { + LzmaDec_FreeProbs(p, alloc); + p->probs = (CLzmaProb *)alloc->Alloc(alloc, numProbs * sizeof(CLzmaProb)); + p->numProbs = numProbs; + if (p->probs == 0) + return SZ_ERROR_MEM; + } + return SZ_OK; +} + +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +{ + CLzmaProps propNew; + RINOK(LzmaProps_Decode(&propNew, props, propsSize)); + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + p->prop = propNew; + return SZ_OK; +} + +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +{ + CLzmaProps propNew; + SizeT dicBufSize; + RINOK(LzmaProps_Decode(&propNew, props, propsSize)); + RINOK(LzmaDec_AllocateProbs2(p, &propNew, alloc)); + dicBufSize = propNew.dicSize; + if (p->dic == 0 || dicBufSize != p->dicBufSize) + { + LzmaDec_FreeDict(p, alloc); + p->dic = (Byte *)alloc->Alloc(alloc, dicBufSize); + if (p->dic == 0) + { + LzmaDec_FreeProbs(p, alloc); + return SZ_ERROR_MEM; + } + } + p->dicBufSize = dicBufSize; + p->prop = propNew; + return SZ_OK; +} + +SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, + ELzmaStatus *status, ISzAlloc *alloc) +{ + CLzmaDec p; + SRes res; + SizeT outSize = *destLen, inSize = *srcLen; + *destLen = *srcLen = 0; + *status = LZMA_STATUS_NOT_SPECIFIED; + if (inSize < RC_INIT_SIZE) + return SZ_ERROR_INPUT_EOF; + LzmaDec_Construct(&p); + RINOK(LzmaDec_AllocateProbs(&p, propData, propSize, alloc)); + p.dic = dest; + p.dicBufSize = outSize; + LzmaDec_Init(&p); + *srcLen = inSize; + res = LzmaDec_DecodeToDic(&p, outSize, src, srcLen, finishMode, status); + *destLen = p.dicPos; + if (res == SZ_OK && *status == LZMA_STATUS_NEEDS_MORE_INPUT) + res = SZ_ERROR_INPUT_EOF; + LzmaDec_FreeProbs(&p, alloc); + return res; +} diff --git a/src/lzma/LzmaDec.h b/src/lzma/LzmaDec.h new file mode 100644 index 0000000..634ea8a --- /dev/null +++ b/src/lzma/LzmaDec.h @@ -0,0 +1,233 @@ +/* LzmaDec.h -- LZMA Decoder +2013-01-18 : Igor Pavlov : Public domain */ + +// No Copyright - Public Domain +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt + +#ifndef __LZMA_DEC_H +#define __LZMA_DEC_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +/* #define _LZMA_PROB32 */ +/* _LZMA_PROB32 can increase the speed on some CPUs, + but memory usage for CLzmaDec::probs will be doubled in that case */ + +#ifdef _LZMA_PROB32 +#define CLzmaProb UInt32 +#else +#define CLzmaProb UInt16 +#endif + + +/* ---------- LZMA Properties ---------- */ + +#define LZMA_PROPS_SIZE 5 + +typedef struct _CLzmaProps +{ + unsigned lc, lp, pb; + UInt32 dicSize; +} CLzmaProps; + +/* LzmaProps_Decode - decodes properties +Returns: + SZ_OK + SZ_ERROR_UNSUPPORTED - Unsupported properties +*/ + +SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size); + + +/* ---------- LZMA Decoder state ---------- */ + +/* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case. + Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */ + +#define LZMA_REQUIRED_INPUT_MAX 20 + +typedef struct +{ + CLzmaProps prop; + CLzmaProb *probs; + Byte *dic; + const Byte *buf; + UInt32 range, code; + SizeT dicPos; + SizeT dicBufSize; + UInt32 processedPos; + UInt32 checkDicSize; + unsigned state; + UInt32 reps[4]; + unsigned remainLen; + int needFlush; + int needInitState; + UInt32 numProbs; + unsigned tempBufSize; + Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; +} CLzmaDec; + +#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } + +void LzmaDec_Init(CLzmaDec *p); + +/* There are two types of LZMA streams: + 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. + 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ + +typedef enum +{ + LZMA_FINISH_ANY, /* finish at any point */ + LZMA_FINISH_END /* block must be finished at the end */ +} ELzmaFinishMode; + +/* ELzmaFinishMode has meaning only if the decoding reaches output limit !!! + + You must use LZMA_FINISH_END, when you know that current output buffer + covers last bytes of block. In other cases you must use LZMA_FINISH_ANY. + + If LZMA decoder sees end marker before reaching output limit, it returns SZ_OK, + and output value of destLen will be less than output buffer size limit. + You can check status result also. + + You can use multiple checks to test data integrity after full decompression: + 1) Check Result and "status" variable. + 2) Check that output(destLen) = uncompressedSize, if you know real uncompressedSize. + 3) Check that output(srcLen) = compressedSize, if you know real compressedSize. + You must use correct finish mode in that case. */ + +typedef enum +{ + LZMA_STATUS_NOT_SPECIFIED, /* use main error code instead */ + LZMA_STATUS_FINISHED_WITH_MARK, /* stream was finished with end mark. */ + LZMA_STATUS_NOT_FINISHED, /* stream was not finished */ + LZMA_STATUS_NEEDS_MORE_INPUT, /* you must provide more input bytes */ + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK /* there is probability that stream was finished without end mark */ +} ELzmaStatus; + +/* ELzmaStatus is used only as output value for function call */ + + +/* ---------- Interfaces ---------- */ + +/* There are 3 levels of interfaces: + 1) Dictionary Interface + 2) Buffer Interface + 3) One Call Interface + You can select any of these interfaces, but don't mix functions from different + groups for same object. */ + + +/* There are two variants to allocate state for Dictionary Interface: + 1) LzmaDec_Allocate / LzmaDec_Free + 2) LzmaDec_AllocateProbs / LzmaDec_FreeProbs + You can use variant 2, if you set dictionary buffer manually. + For Buffer Interface you must always use variant 1. + +LzmaDec_Allocate* can return: + SZ_OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported properties +*/ + +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc); +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc); + +SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAlloc *alloc); +void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); + +/* ---------- Dictionary Interface ---------- */ + +/* You can use it, if you want to eliminate the overhead for data copying from + dictionary to some other external buffer. + You must work with CLzmaDec variables directly in this interface. + + STEPS: + LzmaDec_Constr() + LzmaDec_Allocate() + for (each new stream) + { + LzmaDec_Init() + while (it needs more decompression) + { + LzmaDec_DecodeToDic() + use data from CLzmaDec::dic and update CLzmaDec::dicPos + } + } + LzmaDec_Free() +*/ + +/* LzmaDec_DecodeToDic + + The decoding to internal dictionary buffer (CLzmaDec::dic). + You must manually update CLzmaDec::dicPos, if it reaches CLzmaDec::dicBufSize !!! + +finishMode: + It has meaning only if the decoding reaches output limit (dicLimit). + LZMA_FINISH_ANY - Decode just dicLimit bytes. + LZMA_FINISH_END - Stream must be finished after dicLimit. + +Returns: + SZ_OK + status: + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + SZ_ERROR_DATA - Data error +*/ + +SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); + + +/* ---------- Buffer Interface ---------- */ + +/* It's zlib-like interface. + See LzmaDec_DecodeToDic description for information about STEPS and return results, + but you must use LzmaDec_DecodeToBuf instead of LzmaDec_DecodeToDic and you don't need + to work with CLzmaDec variables manually. + +finishMode: + It has meaning only if the decoding reaches output limit (*destLen). + LZMA_FINISH_ANY - Decode just destLen bytes. + LZMA_FINISH_END - Stream must be finished after (*destLen). +*/ + +SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); + + +/* ---------- One Call Interface ---------- */ + +/* LzmaDecode + +finishMode: + It has meaning only if the decoding reaches output limit (*destLen). + LZMA_FINISH_ANY - Decode just destLen bytes. + LZMA_FINISH_END - Stream must be finished after (*destLen). + +Returns: + SZ_OK + status: + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + SZ_ERROR_DATA - Data error + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_UNSUPPORTED - Unsupported properties + SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src). +*/ + +SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, + ELzmaStatus *status, ISzAlloc *alloc); + +EXTERN_C_END + +#endif diff --git a/src/lzma/Precomp.h b/src/lzma/Precomp.h new file mode 100644 index 0000000..3f3f6dc --- /dev/null +++ b/src/lzma/Precomp.h @@ -0,0 +1,16 @@ +/* Precomp.h -- StdAfx +2013-11-12 : Igor Pavlov : Public domain */ + +// No Copyright - Public Domain +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt + +#ifndef __7Z_PRECOMP_H +#define __7Z_PRECOMP_H + +#include "Compiler.h" +/* #include "7zTypes.h" */ + +#endif diff --git a/src/lzma_reader.cpp b/src/lzma_reader.cpp new file mode 100644 index 0000000..7215132 --- /dev/null +++ b/src/lzma_reader.cpp @@ -0,0 +1,109 @@ +// +// Copyright 2014, 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "lzma_reader.hpp" +#include "error.hpp" +#include "lzma/LzmaDec.h" +#include + +void* SzAlloc( void*, size_t size ) +{ + return malloc( size ); +} + +void SzFree( void*, void * address ) +{ + free( address ); +} + +static ISzAlloc s_alloc = { SzAlloc, SzFree }; + +lzma_reader::lzma_reader( basic_reader * pr ): pr_( pr ), k_( 0 ), m_( 0 ) +{ + typedef char assert_large_enough[ sizeof( state_ ) >= sizeof( CLzmaDec )? 1: -1 ]; + + { + std::size_t r = pr_->read( header_, sizeof( header_ ) ); + + if( r < sizeof( header_ ) ) + { + throw_error( pr_->name(), "could not read LZMA header" ); + } + } + + CLzmaDec * st = (CLzmaDec*)state_; + + LzmaDec_Construct( st ); + + { + SRes r = LzmaDec_Allocate( st, header_, LZMA_PROPS_SIZE, &s_alloc ); + + if( r != SZ_OK ) + { + throw_error( pr_->name(), "could not allocate LZMA state" ); + } + } + + LzmaDec_Init( st ); +} + +lzma_reader::~lzma_reader() +{ + CLzmaDec * st = (CLzmaDec*)state_; + LzmaDec_Free( st, &s_alloc ); +} + +std::string lzma_reader::name() const +{ + return pr_->name(); +} + +std::size_t lzma_reader::read( void * p, std::size_t n ) +{ + CLzmaDec * st = (CLzmaDec*)state_; + unsigned char * p2 = static_cast< unsigned char* >( p ); + + std::size_t r2 = 0; + + while( n > 0 ) + { + if( m_ == 0 ) + { + std::size_t r = pr_->read( buffer_, N ); + + if( r == 0 ) return r2; + + k_ = 0; + m_ = r; + } + + { + std::size_t n2 = n; + std::size_t m2 = m_; + + ELzmaStatus status = LZMA_STATUS_NOT_SPECIFIED; + + SRes r = LzmaDec_DecodeToBuf( st, p2, &n2, buffer_ + k_, &m2, LZMA_FINISH_ANY, &status ); + + if( r != 0 ) + { + return -( 1000 + r ); + } + + k_ += m2; + m_ -= m2; + + p2 += n2; + n -= n2; + + r2 += n2; + } + } + + return r2; +} diff --git a/src/lzma_reader.hpp b/src/lzma_reader.hpp new file mode 100644 index 0000000..04991d8 --- /dev/null +++ b/src/lzma_reader.hpp @@ -0,0 +1,43 @@ +#ifndef LZMA_READER_HPP_INCLUDED +#define LZMA_READER_HPP_INCLUDED + +// +// Copyright 2014 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "basic_reader.hpp" + +class lzma_reader: public basic_reader +{ +private: + + basic_reader * pr_; + + unsigned char header_[ 13 ]; + unsigned char state_[ 128 ]; + + static int const N = 4096; + + unsigned char buffer_[ N ]; + int k_, m_; + +private: + + lzma_reader( lzma_reader const & ); + lzma_reader& operator=( lzma_reader const & ); + +public: + + explicit lzma_reader( basic_reader * pr ); + + ~lzma_reader(); + + virtual std::string name() const; + virtual std::size_t read( void * p, std::size_t n ); +}; + +#endif // #ifndef LZMA_READER_HPP_INCLUDED diff --git a/src/message.cpp b/src/message.cpp new file mode 100644 index 0000000..977d42f --- /dev/null +++ b/src/message.cpp @@ -0,0 +1,50 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "message.hpp" +#include +#include + +static int s_level; + +int get_message_level() +{ + return s_level; +} + +void set_message_level( int level ) +{ + s_level = level; +} + +void increase_message_level() +{ + ++s_level; +} + +void decrease_message_level() +{ + --s_level; +} + +void msg_printf( int level, char const * format, ... ) +{ + if( level <= s_level ) + { + fprintf( stderr, "bpm: " ); + + va_list args; + va_start( args, format ); + + vfprintf( stderr, format, args ); + + va_end( args ); + + fprintf( stderr, "\n" ); + } +} diff --git a/src/message.hpp b/src/message.hpp new file mode 100644 index 0000000..3c868ac --- /dev/null +++ b/src/message.hpp @@ -0,0 +1,20 @@ +#ifndef MESSAGE_HPP_INCLUDED +#define MESSAGE_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +int get_message_level(); +void set_message_level( int level ); + +void increase_message_level(); +void decrease_message_level(); + +void msg_printf( int level, char const * format, ... ); + +#endif // #ifndef MESSAGE_HPP_INCLUDED diff --git a/src/options.cpp b/src/options.cpp new file mode 100644 index 0000000..9af0e42 --- /dev/null +++ b/src/options.cpp @@ -0,0 +1,40 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "options.hpp" + +void parse_options( char const * * & argv, void (*handle_option)( std::string const & opt ) ) +{ + while( argv[ 0 ] && ( argv[ 0 ][ 0 ] == '-' || argv[ 0 ][ 0 ] == '+' ) ) + { + if( argv[ 0 ][ 0 ] == '-' && argv[ 0 ][ 1 ] == '-' ) + { + if( argv[ 0 ][ 2 ] == 0 ) + { + // -- turns off option processing + + ++argv; + break; + } + else + { + handle_option( argv[ 0 ] ); + } + } + else + { + for( char const * p = argv[ 0 ] + 1; *p; ++p ) + { + char opt[ 3 ] = { argv[ 0 ][ 0 ], *p, 0 }; + handle_option( opt ); + } + } + + ++argv; + } +} diff --git a/src/options.hpp b/src/options.hpp new file mode 100644 index 0000000..e29635b --- /dev/null +++ b/src/options.hpp @@ -0,0 +1,16 @@ +#ifndef OPTIONS_HPP_INCLUDED +#define OPTIONS_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include + +void parse_options( char const * * & argv, void (*handle_option)( std::string const & opt ) ); + +#endif // #ifndef OPTIONS_HPP_INCLUDED diff --git a/src/package_path.cpp b/src/package_path.cpp new file mode 100644 index 0000000..38195b4 --- /dev/null +++ b/src/package_path.cpp @@ -0,0 +1,23 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "package_path.hpp" +#include "config.hpp" +#include + +std::string get_package_path() +{ + std::string path = config_get_option( "package_path" ); + + if( path.substr( 0, 7 ) != "http://" || *path.rbegin() != '/' ) + { + throw std::runtime_error( "invalid package_path '" + path + "' in bpm.conf" ); + } + + return path; +} diff --git a/src/package_path.hpp b/src/package_path.hpp new file mode 100644 index 0000000..77ace3d --- /dev/null +++ b/src/package_path.hpp @@ -0,0 +1,16 @@ +#ifndef PACKAGE_PATH_HPP_INCLUDED +#define PACKAGE_PATH_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include + +std::string get_package_path(); + +#endif // #ifndef PACKAGE_PATH_HPP_INCLUDED diff --git a/src/string.cpp b/src/string.cpp new file mode 100644 index 0000000..151a22f --- /dev/null +++ b/src/string.cpp @@ -0,0 +1,19 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "string.hpp" + +void remove_trailing( std::string & s, char ch ) +{ + std::string::iterator last = s.end(); + + if( last != s.begin() && ( --last, *last == ch ) ) + { + s.erase( last ); + } +} diff --git a/src/string.hpp b/src/string.hpp new file mode 100644 index 0000000..9af791c --- /dev/null +++ b/src/string.hpp @@ -0,0 +1,16 @@ +#ifndef STRING_HPP_INCLUDED +#define STRING_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include + +void remove_trailing( std::string & s, char ch ); + +#endif // #ifndef STRING_HPP_INCLUDED diff --git a/src/tar.cpp b/src/tar.cpp new file mode 100644 index 0000000..c493694 --- /dev/null +++ b/src/tar.cpp @@ -0,0 +1,300 @@ +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "tar.hpp" +#include "error.hpp" +#include "fs.hpp" +#include "message.hpp" +#include +#include +#include +#include +#include +#include + +static void throw_eof_error( std::string const & name ) +{ + throw_error( name, "unexpected end of file" ); +} + +static bool contains_dotdot( std::string const & fn ) +{ + std::size_t i = 0; + + for( ;; ) + { + std::size_t j = fn.find( '/', i ); + + if( j == std::string::npos ) + { + return fn.substr( i ) == ".."; + } + + if( fn.substr( i, j - i ) == ".." ) + { + return true; + } + + i = j + 1; + } +} + +int const N = 512; + +static bool read_header( basic_reader * pr, char (&header)[ N ], std::string & fn, char & type, long long & size, int & mode, long long & mtime ) +{ + { + std::size_t r = pr->read( header, N ); + + if( r == 0 ) return true; // EOF + + if( r < N ) + { + throw_eof_error( pr->name() ); + } + } + + if( header[0] == 0 ) + { + // empty block + + unsigned m = 0; + + for( int i = 0; i < N; ++i ) + { + m |= header[ i ]; + } + + if( m != 0 ) + { + throw_error( pr->name(), "bad block" ); + } + + return false; + } + + // check header checksum + + int checksum = 0; + std::sscanf( header + 148, "%8o", &checksum ); + + { + int s = 0; + + std::memset( header + 148, ' ', 8 ); + + for( int i = 0; i < N; ++i ) + { + s += static_cast< unsigned char >( header[ i ] ); + } + + if( s != checksum ) + { + throw_error( pr->name(), "header checksum mismatch" ); + } + } + + header[ 99 ] = 0; + fn = header; + + header[ 257 + 5 ] = 0; + std::string magic( header + 257 ); + + if( magic == "ustar" ) + { + header[ 345 + 130 ] = 0; + std::string prefix( header + 345 ); + + fn = prefix + fn; + } + + type = header[ 156 ]; + + size = 0; + std::sscanf( header + 124, "%12llo", &size ); + + mode = 0; + std::sscanf( header + 100, "%8o", &mode ); + + mtime = 0; + std::sscanf( header + 136, "%12llo", &mtime ); + + return false; +} + +static void read_long_name( basic_reader * pr, long long size, std::string & fn ) +{ + fn.clear(); + + long long k = 0; + + while( k < size ) + { + char data[ N ]; + + { + std::size_t r = pr->read( data, N ); + + if( r < N ) + { + throw_eof_error( pr->name() ); + } + } + + unsigned m; + + if( k + N <= size ) + { + m = N; + } + else + { + assert( size - k <= UINT_MAX ); + assert( size - k <= N ); + + m = static_cast< unsigned >( size - k ); + } + + fn.append( data, m ); + + k += N; + } +} + +void tar_extract( basic_reader * pr, std::string const & prefix, std::set< std::string > const & whitelist ) +{ + msg_printf( 1, "extracting from '%s'", pr->name().c_str() ); + + for( ;; ) + { + char header[ N ]; + + std::string fn; + + char type = 0; + long long size = 0; + int mode = 0; + long long mtime = 0; + + if( read_header( pr, header, fn, type, size, mode, mtime ) ) + { + break; // EOF + } + + if( header[ 0 ] == 0 ) + { + continue; + } + + if( type == 'L' ) + { + // GNU long name extension + + read_long_name( pr, size, fn ); + + std::string fn2; + + if( read_header( pr, header, fn2, type, size, mode, mtime ) ) + { + throw_eof_error( pr->name() ); + } + } + + if( ( fn.substr( 0, prefix.size() ) != prefix && whitelist.count( fn ) == 0 ) || contains_dotdot( fn ) ) + { + throw_error( pr->name(), "disallowed file name: '" + fn + "'" ); + } + + if( type != 0 && type != '0' && type != '5' ) + { + throw_error( pr->name(), "disallowed file type" ); + } + + msg_printf( 2, "extracting '%s'", fn.c_str() ); + + if( type == '5' ) // directory + { + if( size != 0 ) + { + throw_error( pr->name(), "directory with nonzero size: '" + fn + "'" ); + } + + int r = fs_mkdir( fn, 0755 ); + + if( r < 0 ) + { + throw_errno_error( fn, "create error", errno ); + } + + if( fn != prefix ) + { + fs_utime( fn, mtime, std::time( 0 ) ); + } + } + else // regular file + { + int fd = fs_creat( fn, 0644 ); + + if( fd < 0 ) + { + throw_errno_error( fn, "create error", errno ); + } + + long long k = 0; + + while( k < size ) + { + char data[ N ]; + + { + std::size_t r = pr->read( data, N ); + + if( r < N ) + { + throw_eof_error( pr->name() ); + } + } + + unsigned m; + + if( k + N <= size ) + { + m = N; + } + else + { + assert( size - k <= UINT_MAX ); + assert( size - k <= N ); + + m = static_cast< unsigned >( size - k ); + } + + int r = fs_write( fd, data, m ); + + if( r < 0 ) + { + int r2 = errno; + + fs_close( fd ); + throw_errno_error( fn, "write error", r2 ); + } + + if( r != m ) + { + fs_close( fd ); + throw_errno_error( fn, "write error", ENOSPC ); + } + + k += N; + } + + fs_close( fd ); + fs_utime( fn, mtime, std::time( 0 ) ); + } + } +} diff --git a/src/tar.hpp b/src/tar.hpp new file mode 100644 index 0000000..ece9ad9 --- /dev/null +++ b/src/tar.hpp @@ -0,0 +1,18 @@ +#ifndef TAR_HPP_INCLUDED +#define TAR_HPP_INCLUDED + +// +// Copyright 2015 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "basic_reader.hpp" +#include +#include + +void tar_extract( basic_reader * pr, std::string const & prefix, std::set< std::string > const & whitelist ); + +#endif // #ifndef TAR_HPP_INCLUDED diff --git a/src/tcp_reader.cpp b/src/tcp_reader.cpp new file mode 100644 index 0000000..40dcf7f --- /dev/null +++ b/src/tcp_reader.cpp @@ -0,0 +1,161 @@ +// +// Copyright 2014 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "tcp_reader.hpp" +#include "error.hpp" +#include +#include + +#ifdef _WIN32 +# include +#else + +#include +#include +#include +#include +#include +#include +#include +#include + +typedef int SOCKET; + +int const INVALID_SOCKET = -1; + +inline int closesocket( SOCKET s ) +{ + return close( s ); +} + +inline int WSAGetLastError() +{ + return errno; +} + +#endif + +static bool is_dotted_numeric( char const * p ) +{ + for( ; *p; ++p ) + { + if( !( *p >= '0' && *p <= '9' ) && *p != '.' ) + { + return false; + } + } + + return true; +} + +tcp_reader::tcp_reader( std::string const & host, int port ) +{ + { + char buffer[ 32 ]; + std::sprintf( buffer, "%d", port ); + + name_ = host + ':' + buffer; + } + + if( port <= 0 || port >= 65536 ) + { + throw_error( name_, "invalid port number" ); + } + + unsigned long a; + + char const * p = host.c_str(); + + if( is_dotted_numeric( p ) ) + { + a = inet_addr( p ); + + if( a == INADDR_NONE || a == 0 ) + { + throw_error( host, "invalid host address" ); + } + } + else + { + hostent const * q = gethostbyname( p ); + + if( q == 0 || q->h_length != 4 ) + { + throw_error( host, "unable to resolve host name" ); + } + + a = *reinterpret_cast( q->h_addr_list[0] ); + } + + sk_ = ::socket( AF_INET, SOCK_STREAM, IPPROTO_TCP ); + + if( sk_ == INVALID_SOCKET ) + { + throw_socket_error( name_, "socket create error", WSAGetLastError() ); + } + + sockaddr_in addr = { AF_INET }; + + addr.sin_addr.s_addr = a; + addr.sin_port = htons( static_cast( port ) ); + + int r = ::connect( sk_, (sockaddr const *)&addr, sizeof(addr) ); + + if( r != 0 ) + { + int r2 = WSAGetLastError(); + + closesocket( sk_ ); + + throw_socket_error( name_, "TCP connect error", r2 ); + } +} + +tcp_reader::~tcp_reader() +{ + shutdown( sk_, 2 ); + closesocket( sk_ ); +} + +std::string tcp_reader::name() const +{ + return name_; +} + +std::size_t tcp_reader::read( void * p, std::size_t n ) +{ + assert( n <= INT_MAX ); + + int r = ::recv( sk_, static_cast< char* >( p ), n, 0 ); + + if( r < 0 ) + { + throw_socket_error( name(), "TCP receive error", WSAGetLastError() ); + } + + return r; +} + +void tcp_reader::write( void const * p, std::size_t n ) +{ + assert( n <= INT_MAX ); + + int r = ::send( sk_, static_cast< char const* >( p ), n, 0 ); + + if( r < 0 || r < n ) + { + throw_socket_error( name(), "TCP send error", WSAGetLastError() ); + } +} + +#if defined( _WIN32 ) + +static WSADATA s_wd; +static int s_wsa_startup = WSAStartup( MAKEWORD( 1, 1 ), &s_wd ); + +#endif // defined( _WIN32 ) diff --git a/src/tcp_reader.hpp b/src/tcp_reader.hpp new file mode 100644 index 0000000..594b301 --- /dev/null +++ b/src/tcp_reader.hpp @@ -0,0 +1,40 @@ +#ifndef TCP_READER_HPP_INCLUDED +#define TCP_READER_HPP_INCLUDED + +// +// Copyright 2014 Peter Dimov +// +// Distributed under the Boost Software License, Version 1.0. +// See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt +// + +#include "basic_reader.hpp" + +class tcp_reader: public basic_reader +{ +private: + + std::ptrdiff_t sk_; + std::string name_; + +private: + + tcp_reader( tcp_reader const & ); + tcp_reader& operator=( tcp_reader const & ); + +public: + + explicit tcp_reader( std::string const & host, int port ); + + ~tcp_reader(); + + virtual std::string name() const; + virtual std::size_t read( void * p, std::size_t n ); + +protected: + + void write( void const * p, std::size_t n ); +}; + +#endif // #ifndef TCP_READER_HPP_INCLUDED