From 2bcd6d088b35fdd4782742d377d8ce04db960f91 Mon Sep 17 00:00:00 2001 From: Emil Dotchevski Date: Sun, 17 Nov 2019 19:37:33 -0800 Subject: [PATCH] Benchmark tweaks --- benchmark/deep_stack_leaf.cpp | 184 ++++++++++++++++++------------- benchmark/deep_stack_outcome.cpp | 149 ++++++++++++++++--------- 2 files changed, 209 insertions(+), 124 deletions(-) diff --git a/benchmark/deep_stack_leaf.cpp b/benchmark/deep_stack_leaf.cpp index bf63a16..c44c136 100644 --- a/benchmark/deep_stack_leaf.cpp +++ b/benchmark/deep_stack_leaf.cpp @@ -27,36 +27,36 @@ # include #endif #include +#include #include #include #include +#ifndef LEAF_NO_EXCEPTIONS +# error Please disable exception handling. +#endif + +namespace boost +{ + void throw_exception( std::exception const & e ) + { + std::cerr << "Terminating due to a C++ exception under LEAF_NO_EXCEPTIONS: " << e.what(); + std::terminate(); + } +} + +////////////////////////////////////// + namespace leaf = boost::leaf; -// Disable inlining for correct benchmarking. #ifdef _MSC_VER # define NOINLINE __declspec(noinline) #else # define NOINLINE __attribute__((noinline)) #endif -#ifdef LEAF_NO_EXCEPTIONS -// leaf::result::value() calls throw_exception to throw if it has no value. -// When exceptions are disabled, throw_exception is left undefined, and the user -// is required to define it (it may not return). -namespace boost -{ - [[noreturn]] void throw_exception( std::exception const & e ) - { - std::cerr << "Terminating due to a C++ exception under LEAF_NO_EXCEPTIONS: " << e.what(); - std::terminate(); - } -} -#endif - ////////////////////////////////////// -// A simple error code type. enum class e_error_code { ec1=1, @@ -69,7 +69,6 @@ namespace boost { namespace leaf { } } -// This error type has a large size and takes at least a second to init or move. // Note: in LEAF, handling of error objects is O(1) no matter how many stack frames. struct e_heavy_payload { @@ -77,63 +76,91 @@ struct e_heavy_payload e_heavy_payload() noexcept { - std::memset(value, 0, sizeof(value)); + std::memset(value, std::rand(), sizeof(value)); } }; +template +E make_error() noexcept; + +template <> +inline e_error_code make_error() noexcept +{ + return (std::rand()%2) ? e_error_code::ec1 : e_error_code::ec2; +} + +template <> +inline e_heavy_payload make_error() noexcept +{ + return e_heavy_payload(); +} + +inline bool should_fail( int failure_rate ) noexcept +{ + assert(failure_rate>=0); + assert(failure_rate<=100); + return (std::rand()%100) < failure_rate; +} + ////////////////////////////////////// -template +template struct benchmark_check_error_noinline { - NOINLINE static leaf::result f() noexcept + NOINLINE static leaf::result f( int failure_rate ) noexcept { - LEAF_AUTO(x, (benchmark_check_error_noinline::f())); + LEAF_AUTO(x, (benchmark_check_error_noinline::f(failure_rate))); return x+1; } }; -template -struct benchmark_check_error_noinline<0, T, E...> +template +struct benchmark_check_error_noinline<0, T, E> { - NOINLINE static leaf::result f() noexcept + NOINLINE static leaf::result f( int failure_rate ) noexcept { - return leaf::new_error(E{}...); + if( should_fail(failure_rate) ) + return leaf::new_error(make_error()); + else + return T{ }; } }; ////////////////////////////////////// -template +template struct benchmark_check_error_inline { - NOINLINE static leaf::result f() noexcept + static leaf::result f( int failure_rate ) noexcept { - LEAF_AUTO(x, (benchmark_check_error_inline::f())); + LEAF_AUTO(x, (benchmark_check_error_inline::f(failure_rate))); return x+1; } }; -template -struct benchmark_check_error_inline<0, T, E...> +template +struct benchmark_check_error_inline<0, T, E> { - NOINLINE static leaf::result f() noexcept + static leaf::result f( int failure_rate ) noexcept { - return leaf::new_error(E{}...); + if( should_fail(failure_rate) ) + return leaf::new_error(make_error()); + else + return T{ }; } }; ////////////////////////////////////// -template +template struct benchmark_handle_some_noinline { - NOINLINE static leaf::result f() noexcept + NOINLINE static leaf::result f( int failure_rate ) noexcept { return leaf::try_handle_some( - []() -> leaf::result + [=]() -> leaf::result { - LEAF_AUTO(x, (benchmark_handle_some_noinline::f())); + LEAF_AUTO(x, (benchmark_handle_some_noinline::f(failure_rate))); return x+1; }, []( leaf::match ) @@ -143,26 +170,29 @@ struct benchmark_handle_some_noinline } }; -template -struct benchmark_handle_some_noinline<0, T, E...> +template +struct benchmark_handle_some_noinline<0, T, E> { - NOINLINE static leaf::result f() noexcept + NOINLINE static leaf::result f( int failure_rate ) noexcept { - return leaf::new_error(E{}...); + if( should_fail(failure_rate) ) + return leaf::new_error(make_error()); + else + return T{ }; } }; ////////////////////////////////////// -template +template struct benchmark_handle_some_inline { - static leaf::result f() noexcept + static leaf::result f( int failure_rate ) noexcept { return leaf::try_handle_some( - []() -> leaf::result + [=]() -> leaf::result { - LEAF_AUTO(x, (benchmark_handle_some_inline::f())); + LEAF_AUTO(x, (benchmark_handle_some_inline::f(failure_rate))); return x+1; }, []( leaf::match ) @@ -172,24 +202,27 @@ struct benchmark_handle_some_inline } }; -template -struct benchmark_handle_some_inline<0, T, E...> +template +struct benchmark_handle_some_inline<0, T, E> { - static leaf::result f() noexcept + static leaf::result f( int failure_rate ) noexcept { - return leaf::new_error(E{}...); + if( should_fail(failure_rate) ) + return leaf::new_error(make_error()); + else + return T{ }; } }; ////////////////////////////////////// template -int runner() noexcept +int runner( int failure_rate ) noexcept { return leaf::try_handle_all( - [] + [=] { - return Benchmark::f(); + return Benchmark::f(failure_rate); }, []( e_error_code const & ) { @@ -215,7 +248,7 @@ int print_elapsed_time( F && f ) for( int i = 0; i!=Iterations; ++i ) val += std::forward(f)(); auto stop = std::chrono::system_clock::now(); - std::cout << std::setw(10) << std::chrono::duration_cast(stop-start).count(); + std::cout << std::setw(10) << std::chrono::duration_cast(stop-start).count(); return val; } @@ -224,30 +257,33 @@ int print_elapsed_time( F && f ) int main() { int const depth = 200; - int const iteration_count = 50000; + int const iteration_count = 5000; + int const test_rates[ ] = { 5, 50, 95 }; int x=0; - std::cout << - iteration_count << " iterations, call depth " << depth << ", sizeof(e_heavy_payload) = " << sizeof(e_heavy_payload) << ":" - "\n" - "\nError type | At each level | Inlining | Elapsed ms" - "\n----------------|--------------------|----------|-----------"; - std::cout << "\ne_error_code | LEAF_AUTO | Disabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_error_code | LEAF_AUTO | Enabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_error_code | try_handle_some | Disabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_error_code | try_handle_some | Enabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_heavy_payload | LEAF_AUTO | Disabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_heavy_payload | LEAF_AUTO | Enabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_heavy_payload | try_handle_some | Disabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_heavy_payload | try_handle_some | Enabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); + std::cout << iteration_count << " iterations, call depth " << depth << ", sizeof(e_heavy_payload) = " << sizeof(e_heavy_payload); + for( auto fr : test_rates ) + { + std::cout << "\n" + "\nError type | At each level | Inlining | Elapsed μs" + "\n----------------|--------------------|----------|----------- failure rate = " << fr << '%'; + std::cout << "\ne_error_code | LEAF_AUTO | Disabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_error_code | LEAF_AUTO | Enabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_error_code | try_handle_some | Disabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_error_code | try_handle_some | Enabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_heavy_payload | LEAF_AUTO | Disabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_heavy_payload | LEAF_AUTO | Enabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_heavy_payload | try_handle_some | Disabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_heavy_payload | try_handle_some | Enabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + }; std::cout << std::endl; return x; diff --git a/benchmark/deep_stack_outcome.cpp b/benchmark/deep_stack_outcome.cpp index dace823..06ae113 100644 --- a/benchmark/deep_stack_outcome.cpp +++ b/benchmark/deep_stack_outcome.cpp @@ -17,19 +17,34 @@ // // Benchmarking is run with inlining enabled as well as disabled. -#include +#include #include #include +#include #include #include #include +#ifndef BOOST_NO_EXCEPTIONS +# error Please disable exception handling. +#endif + +namespace boost +{ + void throw_exception( std::exception const & e ) + { + std::cerr << "Terminating due to a C++ exception under BOOST_NO_EXCEPTIONS: " << e.what(); + std::terminate(); + } +} + +////////////////////////////////////// + namespace outcome = boost::outcome_v2; template -using result = outcome::outcome; +using result = outcome::std_outcome; -// Disable inlining for correct benchmarking. #ifdef _MSC_VER # define NOINLINE __declspec(noinline) #else @@ -38,31 +53,50 @@ using result = outcome::outcome; ////////////////////////////////////// -// A simple error code type. enum class e_error_code { ec1=1, ec2 }; -// This error type has a large size and takes at least a second to init or move. -// Note: in LEAF, handling of error objects is O(1) no matter how many stack frames. struct e_heavy_payload { char value[4096]; e_heavy_payload() noexcept { - std::memset(value, 0, sizeof(value)); + std::memset(value, std::rand(), sizeof(value)); } }; -bool check_handle_some_value( e_error_code e ) +template +E make_error() noexcept; + +template <> +inline e_error_code make_error() noexcept +{ + return (std::rand()%2) ? e_error_code::ec1 : e_error_code::ec2; +} + +template <> +inline e_heavy_payload make_error() noexcept +{ + return e_heavy_payload(); +} + +inline bool should_fail( int failure_rate ) noexcept +{ + assert(failure_rate>=0); + assert(failure_rate<=100); + return (std::rand()%100) < failure_rate; +} + +bool check_handle_some_value( e_error_code e ) noexcept { return e==e_error_code::ec2; } -bool check_handle_some_value( e_heavy_payload const & ) +bool check_handle_some_value( e_heavy_payload const & ) noexcept { return false; } @@ -72,9 +106,9 @@ bool check_handle_some_value( e_heavy_payload const & ) template struct benchmark_check_error_noinline { - NOINLINE static result f() noexcept + NOINLINE static result f( int failure_rate ) noexcept { - BOOST_OUTCOME_TRY(x, (benchmark_check_error_noinline::f())); + BOOST_OUTCOME_TRY(x, (benchmark_check_error_noinline::f(failure_rate))); return x+1; } }; @@ -82,9 +116,12 @@ struct benchmark_check_error_noinline template struct benchmark_check_error_noinline<0, T, E> { - NOINLINE static result f() noexcept + static result f( int failure_rate ) noexcept { - return E{}; + if( should_fail(failure_rate) ) + return make_error(); + else + return T{ }; } }; @@ -93,9 +130,9 @@ struct benchmark_check_error_noinline<0, T, E> template struct benchmark_check_error_inline { - NOINLINE static result f() noexcept + static result f( int failure_rate ) noexcept { - BOOST_OUTCOME_TRY(x, (benchmark_check_error_noinline::f())); + BOOST_OUTCOME_TRY(x, (benchmark_check_error_inline::f(failure_rate))); return x+1; } }; @@ -103,9 +140,12 @@ struct benchmark_check_error_inline template struct benchmark_check_error_inline<0, T, E> { - NOINLINE static result f() noexcept + static result f( int failure_rate ) noexcept { - return E{}; + if( should_fail(failure_rate) ) + return make_error(); + else + return T{ }; } }; @@ -114,9 +154,9 @@ struct benchmark_check_error_inline<0, T, E> template struct benchmark_handle_some_noinline { - NOINLINE static result f() noexcept + NOINLINE static result f( int failure_rate ) noexcept { - if( auto r = benchmark_handle_some_noinline::f() ) + if( auto r = benchmark_handle_some_noinline::f(failure_rate) ) return r.value()+1; else if( check_handle_some_value(r.error()) ) return 1; @@ -128,9 +168,12 @@ struct benchmark_handle_some_noinline template struct benchmark_handle_some_noinline<0, T, E> { - NOINLINE static result f() noexcept + static result f( int failure_rate ) noexcept { - return E{}; + if( should_fail(failure_rate) ) + return make_error(); + else + return T{ }; } }; @@ -139,9 +182,9 @@ struct benchmark_handle_some_noinline<0, T, E> template struct benchmark_handle_some_inline { - static result f() noexcept + static result f( int failure_rate ) noexcept { - if( auto r = benchmark_handle_some_noinline::f() ) + if( auto r = benchmark_handle_some_inline::f(failure_rate) ) return r.value()+1; else if( check_handle_some_value(r.error()) ) return 1; @@ -153,18 +196,21 @@ struct benchmark_handle_some_inline template struct benchmark_handle_some_inline<0, T, E> { - static result f() noexcept + static result f( int failure_rate ) noexcept { - return E{}; + if( should_fail(failure_rate) ) + return make_error(); + else + return T{ }; } }; ////////////////////////////////////// template -int runner() noexcept +int runner( int failure_rate ) noexcept { - if( auto r = Benchmark::f() ) + if( auto r = Benchmark::f(failure_rate) ) return r.value(); else return -1; @@ -180,7 +226,7 @@ int print_elapsed_time( F && f ) for( int i = 0; i!=Iterations; ++i ) val += std::forward(f)(); auto stop = std::chrono::system_clock::now(); - std::cout << std::setw(10) << std::chrono::duration_cast(stop-start).count(); + std::cout << std::setw(10) << std::chrono::duration_cast(stop-start).count(); return val; } @@ -189,30 +235,33 @@ int print_elapsed_time( F && f ) int main() { int const depth = 200; - int const iteration_count = 50000; + int const iteration_count = 5000; + int const test_rates[ ] = { 5, 50, 95 }; int x=0; - std::cout << - iteration_count << " iterations, call depth " << depth << ", sizeof(e_heavy_payload) = " << sizeof(e_heavy_payload) << ":" - "\n" - "\nError type | At each level | Inlining | Elapsed ms" - "\n----------------|--------------------|----------|-----------"; - std::cout << "\ne_error_code | OUTCOME_TRY | Disabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_error_code | OUTCOME_TRY | Enabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_error_code | Handle some errors | Disabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_error_code | Handle some errors | Enabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_heavy_payload | OUTCOME_TRY | Disabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_heavy_payload | OUTCOME_TRY | Enabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_heavy_payload | Handle some errors | Disabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); - std::cout << "\ne_heavy_payload | Handle some errors | Enabled | "; - x += print_elapsed_time( [ ] { return runner>(); } ); + std::cout << iteration_count << " iterations, call depth " << depth << ", sizeof(e_heavy_payload) = " << sizeof(e_heavy_payload); + for( auto fr : test_rates ) + { + std::cout << "\n" + "\nError type | At each level | Inlining | Elapsed μs" + "\n----------------|--------------------|----------|----------- failure rate = " << fr << '%'; + std::cout << "\ne_error_code | OUTCOME_TRY | Disabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_error_code | OUTCOME_TRY | Enabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_error_code | Handle some errors | Disabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_error_code | Handle some errors | Enabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_heavy_payload | OUTCOME_TRY | Disabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_heavy_payload | OUTCOME_TRY | Enabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_heavy_payload | Handle some errors | Disabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + std::cout << "\ne_heavy_payload | Handle some errors | Enabled | "; + x += print_elapsed_time( [=] { return runner>(fr); } ); + }; std::cout << std::endl; return x;