From e699167d48ac5d95eb9443075664c819350d5484 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Tue, 17 Jan 2023 09:13:43 -0800 Subject: [PATCH 01/63] to_chars integer parser --- include/boost/charconv/to_chars.hpp | 79 +++++++++++++++++++++++++++-- src/to_chars.cpp | 6 --- 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 5362971..f5b136b 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -7,6 +7,10 @@ #define BOOST_CHARCONV_TO_CHARS_HPP_INCLUDED #include +#include +#include +#include +#include namespace boost { namespace charconv { @@ -15,19 +19,88 @@ namespace boost { namespace charconv { struct to_chars_result { char* ptr; + + // Values: + // 0 = no error + // EINVAL = invalid_argument + // ERANGE = result_out_of_range int ec; - friend bool operator==(const to_chars_result& lhs, const to_chars_result& rhs) + + constexpr friend bool operator==(const to_chars_result& lhs, const to_chars_result& rhs) noexcept { return lhs.ptr == rhs.ptr && lhs.ec == rhs.ec; } - friend bool operator!=(const to_chars_result& lhs, const to_chars_result& rhs) + constexpr friend bool operator!=(const to_chars_result& lhs, const to_chars_result& rhs) noexcept { return !(lhs == rhs); } }; -BOOST_CHARCONV_DECL to_chars_result to_chars(char* first, char* last, int value, int base = 10); +namespace detail { + +static constexpr std::array radix_table = {{ + '0', '0', '0', '1', '0', '2', '0', '3', '0', '4', + '0', '5', '0', '6', '0', '7', '0', '8', '0', '9', + '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', + '1', '5', '1', '6', '1', '7', '1', '8', '1', '9', + '2', '0', '2', '1', '2', '2', '2', '3', '2', '4', + '2', '5', '2', '6', '2', '7', '2', '8', '2', '9', + '3', '0', '3', '1', '3', '2', '3', '3', '3', '4', + '3', '5', '3', '6', '3', '7', '3', '8', '3', '9', + '4', '0', '4', '1', '4', '2', '4', '3', '4', '4', + '4', '5', '4', '6', '4', '7', '4', '8', '4', '9', + '5', '0', '5', '1', '5', '2', '5', '3', '5', '4', + '5', '5', '5', '6', '5', '7', '5', '8', '5', '9', + '6', '0', '6', '1', '6', '2', '6', '3', '6', '4', + '6', '5', '6', '6', '6', '7', '6', '8', '6', '9', + '7', '0', '7', '1', '7', '2', '7', '3', '7', '4', + '7', '5', '7', '6', '7', '7', '7', '8', '7', '9', + '8', '0', '8', '1', '8', '2', '8', '3', '8', '4', + '8', '5', '8', '6', '8', '7', '8', '8', '8', '9', + '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', + '9', '5', '9', '6', '9', '7', '9', '8', '9', '9' +}}; + +// TODO: Should we do a specialized base 10 algorithm, and then one for the rest? I think yes +// See: https://jk-jeon.github.io/posts/2022/02/jeaiii-algorithm/ +template +BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) +{ + std::snprintf( first, last - first - 1, "%d", value ); + return { first + std::strlen( first ), 0 }; +} + +// All other bases +template +BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value, int base) +{ + // Check pre-conditions + BOOST_CHARCONV_ASSERT_MSG(base >= 2 && base <= 36, "Base must be between 2 and 36 (inclusive)"); + if (!(first <= last)) + { + return {last, EINVAL}; + } + + std::snprintf( first, last - first - 1, "%d", value ); + return { first + std::strlen( first ), 0 }; +} + +} // Namespace detail + +template ::value, bool>::type = true> +BOOST_CXX14_CONSTEXPR to_chars_result to_chars(char* first, char* last, Integer value, BOOST_ATTRIBUTE_UNUSED int base = 10) noexcept +{ + if (base == 10) + { + return detail::to_chars_integer_impl(first, last, value); + } + + return detail::to_chars_integer_impl(first, last, value, base); +} + +template <> +BOOST_CXX14_CONSTEXPR to_chars_result to_chars(char* first, char* last, bool value, int base) noexcept = delete; } // namespace charconv } // namespace boost diff --git a/src/to_chars.cpp b/src/to_chars.cpp index ec0e927..dce2a57 100644 --- a/src/to_chars.cpp +++ b/src/to_chars.cpp @@ -6,9 +6,3 @@ #include #include #include - -boost::charconv::to_chars_result boost::charconv::to_chars(char* first, char* last, int value, int /*base*/) -{ - std::snprintf( first, last - first - 1, "%d", value ); - return { first + std::strlen( first ), 0 }; -} From e0df5fb2025ed661a48cf4ab5d0240b41eacd01d Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Tue, 17 Jan 2023 09:17:19 -0800 Subject: [PATCH 02/63] Add missing header [ci skip] --- include/boost/charconv/to_chars.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index f5b136b..7e9ca2a 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace boost { namespace charconv { From fa957d7fe8522ad9a3e34f8ff0f338e9043d0317 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Tue, 17 Jan 2023 12:48:50 -0800 Subject: [PATCH 03/63] Add 32-bit base10 decomposition --- include/boost/charconv/to_chars.hpp | 53 +++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 7e9ca2a..b7b27a9 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace boost { namespace charconv { @@ -40,7 +41,7 @@ struct to_chars_result namespace detail { -static constexpr std::array radix_table = {{ + static constexpr char radix_table[] = { '0', '0', '0', '1', '0', '2', '0', '3', '0', '4', '0', '5', '0', '6', '0', '7', '0', '8', '0', '9', '1', '0', '1', '1', '1', '2', '1', '3', '1', '4', @@ -61,22 +62,62 @@ static constexpr std::array radix_table = {{ '8', '5', '8', '6', '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', '9', '6', '9', '7', '9', '8', '9', '9' -}}; +}; // TODO: Should we do a specialized base 10 algorithm, and then one for the rest? I think yes // See: https://jk-jeon.github.io/posts/2022/02/jeaiii-algorithm/ +// https://arxiv.org/abs/2101.11408 +BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) +{ + constexpr auto mask = (static_cast(1) << 57) - 1; // D = 57 so 2^D - 1 + constexpr auto magic_multiplier = static_cast(1441151881); // floor(2*D / 10*k) where D is 57 and k is 8 + auto y = value * magic_multiplier; + + for (std::size_t i {}; i < 10; i += 2) + { + std::memcpy(buffer + i, radix_table + static_cast(y >> 57) * 2, 2); + y &= mask; + y *= 100; + } + + return buffer + 10; +} + template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) -{ - std::snprintf( first, last - first - 1, "%d", value ); +{ + char buffer[10] {}; + + if (!(first <= last)) + { + return {last, EINVAL}; + } + + // TODO: Only use this for strlen < 10 + decompose32(value, buffer); + + std::size_t i {}; + while (buffer[i] == '0') + { + ++i; + } + + for (; i < 10; ++i) + { + *first = buffer[i]; + ++first; + } + + return {first, 0}; + + //std::snprintf( first, last - first - 1, "%d", value ); return { first + std::strlen( first ), 0 }; } // All other bases template -BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value, int base) +BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value, int base) noexcept { - // Check pre-conditions BOOST_CHARCONV_ASSERT_MSG(base >= 2 && base <= 36, "Base must be between 2 and 36 (inclusive)"); if (!(first <= last)) { From f5258519d66d1b3271ba8b221e46a246ceb278d0 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Tue, 17 Jan 2023 12:58:03 -0800 Subject: [PATCH 04/63] Replace memcpy and fix warning --- include/boost/charconv/to_chars.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index b7b27a9..8c6b0c8 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -75,7 +75,11 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) for (std::size_t i {}; i < 10; i += 2) { - std::memcpy(buffer + i, radix_table + static_cast(y >> 57) * 2, 2); + // Replaces std::memcpy(buffer + i, radix_table + static_cast(y >> 57) * 2, 2) + // since it would not be constexpr + const char* temp = {radix_table + static_cast(y >> 57) * 2}; + buffer[i] = temp[0]; + buffer[i+1] = temp[1]; y &= mask; y *= 100; } @@ -119,6 +123,7 @@ template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value, int base) noexcept { BOOST_CHARCONV_ASSERT_MSG(base >= 2 && base <= 36, "Base must be between 2 and 36 (inclusive)"); + (void)base; if (!(first <= last)) { return {last, EINVAL}; From 88d522553f0cfd0df9da97cc9af0c7c6da26c8ce Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 18 Jan 2023 07:41:43 -0800 Subject: [PATCH 05/63] More efficiently get rid of leading zeros --- include/boost/charconv/to_chars.hpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 8c6b0c8..2c86a63 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -90,6 +90,7 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) { + // decompose32 will always return 10 digits (including leading 0s) char buffer[10] {}; if (!(first <= last)) @@ -97,7 +98,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l return {last, EINVAL}; } - // TODO: Only use this for strlen < 10 + // TODO: Only use this for strlen <= 10 decompose32(value, buffer); std::size_t i {}; @@ -105,17 +106,9 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l { ++i; } + std::memcpy(first, buffer + i, sizeof(buffer) - i); - for (; i < 10; ++i) - { - *first = buffer[i]; - ++first; - } - - return {first, 0}; - - //std::snprintf( first, last - first - 1, "%d", value ); - return { first + std::strlen( first ), 0 }; + return {first + std::strlen(first), 0}; } // All other bases @@ -136,7 +129,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l } // Namespace detail template ::value, bool>::type = true> -BOOST_CXX14_CONSTEXPR to_chars_result to_chars(char* first, char* last, Integer value, BOOST_ATTRIBUTE_UNUSED int base = 10) noexcept +BOOST_CXX14_CONSTEXPR to_chars_result to_chars(char* first, char* last, Integer value, int base = 10) noexcept { if (base == 10) { From 807c3ea028775e98c9a64eb95a9e16727ecf37c6 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 18 Jan 2023 10:08:18 -0800 Subject: [PATCH 06/63] Add framework to decompose larger values --- include/boost/charconv/to_chars.hpp | 38 ++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 2c86a63..9c693dc 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -64,7 +65,6 @@ namespace detail { '9', '5', '9', '6', '9', '7', '9', '8', '9', '9' }; -// TODO: Should we do a specialized base 10 algorithm, and then one for the rest? I think yes // See: https://jk-jeon.github.io/posts/2022/02/jeaiii-algorithm/ // https://arxiv.org/abs/2101.11408 BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) @@ -98,16 +98,36 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l return {last, EINVAL}; } - // TODO: Only use this for strlen <= 10 - decompose32(value, buffer); - - std::size_t i {}; - while (buffer[i] == '0') + // If the type is less than 32 bits we can use this without change + // If the type is greater than 32 bits we use a binary search tree to figure out how many digits + // are present and then decompose the value into two (or more) std::uint32_t of known length so that we + // don't have the issue of removing leading zeros from the least significant digits + if (static_cast(value) <= (std::numeric_limits::max)()) { - ++i; - } - std::memcpy(first, buffer + i, sizeof(buffer) - i); + decompose32(value, buffer); + std::size_t i {}; + while (buffer[i] == '0') + { + ++i; + } + std::memcpy(first, buffer + i, sizeof(buffer) - i); + } + else if (static_cast(value) <= (std::numeric_limits::max)()) + { + + } + #if 0 + // unsigned __128 requires 4 shifts + // Could just recursivly call the uint64_t twice and then compose 2x 64 bits + else if (static_cast(value) <= (std::numeric_limits::max)()) + #endif + else + { + BOOST_CHARCONV_ASSERT_MSG(sizeof(Integer) < 1, "Your type is unsupported. Use a built-in integral type"); + } + + return {first + std::strlen(first), 0}; } From dea69677ebb863923591340a560bb34585ccbae7 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 18 Jan 2023 11:36:04 -0800 Subject: [PATCH 07/63] Add binary search trees for 32, 64 and 128 bit types --- .../charconv/detail/integer_search_trees.hpp | 233 ++++++++++++++++++ include/boost/charconv/to_chars.hpp | 1 + 2 files changed, 234 insertions(+) create mode 100644 include/boost/charconv/detail/integer_search_trees.hpp diff --git a/include/boost/charconv/detail/integer_search_trees.hpp b/include/boost/charconv/detail/integer_search_trees.hpp new file mode 100644 index 0000000..41f23fb --- /dev/null +++ b/include/boost/charconv/detail/integer_search_trees.hpp @@ -0,0 +1,233 @@ +// Copyright 2023 Matt Borland +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef BOOST_CHARCONV_DETAIL_INTEGER_SEARCH_TREES_HPP +#define BOOST_CHARCONV_DETAIL_INTEGER_SEARCH_TREES_HPP + +// https://stackoverflow.com/questions/1489830/efficient-way-to-determine-number-of-digits-in-an-integer?page=1&tab=scoredesc#tab-top +// https://graphics.stanford.edu/~seander/bithacks.html + +#include "../config.hpp" +#include +#include +#include + +// Generic solution +template +BOOST_CXX14_CONSTEXPR int num_digits(T x) noexcept +{ + int digits = 0; + + while (x) + { + x /= 10; + ++digits; + } + + return digits; +} + +template <> +BOOST_CXX14_CONSTEXPR int num_digits(std::uint32_t x) noexcept +{ + if (x >= UINT32_C(10000)) + { + if (x >= UINT32_C(10000000)) + { + if (x >= UINT32_C(100000000)) + { + if (x >= UINT32_C(1000000000)) + { + return 10; + } + return 9; + } + return 8; + } + + else if (x >= UINT32_C(100000)) + { + if (x >= UINT32_C(1000000)) + { + return 7; + } + return 6; + } + return 5; + } + else if (x >= UINT32_C(100)) + { + if (x >= UINT32_C(1000)) + { + return 4; + } + return 3; + } + else if (x >= UINT32_C(10)) + { + return 2; + } + + return 1; +} + +template <> +BOOST_CXX14_CONSTEXPR int num_digits(std::uint64_t x) noexcept +{ + if (x >= UINT64_C(10000000000)) + { + if (x >= UINT64_C(100000000000000)) + { + if (x >= UINT64_C(10000000000000000)) + { + if (x >= UINT64_C(100000000000000000)) + { + if (x >= UINT64_C(1000000000000000000)) + { + if (x >= UINT64_C(10000000000000000000)) + { + return 20; + } + return 19; + } + return 18; + } + return 17; + } + else if (x >= UINT64_C(1000000000000000)) + { + return 16; + } + return 15; + } + if (x >= UINT64_C(1000000000000)) + { + if (x >= UINT64_C(10000000000000)) + { + return 14; + } + return 13; + } + if (x >= UINT64_C(100000000000)) + { + return 12; + } + return 11; + } + else if (x >= UINT64_C(100000)) + { + if (x >= UINT64_C(10000000)) + { + if (x >= UINT64_C(100000000)) + { + if (x >= UINT64_C(1000000000)) + { + return 10; + } + return 9; + } + return 8; + } + if (x >= UINT64_C(1000000)) + { + return 7; + } + return 6; + } + if (x >= UINT64_C(100)) + { + if (x >= UINT64_C(1000)) + { + if (x >= UINT64_C(10000)) + { + return 5; + } + return 4; + } + return 3; + } + if (x >= UINT64_C(10)) + { + return 2; + } + return 1; +} + +#ifdef __GLIBCXX_TYPE_INT_N_0 +static constexpr std::array powers_of_10 = +{{ + 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, + 100000000000, 1000000000000, 10000000000000, 100000000000000, 1000000000000000, 10000000000000000, + 100000000000000000, 1000000000000000000, 10000000000000000000 +}}; + +// Assume that if someone is using 128 bit ints they are favoring the top end of the range +// Max value is 340,282,366,920,938,463,463,374,607,431,768,211,455 (39 digits) +BOOST_CXX14_CONSTEXPR int num_digits(unsigned __int128 x) noexcept +{ + // There is not literal for unsigned __int128 so we need to calculate them using the max value of the + // std::uint64_t powers of 10 + constexpr unsigned __int128 digits_39 = static_cast(powers_of_10[19]) * static_cast(powers_of_10[18]); + constexpr unsigned __int128 digits_38 = digits_39 / 10; + constexpr unsigned __int128 digits_37 = digits_38 / 10; + constexpr unsigned __int128 digits_36 = digits_37 / 10; + constexpr unsigned __int128 digits_35 = digits_36 / 10; + constexpr unsigned __int128 digits_34 = digits_35 / 10; + constexpr unsigned __int128 digits_33 = digits_34 / 10; + constexpr unsigned __int128 digits_32 = digits_33 / 10; + constexpr unsigned __int128 digits_31 = digits_32 / 10; + constexpr unsigned __int128 digits_30 = digits_31 / 10; + constexpr unsigned __int128 digits_29 = digits_30 / 10; + constexpr unsigned __int128 digits_28 = digits_29 / 10; + constexpr unsigned __int128 digits_27 = digits_28 / 10; + constexpr unsigned __int128 digits_26 = digits_27 / 10; + constexpr unsigned __int128 digits_25 = digits_26 / 10; + constexpr unsigned __int128 digits_24 = digits_25 / 10; + constexpr unsigned __int128 digits_23 = digits_24 / 10; + constexpr unsigned __int128 digits_22 = digits_23 / 10; + constexpr unsigned __int128 digits_21 = digits_22 / 10; + + return (x > digits_39) ? 39 : + (x > digits_38) ? 38 : + (x > digits_37) ? 37 : + (x > digits_36) ? 36 : + (x > digits_35) ? 35 : + (x > digits_34) ? 34 : + (x > digits_33) ? 33 : + (x > digits_32) ? 32 : + (x > digits_31) ? 31 : + (x > digits_30) ? 30 : + (x > digits_29) ? 29 : + (x > digits_28) ? 28 : + (x > digits_27) ? 27 : + (x > digits_26) ? 26 : + (x > digits_25) ? 25 : + (x > digits_24) ? 24 : + (x > digits_23) ? 23 : + (x > digits_22) ? 22 : + (x > digits_21) ? 21 : + (x > powers_of_10[19]) ? 20 : + (x > powers_of_10[18]) ? 19 : + (x > powers_of_10[17]) ? 18 : + (x > powers_of_10[16]) ? 17 : + (x > powers_of_10[15]) ? 16 : + (x > powers_of_10[14]) ? 15 : + (x > powers_of_10[13]) ? 14 : + (x > powers_of_10[12]) ? 13 : + (x > powers_of_10[11]) ? 12 : + (x > powers_of_10[10]) ? 11 : + (x > powers_of_10[9]) ? 10 : + (x > powers_of_10[8]) ? 9 : + (x > powers_of_10[7]) ? 8 : + (x > powers_of_10[6]) ? 7 : + (x > powers_of_10[5]) ? 6 : + (x > powers_of_10[4]) ? 5 : + (x > powers_of_10[3]) ? 4 : + (x > powers_of_10[2]) ? 3 : + (x > powers_of_10[1]) ? 2 : + (x > powers_of_10[0]) ? 1 : 0; +} +#endif // 128-bit support + +#endif // BOOST_CHARCONV_DETAIL_INTEGER_SEARCH_TREES_HPP diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 9c693dc..577745b 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -6,6 +6,7 @@ #ifndef BOOST_CHARCONV_TO_CHARS_HPP_INCLUDED #define BOOST_CHARCONV_TO_CHARS_HPP_INCLUDED +#include #include #include #include From e8db8b0bf4e182683f8a7cdbc5664f43688c83cc Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 18 Jan 2023 11:47:07 -0800 Subject: [PATCH 08/63] Fix init of power_of_10 array --- include/boost/charconv/detail/integer_search_trees.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/boost/charconv/detail/integer_search_trees.hpp b/include/boost/charconv/detail/integer_search_trees.hpp index 41f23fb..f014567 100644 --- a/include/boost/charconv/detail/integer_search_trees.hpp +++ b/include/boost/charconv/detail/integer_search_trees.hpp @@ -157,9 +157,10 @@ BOOST_CXX14_CONSTEXPR int num_digits(std::uint64_t x) noexcept #ifdef __GLIBCXX_TYPE_INT_N_0 static constexpr std::array powers_of_10 = {{ - 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, - 100000000000, 1000000000000, 10000000000000, 100000000000000, 1000000000000000, 10000000000000000, - 100000000000000000, 1000000000000000000, 10000000000000000000 + UINT64_C(1), UINT64_C(10), UINT64_C(100), UINT64_C(1000), UINT64_C(10000), UINT64_C(100000), UINT64_C(1000000), + UINT64_C(10000000), UINT64_C(100000000), UINT64_C(1000000000), UINT64_C(10000000000), UINT64_C(100000000000), + UINT64_C(1000000000000), UINT64_C(10000000000000), UINT64_C(100000000000000), UINT64_C(1000000000000000), + UINT64_C(10000000000000000), UINT64_C(100000000000000000), UINT64_C(1000000000000000000), UINT64_C(10000000000000000000) }}; // Assume that if someone is using 128 bit ints they are favoring the top end of the range From 0e5fa4c26b40976cc75da17947a3b353691d8637 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 18 Jan 2023 12:04:59 -0800 Subject: [PATCH 09/63] Replace std::array::operator[] since it is not constexpr in C++11 --- include/boost/charconv/detail/integer_search_trees.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/charconv/detail/integer_search_trees.hpp b/include/boost/charconv/detail/integer_search_trees.hpp index f014567..3ef1ff4 100644 --- a/include/boost/charconv/detail/integer_search_trees.hpp +++ b/include/boost/charconv/detail/integer_search_trees.hpp @@ -169,7 +169,7 @@ BOOST_CXX14_CONSTEXPR int num_digits(unsigned __int128 x) noexcept { // There is not literal for unsigned __int128 so we need to calculate them using the max value of the // std::uint64_t powers of 10 - constexpr unsigned __int128 digits_39 = static_cast(powers_of_10[19]) * static_cast(powers_of_10[18]); + constexpr unsigned __int128 digits_39 = static_cast(UINT64_C(10000000000000000000)) * static_cast(UINT64_C(1000000000000000000)); constexpr unsigned __int128 digits_38 = digits_39 / 10; constexpr unsigned __int128 digits_37 = digits_38 / 10; constexpr unsigned __int128 digits_36 = digits_37 / 10; From 9b350bcdcbca2f97f45c3477401d94e94bef3ab8 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 18 Jan 2023 13:34:33 -0800 Subject: [PATCH 10/63] Simplify 32-bit impl and improve logic --- .../charconv/detail/integer_search_trees.hpp | 7 ++++++- include/boost/charconv/to_chars.hpp | 21 +++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/include/boost/charconv/detail/integer_search_trees.hpp b/include/boost/charconv/detail/integer_search_trees.hpp index 3ef1ff4..30250de 100644 --- a/include/boost/charconv/detail/integer_search_trees.hpp +++ b/include/boost/charconv/detail/integer_search_trees.hpp @@ -13,6 +13,8 @@ #include #include +namespace boost { namespace charconv { namespace detail { + // Generic solution template BOOST_CXX14_CONSTEXPR int num_digits(T x) noexcept @@ -169,7 +171,8 @@ BOOST_CXX14_CONSTEXPR int num_digits(unsigned __int128 x) noexcept { // There is not literal for unsigned __int128 so we need to calculate them using the max value of the // std::uint64_t powers of 10 - constexpr unsigned __int128 digits_39 = static_cast(UINT64_C(10000000000000000000)) * static_cast(UINT64_C(1000000000000000000)); + constexpr unsigned __int128 digits_39 = static_cast(UINT64_C(10000000000000000000)) * + static_cast(UINT64_C(1000000000000000000)); constexpr unsigned __int128 digits_38 = digits_39 / 10; constexpr unsigned __int128 digits_37 = digits_38 / 10; constexpr unsigned __int128 digits_36 = digits_37 / 10; @@ -231,4 +234,6 @@ BOOST_CXX14_CONSTEXPR int num_digits(unsigned __int128 x) noexcept } #endif // 128-bit support +}}} // Namespace boost::charconv::detail + #endif // BOOST_CHARCONV_DETAIL_INTEGER_SEARCH_TREES_HPP diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 577745b..c7048e7 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -90,10 +90,7 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) -{ - // decompose32 will always return 10 digits (including leading 0s) - char buffer[10] {}; - +{ if (!(first <= last)) { return {last, EINVAL}; @@ -103,16 +100,18 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l // If the type is greater than 32 bits we use a binary search tree to figure out how many digits // are present and then decompose the value into two (or more) std::uint32_t of known length so that we // don't have the issue of removing leading zeros from the least significant digits - if (static_cast(value) <= (std::numeric_limits::max)()) + + // TODO: fix this logic for types that could contain values greater than std::uint32_t but don't + // and types smaller than which would overflow on casting + if (std::numeric_limits::digits <= std::numeric_limits::digits || + value <= static_cast((std::numeric_limits::max)())) { + char buffer[10] {}; + const auto num_sig_chars = num_digits(value); + decompose32(value, buffer); - std::size_t i {}; - while (buffer[i] == '0') - { - ++i; - } - std::memcpy(first, buffer + i, sizeof(buffer) - i); + std::memcpy(first, buffer + (sizeof(buffer) - num_sig_chars), num_sig_chars); } else if (static_cast(value) <= (std::numeric_limits::max)()) { From e55057214a29ad11cc882492cbbae3bb4f89c983 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 19 Jan 2023 10:24:10 -0800 Subject: [PATCH 11/63] Move buffer and add float stub to keep MSVC happy --- include/boost/charconv/to_chars.hpp | 9 ++++++--- src/to_chars.cpp | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index c7048e7..e3988fe 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -96,21 +96,21 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l return {last, EINVAL}; } + char buffer[10] {}; // If the type is less than 32 bits we can use this without change // If the type is greater than 32 bits we use a binary search tree to figure out how many digits // are present and then decompose the value into two (or more) std::uint32_t of known length so that we // don't have the issue of removing leading zeros from the least significant digits - // TODO: fix this logic for types that could contain values greater than std::uint32_t but don't - // and types smaller than which would overflow on casting if (std::numeric_limits::digits <= std::numeric_limits::digits || value <= static_cast((std::numeric_limits::max)())) { - char buffer[10] {}; const auto num_sig_chars = num_digits(value); decompose32(value, buffer); + // If constant evaluated use unrolled loop + // If not constant evaluated use memcpy std::memcpy(first, buffer + (sizeof(buffer) - num_sig_chars), num_sig_chars); } else if (static_cast(value) <= (std::numeric_limits::max)()) @@ -162,6 +162,9 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars(char* first, char* last, Integer template <> BOOST_CXX14_CONSTEXPR to_chars_result to_chars(char* first, char* last, bool value, int base) noexcept = delete; +// TODO: Not correct, but need to make MSVC happy while working on integers +BOOST_CHARCONV_DECL to_chars_result to_chars(char* first, char* last, float value) noexcept; + } // namespace charconv } // namespace boost diff --git a/src/to_chars.cpp b/src/to_chars.cpp index dce2a57..b59936c 100644 --- a/src/to_chars.cpp +++ b/src/to_chars.cpp @@ -6,3 +6,10 @@ #include #include #include + + +boost::charconv::to_chars_result boost::charconv::to_chars(char* first, char* last, float value) noexcept +{ + std::snprintf(first, last - first - 1, "%f", value); + return {first + std::strlen(first), 0}; +} From 9664ba51720753366f61532b501d4bbfc3e58339 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 19 Jan 2023 10:47:44 -0800 Subject: [PATCH 12/63] Disable warning C4127 --- include/boost/charconv/to_chars.hpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index e3988fe..809b7c9 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -88,6 +88,11 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) return buffer + 10; } +#ifdef BOOST_MSVC +# pragma warning(push) +# pragma warning(disable: 4127) +#endif + template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) { @@ -97,11 +102,14 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l } char buffer[10] {}; + // If the type is less than 32 bits we can use this without change // If the type is greater than 32 bits we use a binary search tree to figure out how many digits // are present and then decompose the value into two (or more) std::uint32_t of known length so that we // don't have the issue of removing leading zeros from the least significant digits + // Yields: warning C4127: conditional expression is constant becuase first half the expression is constant + // but we need to short circuit to avoid UB on the second half if (std::numeric_limits::digits <= std::numeric_limits::digits || value <= static_cast((std::numeric_limits::max)())) { @@ -109,7 +117,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l decompose32(value, buffer); - // If constant evaluated use unrolled loop + // TODO: If constant evaluated use unrolled loop // If not constant evaluated use memcpy std::memcpy(first, buffer + (sizeof(buffer) - num_sig_chars), num_sig_chars); } @@ -131,6 +139,10 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l return {first + std::strlen(first), 0}; } +#ifdef BOOST_MSVC +# pragma warning(pop) +#endif + // All other bases template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value, int base) noexcept From 69d09f5d6b4b7fdb9555c8ee651674860a7b3def Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 19 Jan 2023 11:50:30 -0800 Subject: [PATCH 13/63] Refactor training --- include/boost/charconv/to_chars.hpp | 8 ++--- test/to_chars.cpp | 50 ++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 809b7c9..6bae33f 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -68,7 +68,7 @@ namespace detail { // See: https://jk-jeon.github.io/posts/2022/02/jeaiii-algorithm/ // https://arxiv.org/abs/2101.11408 -BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) +BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexcept { constexpr auto mask = (static_cast(1) << 57) - 1; // D = 57 so 2^D - 1 constexpr auto magic_multiplier = static_cast(1441151881); // floor(2*D / 10*k) where D is 57 and k is 8 @@ -94,7 +94,7 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) #endif template -BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) +BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) noexcept { if (!(first <= last)) { @@ -149,13 +149,13 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l { BOOST_CHARCONV_ASSERT_MSG(base >= 2 && base <= 36, "Base must be between 2 and 36 (inclusive)"); (void)base; + (void)value; if (!(first <= last)) { return {last, EINVAL}; } - std::snprintf( first, last - first - 1, "%d", value ); - return { first + std::strlen( first ), 0 }; + return {first + std::strlen(first), 0}; } } // Namespace detail diff --git a/test/to_chars.cpp b/test/to_chars.cpp index 8b9a616..95d41f9 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -5,25 +5,45 @@ #include #include +#include #include +#include + +template +void simple_test() +{ + char buffer1[64] {}; + T v = 34; + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "34"); + + boost::charconv::to_chars_result r {r1.ptr, r1.ec}; + BOOST_TEST(r1 == r); + + char buffer2[64] {}; + T v2 = 12; + auto r2 = boost::charconv::to_chars(buffer2, buffer2 + sizeof(buffer2) - 1, v2); + BOOST_TEST(r1 != r2); + BOOST_TEST_EQ(r2.ec, 0); + BOOST_TEST_CSTR_EQ(buffer2, "12"); +} int main() { - char buffer[32] = {}; - - int v = 1048576; - auto r = boost::charconv::to_chars( buffer, buffer + sizeof( buffer ) - 1, v ); - - BOOST_TEST_EQ(r.ec, 0) && BOOST_TEST_CSTR_EQ(buffer, "1048576"); - BOOST_TEST(r == r); - - boost::charconv::to_chars_result r2 {r.ptr, 0}; - BOOST_TEST(r == r2); - - char buffer2[32] = {}; - - auto r3 = boost::charconv::to_chars(buffer2, buffer2 + sizeof(buffer) - 1, v); - BOOST_TEST(r != r3); + simple_test(); + simple_test(); + simple_test(); + simple_test(); + simple_test(); + simple_test(); + simple_test(); + simple_test(); + simple_test(); + simple_test(); + simple_test(); + simple_test(); + simple_test(); return boost::report_errors(); } From 37e301d310f883f673effa882221aaaa2d05e235 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 19 Jan 2023 12:17:07 -0800 Subject: [PATCH 14/63] Fix for warning C4244 --- include/boost/charconv/to_chars.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 6bae33f..1281f43 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -113,9 +113,10 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l if (std::numeric_limits::digits <= std::numeric_limits::digits || value <= static_cast((std::numeric_limits::max)())) { - const auto num_sig_chars = num_digits(value); + const auto converted_value = static_cast(value); + const auto num_sig_chars = num_digits(converted_value); - decompose32(value, buffer); + decompose32(converted_value, buffer); // TODO: If constant evaluated use unrolled loop // If not constant evaluated use memcpy From 6b41bb6e69a811b4d09cdf8033442f499a7f2e1e Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 19 Jan 2023 13:33:00 -0800 Subject: [PATCH 15/63] Add support for negative numbers --- include/boost/charconv/to_chars.hpp | 21 ++++++++++++++++++--- test/to_chars.cpp | 13 +++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 1281f43..b4e2cf6 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -96,12 +96,23 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexc template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) noexcept { + char buffer[10] {}; + BOOST_ATTRIBUTE_UNUSED bool is_negative = false; + if (!(first <= last)) { return {last, EINVAL}; } - - char buffer[10] {}; + + // Strip the sign from the value and apply at the end after parsing if the type is signed + BOOST_IF_CONSTEXPR (std::is_signed::value) + { + if (value < 0) + { + value = -value; + is_negative = true; + } + } // If the type is less than 32 bits we can use this without change // If the type is greater than 32 bits we use a binary search tree to figure out how many digits @@ -120,6 +131,11 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l // TODO: If constant evaluated use unrolled loop // If not constant evaluated use memcpy + if (is_negative) + { + *first++ = '-'; + } + std::memcpy(first, buffer + (sizeof(buffer) - num_sig_chars), num_sig_chars); } else if (static_cast(value) <= (std::numeric_limits::max)()) @@ -136,7 +152,6 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l BOOST_CHARCONV_ASSERT_MSG(sizeof(Integer) < 1, "Your type is unsupported. Use a built-in integral type"); } - return {first + std::strlen(first), 0}; } diff --git a/test/to_chars.cpp b/test/to_chars.cpp index 95d41f9..b96aeaf 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -9,6 +9,16 @@ #include #include +template +void negative_vals_test() +{ + char buffer1[10] {}; + T v = -4321; + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "-4321"); +} + template void simple_test() { @@ -45,5 +55,8 @@ int main() simple_test(); simple_test(); + negative_vals_test(); + negative_vals_test(); + return boost::report_errors(); } From 302e9f5f09fbd5a4bc3f1eeab12c8b45d325cb0c Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 20 Jan 2023 08:48:35 -0800 Subject: [PATCH 16/63] Workaround for warning C4146 --- include/boost/charconv/detail/apply_sign.hpp | 29 ++++++++++++++++++++ include/boost/charconv/from_chars.hpp | 13 +-------- include/boost/charconv/to_chars.hpp | 5 ++-- 3 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 include/boost/charconv/detail/apply_sign.hpp diff --git a/include/boost/charconv/detail/apply_sign.hpp b/include/boost/charconv/detail/apply_sign.hpp new file mode 100644 index 0000000..244c0bb --- /dev/null +++ b/include/boost/charconv/detail/apply_sign.hpp @@ -0,0 +1,29 @@ +// Copyright 2023 Matt Borland +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef BOOST_CHARCONV_DETAIL_APPLY_SIGN_HPP +#define BOOST_CHARCONV_DETAIL_APPLY_SIGN_HPP + +// Workaround for warning C4146: unary minus operator applied to unsigned type, result still unsigned +// Occurs using MSVC with pre-C++17 language standards + +#include + +namespace boost { namespace charconv { namespace detail { + +template ::value, bool>::type = true> +constexpr Integer apply_sign(Integer val) noexcept +{ + return -val; +} + +template ::value, bool>::type = true> +constexpr Integer apply_sign(Integer val) noexcept +{ + return val; +} + +}}} // Namespaces + +#endif // BOOST_CHARCONV_DETAIL_APPLY_SIGN_HPP diff --git a/include/boost/charconv/from_chars.hpp b/include/boost/charconv/from_chars.hpp index 9e5fcc6..4f8f040 100644 --- a/include/boost/charconv/from_chars.hpp +++ b/include/boost/charconv/from_chars.hpp @@ -6,6 +6,7 @@ #ifndef BOOST_CHARCONV_FROM_CHARS_HPP_INCLUDED #define BOOST_CHARCONV_FROM_CHARS_HPP_INCLUDED +#include #include #include #include @@ -67,18 +68,6 @@ constexpr unsigned char digit_from_char(char val) noexcept return uchar_values[static_cast(val)]; } -template ::value, bool>::type = true> -constexpr Integer apply_sign(Integer val) noexcept -{ - return -val; -} - -template ::value, bool>::type = true> -constexpr Integer apply_sign(Integer val) noexcept -{ - return val; -} - template BOOST_CXX14_CONSTEXPR boost::charconv::from_chars_result from_chars_integer_impl(const char* first, const char* last, Integer& value, int base) noexcept { diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index b4e2cf6..1503c29 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -6,6 +6,7 @@ #ifndef BOOST_CHARCONV_TO_CHARS_HPP_INCLUDED #define BOOST_CHARCONV_TO_CHARS_HPP_INCLUDED +#include #include #include #include @@ -109,7 +110,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l { if (value < 0) { - value = -value; + value = apply_sign(value); is_negative = true; } } @@ -119,7 +120,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l // are present and then decompose the value into two (or more) std::uint32_t of known length so that we // don't have the issue of removing leading zeros from the least significant digits - // Yields: warning C4127: conditional expression is constant becuase first half the expression is constant + // Yields: warning C4127: conditional expression is constant becuase first half of the expression is constant // but we need to short circuit to avoid UB on the second half if (std::numeric_limits::digits <= std::numeric_limits::digits || value <= static_cast((std::numeric_limits::max)())) From fe0260c4075c235017d8bacdd595c0f353c31c3f Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 20 Jan 2023 09:07:52 -0800 Subject: [PATCH 17/63] Add functions to pack and unpack 64 and 128 bit integers --- .../charconv/detail/integer_conversion.hpp | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 include/boost/charconv/detail/integer_conversion.hpp diff --git a/include/boost/charconv/detail/integer_conversion.hpp b/include/boost/charconv/detail/integer_conversion.hpp new file mode 100644 index 0000000..d262618 --- /dev/null +++ b/include/boost/charconv/detail/integer_conversion.hpp @@ -0,0 +1,44 @@ +// Copyright 2023 Matt Borland +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef BOOST_CHARCONV_DETAIL_CONCATENATE_HPP +#define BOOST_CHARCONV_DETAIL_CONCATENATE_HPP + +#include +#include +#include + +namespace boost { namespace charconv { namespace detail { + +constexpr std::uint64_t pack(std::uint32_t word1, std::uint32_t word2) noexcept +{ + return static_cast(word1) << 32 | word2; +} + +BOOST_CXX14_CONSTEXPR std::pair unpack(std::uint64_t value) +{ + auto x = static_cast(value >> 32); + auto y = static_cast(value); + + return std::make_pair(x, y); +} + +#ifdef __GLIBCXX_TYPE_INT_N_0 +constexpr unsigned __int128 pack(std::uint64_t word1, std::uint64_t word2) noexcept +{ + return static_cast(word1) << 64 | word2; +} + +BOOST_CXX14_CONSTEXPR std::pair unpack(unsigned __int128 value) +{ + auto x = static_cast(value >> 64); + auto y = static_cast(value); + + return std::make_pair(x, y); +} +#endif // defined(__GLIBCXX_TYPE_INT_N_0) + +}}} // Namespaces + +#endif // BOOST_CHARCONV_DETAIL_CONCATENATE_HPP From 52feaddc9df2f9510dd9e5414c940911a4440772 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 20 Jan 2023 10:41:15 -0800 Subject: [PATCH 18/63] 64bit impl framework --- .../charconv/detail/integer_conversion.hpp | 5 ++-- include/boost/charconv/to_chars.hpp | 25 ++++++++++++++++++- test/to_chars.cpp | 13 ++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/include/boost/charconv/detail/integer_conversion.hpp b/include/boost/charconv/detail/integer_conversion.hpp index d262618..093cb6a 100644 --- a/include/boost/charconv/detail/integer_conversion.hpp +++ b/include/boost/charconv/detail/integer_conversion.hpp @@ -7,6 +7,7 @@ #include #include +#include #include namespace boost { namespace charconv { namespace detail { @@ -18,8 +19,8 @@ constexpr std::uint64_t pack(std::uint32_t word1, std::uint32_t word2) noexcept BOOST_CXX14_CONSTEXPR std::pair unpack(std::uint64_t value) { - auto x = static_cast(value >> 32); - auto y = static_cast(value); + auto x = static_cast((value & UINT64_MAX) >> UINT64_C(32)); + auto y = static_cast(value & UINT32_MAX); return std::make_pair(x, y); } diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 1503c29..38ef425 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -8,10 +8,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -139,9 +141,30 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l std::memcpy(first, buffer + (sizeof(buffer) - num_sig_chars), num_sig_chars); } - else if (static_cast(value) <= (std::numeric_limits::max)()) + else if (std::numeric_limits::digits <= std::numeric_limits::digits || + static_cast(value) <= (std::numeric_limits::max)()) { + const auto converted_value = static_cast(value); + const auto num_sig_chars = num_digits(converted_value); + const auto pair_32bit_values = unpack(converted_value); + const int first_value_chars = num_digits(converted_value); + const int second_value_chars = num_digits(converted_value); + + // TODO: Remove when done with debugging + assert(first_value_chars + second_value_chars <= num_sig_chars); + + decompose32(pair_32bit_values.first, buffer); + + if (is_negative) + { + *first++ = '-'; + } + + std::memcpy(first, buffer + (sizeof(buffer) - first_value_chars), first_value_chars); + + decompose32(pair_32bit_values.second, buffer); + std::memcpy(first + first_value_chars, buffer, sizeof(buffer)); } #if 0 // unsigned __128 requires 4 shifts diff --git a/test/to_chars.cpp b/test/to_chars.cpp index b96aeaf..5cc6e3a 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -9,6 +9,16 @@ #include #include +template +void sixty_four_bit_tests() +{ + char buffer1[64] {}; + T v1 = -1234; + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "-1234"); +} + template void negative_vals_test() { @@ -58,5 +68,8 @@ int main() negative_vals_test(); negative_vals_test(); + sixty_four_bit_tests(); + sixty_four_bit_tests(); + return boost::report_errors(); } From 4765f06652bda1785f5b355efca61ed30c46a916 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 20 Jan 2023 11:23:10 -0800 Subject: [PATCH 19/63] Fixes to 64 bit testing --- include/boost/charconv/to_chars.hpp | 11 ++++++----- test/to_chars.cpp | 6 ++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 38ef425..c0f7279 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -147,14 +147,15 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l const auto converted_value = static_cast(value); const auto num_sig_chars = num_digits(converted_value); - const auto pair_32bit_values = unpack(converted_value); - const int first_value_chars = num_digits(converted_value); - const int second_value_chars = num_digits(converted_value); + const auto x = static_cast(converted_value / UINT64_C(10000000000)); + const auto y = static_cast(converted_value % UINT64_C(10000000000)); + const int first_value_chars = num_digits(x); + const int second_value_chars = num_digits(y); // TODO: Remove when done with debugging assert(first_value_chars + second_value_chars <= num_sig_chars); - decompose32(pair_32bit_values.first, buffer); + decompose32(x, buffer); if (is_negative) { @@ -163,7 +164,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l std::memcpy(first, buffer + (sizeof(buffer) - first_value_chars), first_value_chars); - decompose32(pair_32bit_values.second, buffer); + decompose32(y, buffer); std::memcpy(first + first_value_chars, buffer, sizeof(buffer)); } #if 0 diff --git a/test/to_chars.cpp b/test/to_chars.cpp index 5cc6e3a..48ef6c3 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -17,6 +17,12 @@ void sixty_four_bit_tests() auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1); BOOST_TEST_EQ(r1.ec, 0); BOOST_TEST_CSTR_EQ(buffer1, "-1234"); + + char buffer2[64] {}; + T v2 = 1234123412341234; + auto r2 = boost::charconv::to_chars(buffer2, buffer2 + sizeof(buffer2) - 1, v2); + BOOST_TEST_EQ(r2.ec, 0); + BOOST_TEST_CSTR_EQ(buffer2, "1234123412341234"); } template From d372d6ab0a447ce2be71fa61b256b503efc32088 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 20 Jan 2023 12:02:13 -0800 Subject: [PATCH 20/63] Remove unused variables --- include/boost/charconv/to_chars.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index c0f7279..f6a4448 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -145,15 +145,10 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l static_cast(value) <= (std::numeric_limits::max)()) { const auto converted_value = static_cast(value); - const auto num_sig_chars = num_digits(converted_value); const auto x = static_cast(converted_value / UINT64_C(10000000000)); const auto y = static_cast(converted_value % UINT64_C(10000000000)); const int first_value_chars = num_digits(x); - const int second_value_chars = num_digits(y); - - // TODO: Remove when done with debugging - assert(first_value_chars + second_value_chars <= num_sig_chars); decompose32(x, buffer); From 6a9eeacafe18c1b2a2a616265676a59573bc4690 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 20 Jan 2023 12:27:09 -0800 Subject: [PATCH 21/63] Add edge case tests --- test/to_chars.cpp | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/test/to_chars.cpp b/test/to_chars.cpp index 48ef6c3..c7090ff 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -13,18 +14,42 @@ template void sixty_four_bit_tests() { char buffer1[64] {}; - T v1 = -1234; + T v1 = T(-1234); auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1); BOOST_TEST_EQ(r1.ec, 0); BOOST_TEST_CSTR_EQ(buffer1, "-1234"); char buffer2[64] {}; - T v2 = 1234123412341234; + T v2 = T(1234123412341234); auto r2 = boost::charconv::to_chars(buffer2, buffer2 + sizeof(buffer2) - 1, v2); BOOST_TEST_EQ(r2.ec, 0); BOOST_TEST_CSTR_EQ(buffer2, "1234123412341234"); } +template <> +void sixty_four_bit_tests() +{ + char buffer1[64] {}; + std::uint64_t v1 = (std::numeric_limits::max)(); + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "18446744073709551615"); + + // Cutting this value in half would overflow a 32 bit unsigned for the back 10 digits + char buffer2[64] {}; + std::uint64_t v2 = UINT64_C(9999999999999999999); + auto r2 = boost::charconv::to_chars(buffer2, buffer2 + sizeof(buffer2) - 1, v2); + BOOST_TEST_EQ(r2.ec, 0); + BOOST_TEST_CSTR_EQ(buffer2, "9999999999999999999"); + + // Account for zeros in the back half of the split + char buffer3[64] {}; + std::uint64_t v3 = UINT64_C(10000000000000000000); + auto r3 = boost::charconv::to_chars(buffer3, buffer3 + sizeof(buffer3) - 1, v3); + BOOST_TEST_EQ(r3.ec, 0); + BOOST_TEST_CSTR_EQ(buffer3, "10000000000000000000"); +} + template void negative_vals_test() { @@ -76,6 +101,7 @@ int main() sixty_four_bit_tests(); sixty_four_bit_tests(); + sixty_four_bit_tests(); return boost::report_errors(); } From 4f46f6a31dcc81964282d46df8aaafaf5ba6e032 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 25 Jan 2023 13:00:54 -0800 Subject: [PATCH 22/63] Avoid overflow by only using 9 digits per uint32_t --- include/boost/charconv/to_chars.hpp | 54 +++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index f6a4448..43c8cc3 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -144,23 +144,57 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l else if (std::numeric_limits::digits <= std::numeric_limits::digits || static_cast(value) <= (std::numeric_limits::max)()) { - const auto converted_value = static_cast(value); + auto converted_value = static_cast(value); + const auto converted_value_digits = num_digits(converted_value); - const auto x = static_cast(converted_value / UINT64_C(10000000000)); - const auto y = static_cast(converted_value % UINT64_C(10000000000)); - const int first_value_chars = num_digits(x); - - decompose32(x, buffer); - if (is_negative) { *first++ = '-'; } - std::memcpy(first, buffer + (sizeof(buffer) - first_value_chars), first_value_chars); + // Only store 9 digits in each to avoid overflow + if (num_digits(converted_value) <= 18) + { + const auto x = static_cast(converted_value / UINT64_C(1000000000)); + const auto y = static_cast(converted_value % UINT64_C(1000000000)); + const int first_value_chars = num_digits(x); - decompose32(y, buffer); - std::memcpy(first + first_value_chars, buffer, sizeof(buffer)); + decompose32(x, buffer); + std::memcpy(first, buffer + (sizeof(buffer) - first_value_chars), first_value_chars); + + decompose32(y, buffer); + std::memcpy(first + first_value_chars, buffer + 1, sizeof(buffer) - 1); + } + else + { + const auto x = static_cast(converted_value / UINT64_C(100000000000)); + converted_value -= x * UINT64_C(100000000000); + const auto y = static_cast(converted_value / UINT64_C(100)); + const auto z = static_cast(converted_value % UINT64_C(100)); + + if (converted_value_digits == 19) + { + decompose32(x, buffer); + std::memcpy(first, buffer + 2, sizeof(buffer) - 2); + + decompose32(y, buffer); + std::memcpy(first + 8, buffer + 1, sizeof(buffer) - 1); + + decompose32(z, buffer); + std::memcpy(first + 17, buffer + 8, 2); + } + else // 20 + { + decompose32(x, buffer); + std::memcpy(first, buffer + 1, sizeof(buffer) - 1); + + decompose32(y, buffer); + std::memcpy(first + 9, buffer + 1, sizeof(buffer) - 1); + + decompose32(z, buffer); + std::memcpy(first + 18, buffer + 8, 2); + } + } } #if 0 // unsigned __128 requires 4 shifts From ff4b840217cb3d1c7dc0846db998e3c7a87575b6 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 25 Jan 2023 13:19:26 -0800 Subject: [PATCH 23/63] Fix MSVC casting warnings --- test/to_chars.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/to_chars.cpp b/test/to_chars.cpp index c7090ff..1d048e2 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -14,13 +14,13 @@ template void sixty_four_bit_tests() { char buffer1[64] {}; - T v1 = T(-1234); + T v1 = static_cast(-1234); auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1); BOOST_TEST_EQ(r1.ec, 0); BOOST_TEST_CSTR_EQ(buffer1, "-1234"); char buffer2[64] {}; - T v2 = T(1234123412341234); + T v2 = static_cast(1234123412341234LL); auto r2 = boost::charconv::to_chars(buffer2, buffer2 + sizeof(buffer2) - 1, v2); BOOST_TEST_EQ(r2.ec, 0); BOOST_TEST_CSTR_EQ(buffer2, "1234123412341234"); @@ -99,7 +99,6 @@ int main() negative_vals_test(); negative_vals_test(); - sixty_four_bit_tests(); sixty_four_bit_tests(); sixty_four_bit_tests(); From c7b2177145e76d09c39d8153e4c4faec20ff852e Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 26 Jan 2023 09:21:33 -0800 Subject: [PATCH 24/63] Add framework for is_constant_evaluated --- .../charconv/detail/is_constant_evaluated.hpp | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 include/boost/charconv/detail/is_constant_evaluated.hpp diff --git a/include/boost/charconv/detail/is_constant_evaluated.hpp b/include/boost/charconv/detail/is_constant_evaluated.hpp new file mode 100644 index 0000000..f5f8fe4 --- /dev/null +++ b/include/boost/charconv/detail/is_constant_evaluated.hpp @@ -0,0 +1,54 @@ +// Copyright John Maddock 2011-2021. +// Copyright Matt Borland 2023. +// Use, modification and distribution are subject to the +// Boost Software License, Version 1.0. (See accompanying file +// LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#ifndef BOOST_CHARCONV_TOOLS_IS_CONSTANT_EVALUATED_HPP +#define BOOST_CHARCONV_TOOLS_IS_CONSTANT_EVALUATED_HPP + +#include + +#ifdef __has_include +# if __has_include() +# include +# ifdef __cpp_lib_is_constant_evaluated +# include +# define BOOST_CHARCONV_HAS_IS_CONSTANT_EVALUATED +# endif +# endif +#endif + +#ifdef __has_builtin +# if __has_builtin(__builtin_is_constant_evaluated) && !defined(BOOST_NO_CXX14_CONSTEXPR) +# define BOOST_CHARCONV_HAS_BUILTIN_IS_CONSTANT_EVALUATED +# endif +#endif + +// +// MSVC also supports __builtin_is_constant_evaluated if it's recent enough: +// +#if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 192528326) +# define BOOST_CHARCONV_HAS_BUILTIN_IS_CONSTANT_EVALUATED +#endif + +// +// As does GCC-9: +// +#if !defined(BOOST_NO_CXX14_CONSTEXPR) && (__GNUC__ >= 9) && !defined(BOOST_CHARCONV_HAS_BUILTIN_IS_CONSTANT_EVALUATED) +# define BOOST_CHARCONV_HAS_BUILTIN_IS_CONSTANT_EVALUATED +#endif + +#if defined(BOOST_CHARCONV_HAS_IS_CONSTANT_EVALUATED) && !defined(BOOST_NO_CXX14_CONSTEXPR) +# define BOOST_CHARCONV_IS_CONSTANT_EVALUATED(x) std::is_constant_evaluated() +#elif defined(BOOST_CHARCONV_HAS_BUILTIN_IS_CONSTANT_EVALUATED) +# define BOOST_CHARCONV_IS_CONSTANT_EVALUATED(x) __builtin_is_constant_evaluated() +#elif !defined(BOOST_NO_CXX14_CONSTEXPR) && (__GNUC__ >= 6) +# define BOOST_CHARCONV_IS_CONSTANT_EVALUATED(x) __builtin_constant_p(x) +# define BOOST_CHARCONV_USING_BUILTIN_CONSTANT_P +#else +# define BOOST_CHARCONV_IS_CONSTANT_EVALUATED(x) false +# define BOOST_CHARCONV_NO_CONSTEXPR_DETECTION +#endif + +#endif // BOOST_CHARCONV_TOOLS_IS_CONSTANT_EVALUATED_HPP From 157729dad4879148155c3e5fe0eeb1ca1d367d1a Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 26 Jan 2023 09:32:19 -0800 Subject: [PATCH 25/63] Add character lookup table --- include/boost/charconv/to_chars.hpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 43c8cc3..d430950 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -67,7 +67,14 @@ namespace detail { '8', '5', '8', '6', '8', '7', '8', '8', '8', '9', '9', '0', '9', '1', '9', '2', '9', '3', '9', '4', '9', '5', '9', '6', '9', '7', '9', '8', '9', '9' -}; + }; + + static constexpr char digit_table[] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z' + }; // See: https://jk-jeon.github.io/posts/2022/02/jeaiii-algorithm/ // https://arxiv.org/abs/2101.11408 From 057df19bf7e01c004754162812df75775e330f68 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 26 Jan 2023 10:26:10 -0800 Subject: [PATCH 26/63] Check STL handling of signed 0 --- test/Jamfile | 2 ++ test/to_chars_integer_STL_comp.cpp | 55 ++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 test/to_chars_integer_STL_comp.cpp diff --git a/test/Jamfile b/test/Jamfile index 5bdc53b..f85981a 100644 --- a/test/Jamfile +++ b/test/Jamfile @@ -1,4 +1,5 @@ # Copyright 2022 Peter Dimov +# Copyright 2023 Matt Borland # Distributed under the Boost Software License, Version 1.0. # https://www.boost.org/LICENSE_1_0.txt @@ -21,3 +22,4 @@ run from_chars.cpp ; run to_chars.cpp ; run roundtrip.cpp ; run from_chars_STL_comp.cpp : : : [ requires cxx17_std_apply ] ; +run to_chars_integer_STL_comp.cpp : : : [ requires cxx17_std_apply ] ; diff --git a/test/to_chars_integer_STL_comp.cpp b/test/to_chars_integer_STL_comp.cpp new file mode 100644 index 0000000..cb66377 --- /dev/null +++ b/test/to_chars_integer_STL_comp.cpp @@ -0,0 +1,55 @@ +// Copyright 2023 Matt Borland +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#include + +#if !defined(BOOST_NO_CXX17_HDR_CHARCONV) && (!defined(__clang_major__) || (defined(__clang_major__) && __clang_major__ > 7)) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +template +void test() +{ + // Signed 0 + char buffer1[64] {}; + T v1 = static_cast(-0); + auto r1 = std::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1); + (void)r1; + BOOST_TEST_CSTR_EQ(buffer1, "0"); +} + +int main() +{ + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + test(); + + return boost::report_errors(); +} + +#else + +int main() +{ + return 0; +} + +#endif From d7d278bcd590a20c0a75c22fa0ec82722d726861 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 26 Jan 2023 11:26:30 -0800 Subject: [PATCH 27/63] Impl for all other bases [ci skip] --- include/boost/charconv/to_chars.hpp | 95 ++++++++++++++++++++++++++++- test/to_chars.cpp | 12 ++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index d430950..53798b8 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -103,6 +103,8 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexc # pragma warning(disable: 4127) #endif +// TODO: Use a temp buffer to hold everything and then write it all into the provided buffer at the end +// If the temp buffer is larger than the provided buffer return EOVERFLOW template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) noexcept { @@ -221,18 +223,105 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l #endif // All other bases +// Use a simple lookup table to put together the Integer in character form template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value, int base) noexcept { BOOST_CHARCONV_ASSERT_MSG(base >= 2 && base <= 36, "Base must be between 2 and 36 (inclusive)"); - (void)base; - (void)value; + + using Unsigned_Integer = typename std::make_unsigned::type; + + const auto output_length = last - first; + if (!(first <= last)) { return {last, EINVAL}; } - return {first + std::strlen(first), 0}; + if (value == 0) + { + *first++ = '0'; + return {first, 0}; + } + + auto unsigned_value = static_cast(value < 0 ? -value : value); + const auto unsigned_base = static_cast(base); + + BOOST_IF_CONSTEXPR (std::is_signed::value) + { + if (value < 0) + { + *first++ = '-'; + } + } + + constexpr unsigned_value zero = 48U; // Char for '0' + std::array buffer = {}; + auto end = buffer.end(); + --end; // Need to point to the last actual element + + // Work from LSB to MSB + switch (base) + { + case 2: + while (unsigned_value != 0) + { + *end-- = static_cast(zero + (unsigned_value & 1U)); // 1<<1 - 1 + value >>= 1U; + } + break; + + case 4: + while (unsigned_value != 0) + { + *end-- = static_cast(zero + (unsigned_value & 3U)); // 1<<2 - 1 + value >>= 2U; + } + break; + + case 8: + while (unsigned_value != 0) + { + *end-- = static_cast(zero + (unsigned_value & 7U)); // 1<<3 - 1 + value >>= 3U; + } + break; + + case 16: + while (unsigned_value != 0) + { + *end-- = static_cast(zero + (unsigned_value & 15U)); // 1<<4 - 1 + value >>= 4U; + } + break; + + case 32: + while (unsigned_value != 0) + { + *end-- = static_cast(zero + (unsigned_value & 31U)); // 1<<5 - 1 + value >>= 5U; + } + break; + + default: + while (unsigned_value != 0) + { + *end-- = digit_table[unsigned_value % unsigned_base]; + unsigned_value /= unsigned_base; + } + break; + } + + const auto num_chars = end - buffer.end() + 1; + + if (num_chars > output_length) + { + return {last, EOVERFLOW}; + } + + std::memcpy(first, buffer.data() + num_chars, num_chars); + + return {first + num_chars, 0}; } } // Namespace detail diff --git a/test/to_chars.cpp b/test/to_chars.cpp index 1d048e2..a9b7650 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -10,6 +10,16 @@ #include #include +template +void base_two_tests() +{ + char buffer1[64] {}; + T v1 = static_cast(42); + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1, 2); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "101010"); +} + template void sixty_four_bit_tests() { @@ -102,5 +112,7 @@ int main() sixty_four_bit_tests(); sixty_four_bit_tests(); + base_two_tests(); + return boost::report_errors(); } From b1e5a06cafbcd591107d4c78ea04b83304483c56 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 26 Jan 2023 11:46:50 -0800 Subject: [PATCH 28/63] Fix type [ci skip] --- include/boost/charconv/to_chars.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 53798b8..28397d0 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -255,7 +255,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l } } - constexpr unsigned_value zero = 48U; // Char for '0' + constexpr Unsigned_Integer zero = 48U; // Char for '0' std::array buffer = {}; auto end = buffer.end(); --end; // Need to point to the last actual element From c06ae72fb936471a649aa6393bf492210f8584eb Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Thu, 26 Jan 2023 12:03:40 -0800 Subject: [PATCH 29/63] Use c-style array instead of std::array [ci skip] --- include/boost/charconv/to_chars.hpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 28397d0..034ad6b 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -231,7 +231,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l using Unsigned_Integer = typename std::make_unsigned::type; - const auto output_length = last - first; + const std::ptrdiff_t output_length = last - first; if (!(first <= last)) { @@ -256,9 +256,10 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l } constexpr Unsigned_Integer zero = 48U; // Char for '0' - std::array buffer = {}; - auto end = buffer.end(); - --end; // Need to point to the last actual element + constexpr auto buffer_size = sizeof(Unsigned_Integer) * CHAR_BIT; + char buffer[buffer_size]; + const char* buffer_end = buffer + buffer_size; + char* end = buffer + buffer_size; // Work from LSB to MSB switch (base) @@ -312,14 +313,14 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l break; } - const auto num_chars = end - buffer.end() + 1; + const std::ptrdiff_t num_chars = buffer_end - end; if (num_chars > output_length) { return {last, EOVERFLOW}; } - std::memcpy(first, buffer.data() + num_chars, num_chars); + std::memcpy(first, buffer + (buffer_size - num_chars), num_chars); return {first + num_chars, 0}; } From 0ac231b27a1ce78651c9c5f3a35ef12011a45892 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 08:52:59 -0800 Subject: [PATCH 30/63] Fix base10 overflow handling --- include/boost/charconv/to_chars.hpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 034ad6b..febbcbc 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -103,12 +103,11 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexc # pragma warning(disable: 4127) #endif -// TODO: Use a temp buffer to hold everything and then write it all into the provided buffer at the end -// If the temp buffer is larger than the provided buffer return EOVERFLOW template BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) noexcept { char buffer[10] {}; + const std::ptrdiff_t user_buffer_size = last - first; BOOST_ATTRIBUTE_UNUSED bool is_negative = false; if (!(first <= last)) @@ -139,6 +138,11 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l const auto converted_value = static_cast(value); const auto num_sig_chars = num_digits(converted_value); + if (num_sig_chars > user_buffer_size) + { + return {last, EOVERFLOW}; + } + decompose32(converted_value, buffer); // TODO: If constant evaluated use unrolled loop @@ -156,6 +160,11 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l auto converted_value = static_cast(value); const auto converted_value_digits = num_digits(converted_value); + if (converted_value_digits > user_buffer_size) + { + return {last, EOVERFLOW}; + } + if (is_negative) { *first++ = '-'; From e00712a6a307d13f9dfdb3b526f6050e6d6883e2 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 09:07:52 -0800 Subject: [PATCH 31/63] Fix handling of all other bases --- include/boost/charconv/to_chars.hpp | 12 ++++++------ test/to_chars.cpp | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index febbcbc..02c9945 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -268,7 +268,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l constexpr auto buffer_size = sizeof(Unsigned_Integer) * CHAR_BIT; char buffer[buffer_size]; const char* buffer_end = buffer + buffer_size; - char* end = buffer + buffer_size; + char* end = buffer + buffer_size - 1; // Work from LSB to MSB switch (base) @@ -277,7 +277,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l while (unsigned_value != 0) { *end-- = static_cast(zero + (unsigned_value & 1U)); // 1<<1 - 1 - value >>= 1U; + unsigned_value >>= 1U; } break; @@ -285,7 +285,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l while (unsigned_value != 0) { *end-- = static_cast(zero + (unsigned_value & 3U)); // 1<<2 - 1 - value >>= 2U; + unsigned_value >>= 2U; } break; @@ -293,7 +293,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l while (unsigned_value != 0) { *end-- = static_cast(zero + (unsigned_value & 7U)); // 1<<3 - 1 - value >>= 3U; + unsigned_value >>= 3U; } break; @@ -301,7 +301,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l while (unsigned_value != 0) { *end-- = static_cast(zero + (unsigned_value & 15U)); // 1<<4 - 1 - value >>= 4U; + unsigned_value >>= 4U; } break; @@ -309,7 +309,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l while (unsigned_value != 0) { *end-- = static_cast(zero + (unsigned_value & 31U)); // 1<<5 - 1 - value >>= 5U; + unsigned_value >>= 5U; } break; diff --git a/test/to_chars.cpp b/test/to_chars.cpp index a9b7650..1d9f9d4 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -17,7 +17,7 @@ void base_two_tests() T v1 = static_cast(42); auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1, 2); BOOST_TEST_EQ(r1.ec, 0); - BOOST_TEST_CSTR_EQ(buffer1, "101010"); + BOOST_TEST(std::memcmp(buffer1, "101010", 6)); } template From 3784c1b81b5e8c5dbaddf91bc33b1f606e4bf9ce Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 09:12:43 -0800 Subject: [PATCH 32/63] Put back memcpy in base10 --- include/boost/charconv/to_chars.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 02c9945..49db685 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -88,9 +88,10 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexc { // Replaces std::memcpy(buffer + i, radix_table + static_cast(y >> 57) * 2, 2) // since it would not be constexpr - const char* temp = {radix_table + static_cast(y >> 57) * 2}; - buffer[i] = temp[0]; - buffer[i+1] = temp[1]; + //const char* temp = {radix_table + static_cast(y >> 57) * 2}; + //buffer[i] = temp[0]; + //buffer[i+1] = temp[1]; + std::memcpy(buffer + i, radix_table + static_cast(y >> 57) * 2, 2); y &= mask; y *= 100; } From 9bc76d803405485c6201db1338041b5a01a50009 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 09:21:08 -0800 Subject: [PATCH 33/63] Fix buffer offset and get rid of memcmp --- include/boost/charconv/to_chars.hpp | 4 ++-- test/to_chars.cpp | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 49db685..404fe44 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -323,8 +323,8 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l break; } - const std::ptrdiff_t num_chars = buffer_end - end; - + const std::ptrdiff_t num_chars = buffer_end - end - 1; + if (num_chars > output_length) { return {last, EOVERFLOW}; diff --git a/test/to_chars.cpp b/test/to_chars.cpp index 1d9f9d4..e698933 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -17,7 +17,7 @@ void base_two_tests() T v1 = static_cast(42); auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1, 2); BOOST_TEST_EQ(r1.ec, 0); - BOOST_TEST(std::memcmp(buffer1, "101010", 6)); + BOOST_TEST_CSTR_EQ(buffer1, "101010"); } template @@ -113,6 +113,7 @@ int main() sixty_four_bit_tests(); base_two_tests(); + base_two_tests(); return boost::report_errors(); } From cce04082b91898941d8bd15ef3b26a4689650fa2 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 09:51:26 -0800 Subject: [PATCH 34/63] Add consteval memcpy --- include/boost/charconv/detail/memcpy.hpp | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 include/boost/charconv/detail/memcpy.hpp diff --git a/include/boost/charconv/detail/memcpy.hpp b/include/boost/charconv/detail/memcpy.hpp new file mode 100644 index 0000000..7b90e9f --- /dev/null +++ b/include/boost/charconv/detail/memcpy.hpp @@ -0,0 +1,46 @@ +// Copyright 2023 Matt Borland +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef BOOST_CHARCONV_DETAIL_MEMCPY_HPP +#define BOOST_CHARCONV_DETAIL_MEMCPY_HPP + +#include +#include +#include + +namespace boost { namespace charconv { namespace detail { + +#if !defined(BOOST_CHARCONV_NO_CONSTEXPR_DETECTION) && defined(BOOST_CXX14_CONSTEXPR) + +#define BOOST_CHARCONV_CONSTEXPR constexpr + +constexpr void* memcpy(void* dest, const void* src, std::size_t count) +{ + if (BOOST_CHARCONV_IS_CONSTANT_EVALUATED(count)) + { + for (std::size_t i = 0; i < count; ++i) + { + *(dest + i) = *(src + i); + } + } + else + { + std::memcpy(dest, src, count); + } +} + +#else // Either not C++14 or no way of telling if we are in a constexpr context + +#define BOOST_CHARCONV_CONSTEXPR + +inline void* memcpy(void* dest, const void* src, std::size_t count) +{ + std::memcpy(dest, src, count); +} + +#endif + +}}} // Namespace boost::charconv::detail + +#endif // BOOST_CHARCONV_DETAIL_MEMCPY_HPP From 2e202cecdbe402246f58b1c0f74aee3bbe087155 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 09:52:48 -0800 Subject: [PATCH 35/63] FIx missing header for CHAR_BIT --- include/boost/charconv/to_chars.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 404fe44..f1d58d5 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace boost { namespace charconv { From 1b37d1cb3d095d371d151637b4ab465ee082faa7 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 10:06:22 -0800 Subject: [PATCH 36/63] Add default initializer to buffer --- include/boost/charconv/to_chars.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index f1d58d5..158e385 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -268,7 +268,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l constexpr Unsigned_Integer zero = 48U; // Char for '0' constexpr auto buffer_size = sizeof(Unsigned_Integer) * CHAR_BIT; - char buffer[buffer_size]; + char buffer[buffer_size] {}; const char* buffer_end = buffer + buffer_size; char* end = buffer + buffer_size - 1; From fb87dfff387b677beaaf3464e225d282a2e0fbd1 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 10:10:25 -0800 Subject: [PATCH 37/63] Fix MSVC C4146 --- include/boost/charconv/to_chars.hpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 158e385..e3033b7 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -255,7 +255,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l return {first, 0}; } - auto unsigned_value = static_cast(value < 0 ? -value : value); + Unsigned_Integer unsigned_value {}; const auto unsigned_base = static_cast(base); BOOST_IF_CONSTEXPR (std::is_signed::value) @@ -263,7 +263,16 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l if (value < 0) { *first++ = '-'; + unsigned_value = apply_sign(value); } + else + { + unsigned_value = value; + } + } + else + { + unsigned_value = value; } constexpr Unsigned_Integer zero = 48U; // Char for '0' From f1fac0a7aeeabc119bc751f6f20d857d2e651bb9 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 10:52:53 -0800 Subject: [PATCH 38/63] Replace std::memcpy with our memcpy --- include/boost/charconv/detail/memcpy.hpp | 6 ++-- include/boost/charconv/to_chars.hpp | 38 +++++++++++------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/include/boost/charconv/detail/memcpy.hpp b/include/boost/charconv/detail/memcpy.hpp index 7b90e9f..39b3819 100644 --- a/include/boost/charconv/detail/memcpy.hpp +++ b/include/boost/charconv/detail/memcpy.hpp @@ -23,10 +23,12 @@ constexpr void* memcpy(void* dest, const void* src, std::size_t count) { *(dest + i) = *(src + i); } + + return dest; } else { - std::memcpy(dest, src, count); + return std::memcpy(dest, src, count); } } @@ -36,7 +38,7 @@ constexpr void* memcpy(void* dest, const void* src, std::size_t count) inline void* memcpy(void* dest, const void* src, std::size_t count) { - std::memcpy(dest, src, count); + return std::memcpy(dest, src, count); } #endif diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index e3033b7..6653e38 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -79,7 +80,7 @@ namespace detail { // See: https://jk-jeon.github.io/posts/2022/02/jeaiii-algorithm/ // https://arxiv.org/abs/2101.11408 -BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexcept +BOOST_CHARCONV_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexcept { constexpr auto mask = (static_cast(1) << 57) - 1; // D = 57 so 2^D - 1 constexpr auto magic_multiplier = static_cast(1441151881); // floor(2*D / 10*k) where D is 57 and k is 8 @@ -87,12 +88,7 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexc for (std::size_t i {}; i < 10; i += 2) { - // Replaces std::memcpy(buffer + i, radix_table + static_cast(y >> 57) * 2, 2) - // since it would not be constexpr - //const char* temp = {radix_table + static_cast(y >> 57) * 2}; - //buffer[i] = temp[0]; - //buffer[i+1] = temp[1]; - std::memcpy(buffer + i, radix_table + static_cast(y >> 57) * 2, 2); + boost::charconv::detail::memcpy(buffer + i, radix_table + static_cast(y >> 57) * 2, 2); y &= mask; y *= 100; } @@ -106,7 +102,7 @@ BOOST_CXX14_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexc #endif template -BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) noexcept +BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) noexcept { char buffer[10] {}; const std::ptrdiff_t user_buffer_size = last - first; @@ -154,7 +150,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l *first++ = '-'; } - std::memcpy(first, buffer + (sizeof(buffer) - num_sig_chars), num_sig_chars); + boost::charconv::detail::memcpy(first, buffer + (sizeof(buffer) - num_sig_chars), num_sig_chars); } else if (std::numeric_limits::digits <= std::numeric_limits::digits || static_cast(value) <= (std::numeric_limits::max)()) @@ -180,10 +176,10 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l const int first_value_chars = num_digits(x); decompose32(x, buffer); - std::memcpy(first, buffer + (sizeof(buffer) - first_value_chars), first_value_chars); + boost::charconv::detail::memcpy(first, buffer + (sizeof(buffer) - first_value_chars), first_value_chars); decompose32(y, buffer); - std::memcpy(first + first_value_chars, buffer + 1, sizeof(buffer) - 1); + boost::charconv::detail::memcpy(first + first_value_chars, buffer + 1, sizeof(buffer) - 1); } else { @@ -195,24 +191,24 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l if (converted_value_digits == 19) { decompose32(x, buffer); - std::memcpy(first, buffer + 2, sizeof(buffer) - 2); + boost::charconv::detail::memcpy(first, buffer + 2, sizeof(buffer) - 2); decompose32(y, buffer); - std::memcpy(first + 8, buffer + 1, sizeof(buffer) - 1); + boost::charconv::detail::memcpy(first + 8, buffer + 1, sizeof(buffer) - 1); decompose32(z, buffer); - std::memcpy(first + 17, buffer + 8, 2); + boost::charconv::detail::memcpy(first + 17, buffer + 8, 2); } else // 20 { decompose32(x, buffer); - std::memcpy(first, buffer + 1, sizeof(buffer) - 1); + boost::charconv::detail::memcpy(first, buffer + 1, sizeof(buffer) - 1); decompose32(y, buffer); - std::memcpy(first + 9, buffer + 1, sizeof(buffer) - 1); + boost::charconv::detail::memcpy(first + 9, buffer + 1, sizeof(buffer) - 1); decompose32(z, buffer); - std::memcpy(first + 18, buffer + 8, 2); + boost::charconv::detail::memcpy(first + 18, buffer + 8, 2); } } } @@ -236,7 +232,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l // All other bases // Use a simple lookup table to put together the Integer in character form template -BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value, int base) noexcept +BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value, int base) noexcept { BOOST_CHARCONV_ASSERT_MSG(base >= 2 && base <= 36, "Base must be between 2 and 36 (inclusive)"); @@ -340,7 +336,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l return {last, EOVERFLOW}; } - std::memcpy(first, buffer + (buffer_size - num_chars), num_chars); + boost::charconv::detail::memcpy(first, buffer + (buffer_size - num_chars), num_chars); return {first + num_chars, 0}; } @@ -348,7 +344,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* l } // Namespace detail template ::value, bool>::type = true> -BOOST_CXX14_CONSTEXPR to_chars_result to_chars(char* first, char* last, Integer value, int base = 10) noexcept +BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars(char* first, char* last, Integer value, int base = 10) noexcept { if (base == 10) { @@ -359,7 +355,7 @@ BOOST_CXX14_CONSTEXPR to_chars_result to_chars(char* first, char* last, Integer } template <> -BOOST_CXX14_CONSTEXPR to_chars_result to_chars(char* first, char* last, bool value, int base) noexcept = delete; +BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars(char* first, char* last, bool value, int base) noexcept = delete; // TODO: Not correct, but need to make MSVC happy while working on integers BOOST_CHARCONV_DECL to_chars_result to_chars(char* first, char* last, float value) noexcept; From 60d819d5b96ebb12e073078105f94a7fcdac6497 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 11:18:48 -0800 Subject: [PATCH 39/63] Add casting to our memcpy --- include/boost/charconv/detail/memcpy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/boost/charconv/detail/memcpy.hpp b/include/boost/charconv/detail/memcpy.hpp index 39b3819..abf314a 100644 --- a/include/boost/charconv/detail/memcpy.hpp +++ b/include/boost/charconv/detail/memcpy.hpp @@ -21,7 +21,7 @@ constexpr void* memcpy(void* dest, const void* src, std::size_t count) { for (std::size_t i = 0; i < count; ++i) { - *(dest + i) = *(src + i); + *((unsigned char*)dest + i) = *((unsigned char*)src + i); } return dest; From 2fbfe9d93ee3c3c5fd7b09807f026c376a789b86 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Fri, 27 Jan 2023 11:36:27 -0800 Subject: [PATCH 40/63] Add overflow tests --- test/to_chars.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/to_chars.cpp b/test/to_chars.cpp index e698933..ab5f7fd 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -10,6 +10,20 @@ #include #include +template +void overflow_tests() +{ + char buffer1[2] {}; + T v1 = static_cast(250); + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1); + BOOST_TEST_EQ(r1.ec, EOVERFLOW); + + char buffer2[3] {}; + T v2 = static_cast(12341234); + auto r2 = boost::charconv::to_chars(buffer2, buffer2 + sizeof(buffer2) - 1, v2); + BOOST_TEST_EQ(r2.ec, EOVERFLOW); +} + template void base_two_tests() { @@ -115,5 +129,7 @@ int main() base_two_tests(); base_two_tests(); + overflow_tests(); + return boost::report_errors(); } From 70472c73d5c632f28d0e86e300c573cc9cd320da Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Tue, 31 Jan 2023 08:15:16 -0800 Subject: [PATCH 41/63] Add test for generic impl --- include/boost/charconv/detail/memcpy.hpp | 2 +- test/to_chars.cpp | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/include/boost/charconv/detail/memcpy.hpp b/include/boost/charconv/detail/memcpy.hpp index abf314a..e5b08dc 100644 --- a/include/boost/charconv/detail/memcpy.hpp +++ b/include/boost/charconv/detail/memcpy.hpp @@ -21,7 +21,7 @@ constexpr void* memcpy(void* dest, const void* src, std::size_t count) { for (std::size_t i = 0; i < count; ++i) { - *((unsigned char*)dest + i) = *((unsigned char*)src + i); + *((unsigned char*)dest + i) = *((const unsigned char*)src + i); } return dest; diff --git a/test/to_chars.cpp b/test/to_chars.cpp index ab5f7fd..8684aa0 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -10,6 +10,23 @@ #include #include +// Tests the generic implementation +template +void base_30_tests() +{ + char buffer1[64] {}; + T v1 = static_cast(1234); + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1, 30); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "1b4"); + + char buffer2[64] {}; + T v2 = static_cast(-4321); + auto r2 = boost::charconv::to_chars(buffer2, buffer2 + sizeof(buffer2) - 1, v2, 30); + BOOST_TEST_EQ(r2.ec, 0); + BOOST_TEST_CSTR_EQ(buffer2, "-4o1"); +} + template void overflow_tests() { @@ -129,6 +146,9 @@ int main() base_two_tests(); base_two_tests(); + base_30_tests(); + base_30_tests(); + overflow_tests(); return boost::report_errors(); From ff1ce048a386e644c54a14cb457c9b118d09cb4c Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Tue, 31 Jan 2023 09:20:31 -0800 Subject: [PATCH 42/63] Changes for constexpr context --- include/boost/charconv/detail/memcpy.hpp | 6 +++--- include/boost/charconv/to_chars.hpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/boost/charconv/detail/memcpy.hpp b/include/boost/charconv/detail/memcpy.hpp index e5b08dc..aebf1b7 100644 --- a/include/boost/charconv/detail/memcpy.hpp +++ b/include/boost/charconv/detail/memcpy.hpp @@ -15,20 +15,20 @@ namespace boost { namespace charconv { namespace detail { #define BOOST_CHARCONV_CONSTEXPR constexpr -constexpr void* memcpy(void* dest, const void* src, std::size_t count) +constexpr char* memcpy(char* dest, const char* src, std::size_t count) { if (BOOST_CHARCONV_IS_CONSTANT_EVALUATED(count)) { for (std::size_t i = 0; i < count; ++i) { - *((unsigned char*)dest + i) = *((const unsigned char*)src + i); + *(dest + i) = *(src + i); } return dest; } else { - return std::memcpy(dest, src, count); + return static_cast(std::memcpy(dest, src, count)); } } diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 6653e38..8edfde8 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -222,7 +222,7 @@ BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char BOOST_CHARCONV_ASSERT_MSG(sizeof(Integer) < 1, "Your type is unsupported. Use a built-in integral type"); } - return {first + std::strlen(first), 0}; + return {--last, 0}; } #ifdef BOOST_MSVC From af3b943333754a5deba2c9091893e3d635abf547 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Tue, 31 Jan 2023 09:56:28 -0800 Subject: [PATCH 43/63] Fix location of returned pointer --- include/boost/charconv/to_chars.hpp | 11 ++++++----- test/to_chars_integer_STL_comp.cpp | 13 +++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 8edfde8..047860a 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -105,6 +105,7 @@ template BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char* last, Integer value) noexcept { char buffer[10] {}; + int converted_value_digits {}; const std::ptrdiff_t user_buffer_size = last - first; BOOST_ATTRIBUTE_UNUSED bool is_negative = false; @@ -134,9 +135,9 @@ BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char value <= static_cast((std::numeric_limits::max)())) { const auto converted_value = static_cast(value); - const auto num_sig_chars = num_digits(converted_value); + converted_value_digits = num_digits(converted_value); - if (num_sig_chars > user_buffer_size) + if (converted_value_digits > user_buffer_size) { return {last, EOVERFLOW}; } @@ -150,13 +151,13 @@ BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char *first++ = '-'; } - boost::charconv::detail::memcpy(first, buffer + (sizeof(buffer) - num_sig_chars), num_sig_chars); + boost::charconv::detail::memcpy(first, buffer + (sizeof(buffer) - converted_value_digits), converted_value_digits); } else if (std::numeric_limits::digits <= std::numeric_limits::digits || static_cast(value) <= (std::numeric_limits::max)()) { auto converted_value = static_cast(value); - const auto converted_value_digits = num_digits(converted_value); + converted_value_digits = num_digits(converted_value); if (converted_value_digits > user_buffer_size) { @@ -222,7 +223,7 @@ BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char BOOST_CHARCONV_ASSERT_MSG(sizeof(Integer) < 1, "Your type is unsupported. Use a built-in integral type"); } - return {--last, 0}; + return {first + converted_value_digits + static_cast(is_negative), 0}; } #ifdef BOOST_MSVC diff --git a/test/to_chars_integer_STL_comp.cpp b/test/to_chars_integer_STL_comp.cpp index cb66377..4a2b0a6 100644 --- a/test/to_chars_integer_STL_comp.cpp +++ b/test/to_chars_integer_STL_comp.cpp @@ -21,11 +21,16 @@ template void test() { // Signed 0 - char buffer1[64] {}; + char buffer1_stl[64] {}; + char buffer1_boost[64] {}; T v1 = static_cast(-0); - auto r1 = std::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1); - (void)r1; - BOOST_TEST_CSTR_EQ(buffer1, "0"); + auto r1_stl = std::to_chars(buffer1_stl, buffer1_stl + sizeof(buffer1_stl) - 1, v1); + auto r1_boost = boost::charconv::to_chars(buffer1_boost, buffer1_boost + sizeof(buffer1_boost) - 1, v1); + BOOST_TEST_EQ(r1_stl.ptr, buffer1_stl + 1); + BOOST_TEST_EQ(r1_boost.ptr, buffer1_boost + 1); + BOOST_TEST_CSTR_EQ(buffer1_stl, "0"); + BOOST_TEST_CSTR_EQ(buffer1_boost, "0"); + } int main() From 12147b861f77b301ef5d62af36f21870252d9e56 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Tue, 31 Jan 2023 10:53:22 -0800 Subject: [PATCH 44/63] Replace 57-bit mask with 32-bit mask Co-authored-by: Junekey Jeon --- include/boost/charconv/to_chars.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 047860a..97d862d 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -1,5 +1,6 @@ // Copyright 2022 Peter Dimov // Copyright 2023 Matt Borland +// Copyright 2023 Junekey Jeon // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -82,13 +83,12 @@ namespace detail { // https://arxiv.org/abs/2101.11408 BOOST_CHARCONV_CONSTEXPR char* decompose32(std::uint32_t value, char* buffer) noexcept { - constexpr auto mask = (static_cast(1) << 57) - 1; // D = 57 so 2^D - 1 - constexpr auto magic_multiplier = static_cast(1441151881); // floor(2*D / 10*k) where D is 57 and k is 8 - auto y = value * magic_multiplier; + constexpr auto mask = (static_cast(1) << 32) - 1; + auto y = ((value * UINT64_C(720575941)) >> 24) + 1; for (std::size_t i {}; i < 10; i += 2) { - boost::charconv::detail::memcpy(buffer + i, radix_table + static_cast(y >> 57) * 2, 2); + boost::charconv::detail::memcpy(buffer + i, radix_table + static_cast(y >> 32) * 2, 2); y &= mask; y *= 100; } From f5f0af78e6933163c5b61bdb3e7cd68c41c09bf4 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Tue, 31 Jan 2023 11:21:42 -0800 Subject: [PATCH 45/63] Add more comparisons against STL --- include/boost/charconv/to_chars.hpp | 4 ++-- test/to_chars_integer_STL_comp.cpp | 33 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/include/boost/charconv/to_chars.hpp b/include/boost/charconv/to_chars.hpp index 97d862d..98304b7 100644 --- a/include/boost/charconv/to_chars.hpp +++ b/include/boost/charconv/to_chars.hpp @@ -308,7 +308,7 @@ BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char case 16: while (unsigned_value != 0) { - *end-- = static_cast(zero + (unsigned_value & 15U)); // 1<<4 - 1 + *end-- = digit_table[unsigned_value & 15U]; // 1<<4 - 1 unsigned_value >>= 4U; } break; @@ -316,7 +316,7 @@ BOOST_CHARCONV_CONSTEXPR to_chars_result to_chars_integer_impl(char* first, char case 32: while (unsigned_value != 0) { - *end-- = static_cast(zero + (unsigned_value & 31U)); // 1<<5 - 1 + *end-- = digit_table[unsigned_value & 31U]; // 1<<5 - 1 unsigned_value >>= 5U; } break; diff --git a/test/to_chars_integer_STL_comp.cpp b/test/to_chars_integer_STL_comp.cpp index 4a2b0a6..2a5378b 100644 --- a/test/to_chars_integer_STL_comp.cpp +++ b/test/to_chars_integer_STL_comp.cpp @@ -31,6 +31,39 @@ void test() BOOST_TEST_CSTR_EQ(buffer1_stl, "0"); BOOST_TEST_CSTR_EQ(buffer1_boost, "0"); + // Binary + char buffer2_stl[64] {}; + char buffer2_boost[64] {}; + T v2 = static_cast(42); + auto r2_stl = std::to_chars(buffer2_stl, buffer2_stl + sizeof(buffer2_stl) - 1, v2, 2); + auto r2_boost = boost::charconv::to_chars(buffer2_boost, buffer2_boost + sizeof(buffer2_boost) - 1, v2, 2); + BOOST_TEST_EQ(r2_stl.ptr, buffer2_stl + 6); + BOOST_TEST_EQ(r2_boost.ptr, buffer2_boost + 6); + BOOST_TEST_CSTR_EQ(buffer2_stl, "101010"); + BOOST_TEST_CSTR_EQ(buffer2_boost, "101010"); + + // Base 10 + char buffer3_stl[64] {}; + char buffer3_boost[64] {}; + T v3 = static_cast(120); + auto r3_stl = std::to_chars(buffer3_stl, buffer3_stl + sizeof(buffer3_stl) - 1, v3); + auto r3_boost = boost::charconv::to_chars(buffer3_boost, buffer3_boost + sizeof(buffer3_boost) - 1, v3); + BOOST_TEST_EQ(r3_stl.ptr, buffer3_stl + 3); + BOOST_TEST_EQ(r3_boost.ptr, buffer3_boost + 3); + BOOST_TEST_CSTR_EQ(buffer3_stl, "120"); + BOOST_TEST_CSTR_EQ(buffer3_boost, "120"); + + // Hexadecimal + char buffer4_stl[64] {}; + char buffer4_boost[64] {}; + T v4 = static_cast(44); + auto r4_stl = std::to_chars(buffer4_stl, buffer4_stl + sizeof(buffer4_stl) - 1, v4, 16); + auto r4_boost = boost::charconv::to_chars(buffer4_boost, buffer4_boost + sizeof(buffer4_boost) - 1, v4, 16); + BOOST_TEST_EQ(r4_stl.ptr, buffer4_stl + 2); + BOOST_TEST_EQ(r4_boost.ptr, buffer4_boost + 2); + BOOST_TEST_CSTR_EQ(buffer4_stl, "2c"); + BOOST_TEST_CSTR_EQ(buffer4_boost, "2c"); + } int main() From f85c68f9d257a799388d69be78a2ab94b7e3820d Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 1 Feb 2023 09:42:01 -0800 Subject: [PATCH 46/63] More STL comparisons --- test/to_chars_integer_STL_comp.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/to_chars_integer_STL_comp.cpp b/test/to_chars_integer_STL_comp.cpp index 2a5378b..0a1c110 100644 --- a/test/to_chars_integer_STL_comp.cpp +++ b/test/to_chars_integer_STL_comp.cpp @@ -64,6 +64,25 @@ void test() BOOST_TEST_CSTR_EQ(buffer4_stl, "2c"); BOOST_TEST_CSTR_EQ(buffer4_boost, "2c"); + // Base 28 + char buffer5_stl[64] {}; + char buffer5_boost[64] {}; + T v5 = static_cast(100); + auto r5_stl = std::to_chars(buffer5_stl, buffer5_stl + sizeof(buffer5_stl) - 1, v5, 28); + auto r5_boost = boost::charconv::to_chars(buffer5_boost, buffer5_boost + sizeof(buffer5_boost) - 1, v5, 28); + BOOST_TEST_EQ(r5_stl.ptr, buffer5_stl + 2); + BOOST_TEST_EQ(r5_boost.ptr, buffer5_boost + 2); + BOOST_TEST_CSTR_EQ(buffer5_stl, "3g"); + BOOST_TEST_CSTR_EQ(buffer5_boost, "3g"); + + // Overflow + char buffer6_stl[1] {}; + char buffer6_boost[1] {}; + T v6 = static_cast(100); + auto r6_stl = std::to_chars(buffer6_stl, buffer6_stl + sizeof(buffer6_stl) - 1, v6); + auto r6_boost = boost::charconv::to_chars(buffer6_boost, buffer6_boost + sizeof(buffer6_boost) - 1, v6); + BOOST_TEST_EQ(r6_stl.ptr, r6_stl.ptr); + BOOST_TEST_EQ(r6_boost.ptr, r6_boost.ptr); } int main() From 20af2a7eda114dd081b9244ecc1a09411e209466 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 1 Feb 2023 09:53:51 -0800 Subject: [PATCH 47/63] Cover more code paths --- test/to_chars.cpp | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/test/to_chars.cpp b/test/to_chars.cpp index 8684aa0..f54017a 100644 --- a/test/to_chars.cpp +++ b/test/to_chars.cpp @@ -10,6 +10,46 @@ #include #include +template +void base_thirtytwo_tests() +{ + char buffer1[64] {}; + T v1 = static_cast(42); + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1, 32); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "1a"); +} + +template +void base_sixteen_tests() +{ + char buffer1[64] {}; + T v1 = static_cast(42); + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1, 16); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "2a"); +} + +template +void base_eight_tests() +{ + char buffer1[64] {}; + T v1 = static_cast(42); + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1, 8); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "52"); +} + +template +void base_four_tests() +{ + char buffer1[64] {}; + T v1 = static_cast(42); + auto r1 = boost::charconv::to_chars(buffer1, buffer1 + sizeof(buffer1) - 1, v1, 4); + BOOST_TEST_EQ(r1.ec, 0); + BOOST_TEST_CSTR_EQ(buffer1, "222"); +} + // Tests the generic implementation template void base_30_tests() @@ -143,9 +183,23 @@ int main() sixty_four_bit_tests(); sixty_four_bit_tests(); + // Tests for every specialized base base_two_tests(); base_two_tests(); + base_four_tests(); + base_four_tests(); + + base_eight_tests(); + base_eight_tests(); + + base_sixteen_tests(); + base_sixteen_tests(); + + base_thirtytwo_tests(); + base_thirtytwo_tests(); + + // The generic impl base_30_tests(); base_30_tests(); From 6b5d05deae9284448e9284c3b125efbc109f8082 Mon Sep 17 00:00:00 2001 From: Matt Borland Date: Wed, 1 Feb 2023 11:38:06 -0800 Subject: [PATCH 48/63] Add docs and rebuild HTML --- doc/charconv.adoc | 1 + doc/charconv.html | 156 +++++++++++++++++++++++++++++++++++- doc/charconv/overview.adoc | 12 ++- doc/charconv/reference.adoc | 17 ++-- doc/charconv/to_chars.adoc | 77 ++++++++++++++++++ 5 files changed, 248 insertions(+), 15 deletions(-) create mode 100644 doc/charconv/to_chars.adoc diff --git a/doc/charconv.adoc b/doc/charconv.adoc index 022fd17..c652ce7 100644 --- a/doc/charconv.adoc +++ b/doc/charconv.adoc @@ -18,6 +18,7 @@ Peter Dimov and Matt Borland include::charconv/overview.adoc[] include::charconv/from_chars.adoc[] +include::charconv/to_chars.adoc[] include::charconv/reference.adoc[] include::charconv/copyright.adoc[] diff --git a/doc/charconv.html b/doc/charconv.html index 5fc9ee3..ae6fbb7 100644 --- a/doc/charconv.html +++ b/doc/charconv.html @@ -684,6 +684,21 @@ pre.rouge { +
  • to_chars + +
  • Reference