general/CLI11
2
0
Fork 0
mirror of https://github.com/CLIUtils/CLI11.git synced 2026-01-19 04:52:08 +00:00
Code Issues Projects Releases Wiki Activity

Fuzz fail new (#1164)

Browse Source 
Additional fuzz failures from longer runs of fuzzer.
Ran the fuzzer for a couple hours. Picked up a few interesting bug particularly in the config out and return.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Philip Top
2025-05-21 19:37:28 -07:00
committed by GitHub
parent 2c787a50ed
commit 50591fb666
19 changed files with 455 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
fuzz/cli11_app_fuzz.cpp
View File

@@ -17,9 +17,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
std::string parseString(reinterpret_cast<const char *>(Data), Size);
CLI::FuzzApp fuzzdata;
CLI::FuzzApp fuzzdata2;
auto app = fuzzdata.generateApp();
auto app2 = fuzzdata2.generateApp();
std::size_t pstring_start{0};
try {
pstring_start = fuzzdata.add_custom_options(app.get(), parseString);
@@ -40,17 +39,21 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
// this just indicates we caught an error known by CLI
return 0; // Non-zero return values are reserved for future use.
}
// should be able to write the config to a file and read from it again
std::string configOut = app->config_to_str();
app->clear();
std::stringstream out(configOut);
if(pstring_start > 0) {
fuzzdata2.add_custom_options(app2.get(), parseString);
}
app2->parse_from_stream(out);
auto result = fuzzdata2.compare(fuzzdata);
if(!result) {
throw CLI::ValidationError("fuzzer", "file input results don't match parse results");
if(fuzzdata.support_config_file_only()) {
CLI::FuzzApp fuzzdata2;
auto app2 = fuzzdata2.generateApp();
// should be able to write the config to a file and read from it again
std::string configOut = app->config_to_str();
std::stringstream out(configOut);
if(pstring_start > 0) {
fuzzdata2.add_custom_options(app2.get(), parseString);
}
app2->parse_from_stream(out);
auto result = fuzzdata2.compare(fuzzdata);
if(!result) {
throw CLI::ValidationError("fuzzer", "file input results don't match parse results");
}
}
return 0;
}

87
fuzz/fuzzApp.cpp
View File

@@ -6,6 +6,7 @@
#include "fuzzApp.hpp"
#include <algorithm>
#include <iostream>
namespace CLI {
/*
@@ -151,7 +152,27 @@ std::shared_ptr<CLI::App> FuzzApp::generateApp() {
return fApp;
}
bool FuzzApp::compare(const FuzzApp &other) const {
static void print_string_comparison(const std::string &s1,
const std::string &s2,
const std::string &prefix,
const std::string &s1name,
const std::string &s2name) {
for(size_t jj = 0; jj < (std::max)(s1.size(), s2.size()); ++jj) {
if(jj >= s1.size()) {
std::cout << prefix << ":" << s1name << "[" << jj << "] = [empty], " << s2name << "[" << jj
<< "]=" << static_cast<int>(s2[jj]) << '\n';
} else if(jj >= s2.size()) {
std::cout << prefix << ":" << s1name << "[" << jj << "]=" << static_cast<int>(s1[jj]) << ", " << s2name
<< "[" << jj << "]=[empty] \n";
} else if(s1[jj] != s2[jj]) {
std::cout << "-->" << prefix << ":" << s1name << "[" << jj << "]=" << static_cast<int>(s1[jj]) << ", "
<< s2name << "[" << jj << "]=" << static_cast<int>(s2[jj]) << '\n';
} else {
std::cout << prefix << ":" << s1name << "[" << jj << "]=" << static_cast<int>(s1[jj]) << '\n';
}
}
}
bool FuzzApp::compare(const FuzzApp &other, bool print_error) const {
if(val32 != other.val32) {
return false;
}
@@ -291,6 +312,20 @@ bool FuzzApp::compare(const FuzzApp &other) const {
std::vector<std::string> res = vstrD;
std::reverse(res.begin(), res.end());
if(res != other.vstrD) {
if(print_error) {
if(res.size() != other.vstrD.size()) {
std::cout << "size is different vstrD.size()=" << res.size()
<< " other.vstrD.size=" << other.vstrD.size() << '\n';
} else {
for(size_t ii = 0; ii < res.size(); ++ii) {
print_string_comparison(res[ii],
other.vstrD[ii],
std::string("string[") + std::to_string(ii) + ']',
"vstrD",
"other.vstrD");
}
}
}
return false;
}
}
@@ -311,8 +346,17 @@ bool FuzzApp::compare(const FuzzApp &other) const {
return false;
}
for(std::size_t ii = 0; ii < custom_string_options.size(); ++ii) {
if(*custom_string_options[ii] != *other.custom_string_options[ii]) {
return false;
if(custom_string_options[ii]->first != other.custom_string_options[ii]->first) {
if(custom_string_options[ii]->second) {
if(print_error) {
print_string_comparison(custom_string_options[ii]->first,
other.custom_string_options[ii]->first,
std::string("custom_string[") + std::to_string(ii) + ']',
"c1",
"other.c1");
}
return false;
}
}
}
// now test custom vector_options
@@ -320,8 +364,10 @@ bool FuzzApp::compare(const FuzzApp &other) const {
return false;
}
for(std::size_t ii = 0; ii < custom_vector_options.size(); ++ii) {
if(*custom_vector_options[ii] != *other.custom_vector_options[ii]) {
return false;
if(custom_vector_options[ii]->first != other.custom_vector_options[ii]->first) {
if(custom_vector_options[ii]->second) {
return false;
}
}
}
return true;
@@ -447,11 +493,17 @@ std::size_t FuzzApp::add_custom_options(CLI::App *app, const std::string &descri
break;
}
std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
custom_string_options.push_back(std::make_shared<std::string>());
auto *opt = app->add_option(name, *(custom_string_options.back()));
custom_string_options.push_back(std::make_shared<std::pair<std::string, bool>>("", true));
auto *opt = app->add_option(name, custom_string_options.back()->first);
if(header_close > current_index + 19) {
std::string attributes = description_string.substr(current_index + 8, header_close - 8 - current_index);
modify_option(opt, attributes);
if(!opt->get_configurable()) {
custom_string_options.back()->second = false;
if(opt->get_required()) {
non_config_required = true;
}
}
}
current_index = end_option + 9;
@@ -465,12 +517,18 @@ std::size_t FuzzApp::add_custom_options(CLI::App *app, const std::string &descri
break;
}
std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
custom_string_options.push_back(std::make_shared<std::string>());
auto *opt = app->add_option(name, *(custom_string_options.back()));
custom_string_options.push_back(std::make_shared<std::pair<std::string, bool>>("", true));
auto *opt = app->add_option(name, custom_string_options.back()->first);
if(header_close > current_index + 17) {
std::string attributes = description_string.substr(current_index + 6, header_close - 6 - current_index);
modify_option(opt, attributes);
if(!opt->get_configurable()) {
custom_string_options.back()->second = false;
if(opt->get_required()) {
non_config_required = true;
}
}
}
current_index = end_option + 7;
} else if(description_string.compare(current_index, 7, "<vector") == 0) {
@@ -483,11 +541,18 @@ std::size_t FuzzApp::add_custom_options(CLI::App *app, const std::string &descri
break;
}
std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
custom_vector_options.push_back(std::make_shared<std::vector<std::string>>());
auto *opt = app->add_option(name, *(custom_vector_options.back()));
custom_vector_options.push_back(std::make_shared<std::pair<std::vector<std::string>, bool>>());
custom_vector_options.back()->second = true;
auto *opt = app->add_option(name, custom_vector_options.back()->first);
if(header_close > current_index + 19) {
std::string attributes = description_string.substr(current_index + 8, header_close - 8 - current_index);
modify_option(opt, attributes);
if(!opt->get_configurable()) {
custom_vector_options.back()->second = false;
if(opt->get_required()) {
non_config_required = true;
}
}
}
current_index = end_option + 9;
} else {

12
fuzz/fuzzApp.hpp
View File

@@ -57,12 +57,13 @@ class FuzzApp {
/** generate a fuzzing application with a bunch of different interfaces*/
std::shared_ptr<CLI::App> generateApp();
/** compare two fuzz apps for equality*/
CLI11_NODISCARD bool compare(const FuzzApp &other) const;
CLI11_NODISCARD bool compare(const FuzzApp &other, bool print_error = false) const;
/** generate additional options based on a string config*/
std::size_t add_custom_options(CLI::App *app, const std::string &description_string);
/** modify an option based on string*/
void modify_option(CLI::Option *opt, const std::string &modifier);
static void modify_option(CLI::Option *opt, const std::string &modifier);
CLI11_NODISCARD bool support_config_file_only() const { return !non_config_required; }
int32_t val32{0};
int16_t val16{0};
int8_t val8{0};
@@ -121,7 +122,10 @@ class FuzzApp {
std::vector<std::string> vstrF{};
std::string mergeBuffer{};
std::vector<std::string> validator_strings{};
std::vector<std::shared_ptr<std::string>> custom_string_options{};
std::vector<std::shared_ptr<std::vector<std::string>>> custom_vector_options{};
std::vector<std::shared_ptr<std::pair<std::string, bool>>> custom_string_options{};
std::vector<std::shared_ptr<std::pair<std::vector<std::string>, bool>>> custom_vector_options{};
private:
bool non_config_required{false};
};
} // namespace CLI

3
include/CLI/App.hpp
View File

@@ -1322,6 +1322,9 @@ class App {
/// Fill in a single config option
bool _parse_single_config(const ConfigItem &item, std::size_t level = 0);
/// @brief store the results for a flag like option
bool _add_flag_like_result(Option *op, const ConfigItem &item, const std::vector<std::string> &inputs);
/// Parse "one" argument (some may eat more than one), delegate to parent if fails, add to missing if missing
/// from main return false if the parse has failed and needs to return to parent
bool _parse_single(std::vector<std::string> &args, bool &positional_only);

2
include/CLI/StringTools.hpp
View File

@@ -251,7 +251,7 @@ CLI11_INLINE std::string add_escaped_characters(const std::string &str);
CLI11_INLINE std::string remove_escaped_characters(const std::string &str);
/// generate a string with all non printable characters escaped to hex codes
CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape);
CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape, bool force = false);
CLI11_INLINE bool is_binary_escaped_string(const std::string &escaped_string);

138
include/CLI/impl/App_inl.hpp
View File

@@ -178,6 +178,12 @@ CLI11_INLINE Option *App::add_option(std::string option_name,
if(op != nullptr && op->get_configurable()) {
throw(OptionAlreadyAdded("added option positional name matches existing option: " + test_name));
}
// need to check if there is another positional with the same name that also doesn't have any long or short
// names
op = get_option_no_throw(myopt.get_single_name());
if(op != nullptr && op->lnames_.empty() && op->snames_.empty()) {
throw(OptionAlreadyAdded("unable to disambiguate with existing option: " + test_name));
}
} else if(parent_ != nullptr) {
for(auto &ln : myopt.lnames_) {
auto *op = parent_->get_option_no_throw(ln);
@@ -1490,6 +1496,67 @@ CLI11_INLINE void App::_parse_config(const std::vector<ConfigItem> &args) {
}
}
CLI11_INLINE bool
App::_add_flag_like_result(Option *op, const ConfigItem &item, const std::vector<std::string> &inputs) {
if(item.inputs.size() <= 1) {
// Flag parsing
auto res = config_formatter_->to_flag(item);
bool converted{false};
if(op->get_disable_flag_override()) {
auto val = detail::to_flag_value(res);
if(val == 1) {
res = op->get_flag_value(item.name, "{}");
converted = true;
}
}
if(!converted) {
errno = 0;
if(res != "{}" || op->get_expected_max() <= 1) {
res = op->get_flag_value(item.name, res);
}
}
op->add_result(res);
return true;
}
if(static_cast<int>(inputs.size()) > op->get_items_expected_max() &&
op->get_multi_option_policy() != MultiOptionPolicy::TakeAll) {
if(op->get_items_expected_max() > 1) {
throw ArgumentMismatch::AtMost(item.fullname(), op->get_items_expected_max(), inputs.size());
}
if(!op->get_disable_flag_override()) {
throw ConversionError::TooManyInputsFlag(item.fullname());
}
// if the disable flag override is set then we must have the flag values match a known flag value
// this is true regardless of the output value, so an array input is possible and must be accounted for
for(const auto &res : inputs) {
bool valid_value{false};
if(op->default_flag_values_.empty()) {
if(res == "true" || res == "false" || res == "1" || res == "0") {
valid_value = true;
}
} else {
for(const auto &valid_res : op->default_flag_values_) {
if(valid_res.second == res) {
valid_value = true;
break;
}
}
}
if(valid_value) {
op->add_result(res);
} else {
throw InvalidError("invalid flag argument given");
}
}
return true;
}
return false;
}
CLI11_INLINE bool App::_parse_single_config(const ConfigItem &item, std::size_t level) {
if(level < item.parents.size()) {
@@ -1574,60 +1641,7 @@ CLI11_INLINE bool App::_parse_single_config(const ConfigItem &item, std::size_t
}
const std::vector<std::string> &inputs = (useBuffer) ? buffer : item.inputs;
if(op->get_expected_min() == 0) {
if(item.inputs.size() <= 1) {
// Flag parsing
auto res = config_formatter_->to_flag(item);
bool converted{false};
if(op->get_disable_flag_override()) {
auto val = detail::to_flag_value(res);
if(val == 1) {
res = op->get_flag_value(item.name, "{}");
converted = true;
}
}
if(!converted) {
errno = 0;
if(res != "{}" || op->get_expected_max() <= 1) {
res = op->get_flag_value(item.name, res);
}
}
op->add_result(res);
return true;
}
if(static_cast<int>(inputs.size()) > op->get_items_expected_max() &&
op->get_multi_option_policy() != MultiOptionPolicy::TakeAll) {
if(op->get_items_expected_max() > 1) {
throw ArgumentMismatch::AtMost(item.fullname(), op->get_items_expected_max(), inputs.size());
}
if(!op->get_disable_flag_override()) {
throw ConversionError::TooManyInputsFlag(item.fullname());
}
// if the disable flag override is set then we must have the flag values match a known flag value
// this is true regardless of the output value, so an array input is possible and must be accounted for
for(const auto &res : inputs) {
bool valid_value{false};
if(op->default_flag_values_.empty()) {
if(res == "true" || res == "false" || res == "1" || res == "0") {
valid_value = true;
}
} else {
for(const auto &valid_res : op->default_flag_values_) {
if(valid_res.second == res) {
valid_value = true;
break;
}
}
}
if(valid_value) {
op->add_result(res);
} else {
throw InvalidError("invalid flag argument given");
}
}
if(_add_flag_like_result(op, item, inputs)) {
return true;
}
}
@@ -1756,9 +1770,21 @@ CLI11_INLINE bool App::_parse_positional(std::vector<std::string> &args, bool ha
if(posOpt->get_trigger_on_parse() && posOpt->current_option_state_ == Option::option_state::callback_run) {
posOpt->clear();
}
posOpt->add_result(positional);
if(posOpt->get_expected_min() == 0) {
ConfigItem item;
item.name = posOpt->pname_;
item.inputs.push_back(positional);
if(!_add_flag_like_result(posOpt, item, item.inputs)) {
posOpt->add_result(positional);
}
} else {
posOpt->add_result(positional);
}
if(posOpt->get_trigger_on_parse()) {
posOpt->run_callback();
if(!posOpt->empty()) {
posOpt->run_callback();
}
}
args.pop_back();

17
include/CLI/impl/Config_inl.hpp
View File

@@ -84,7 +84,17 @@ convert_arg_for_ini(const std::string &arg, char stringQuote, char literalQuote,
}
if(detail::has_escapable_character(arg)) {
if(arg.size() > 100 && !disable_multi_line) {
return std::string(multiline_literal_quote) + arg + multiline_literal_quote;
if(arg.find(multiline_literal_quote) != std::string::npos) {
return binary_escape_string(arg, true);
}
std::string return_string{multiline_literal_quote};
return_string.reserve(7 + arg.size());
if(arg.front() == '\n' || arg.front() == '\r') {
return_string.push_back('\n');
}
return_string.append(arg);
return_string.append(multiline_literal_quote, 3);
return return_string;
}
return std::string(1, stringQuote) + detail::add_escaped_characters(arg) + stringQuote;
}
@@ -393,7 +403,7 @@ inline std::vector<ConfigItem> ConfigBase::from_config(std::istream &input) cons
}
lineExtension = false;
firstLine = false;
if(!l2.empty() && l2.back() == '\\') {
if(!l2.empty() && l2.back() == '\\' && keyChar == '\"') {
lineExtension = true;
l2.pop_back();
}
@@ -543,6 +553,9 @@ ConfigBase::to_config(const App *app, bool default_also, bool write_description,
}
auto results = opt->reduced_results();
if(results.size() > 1 && opt->get_multi_option_policy() == CLI::MultiOptionPolicy::Reverse) {
std::reverse(results.begin(), results.end());
}
std::string value =
detail::ini_join(results, arraySeparator, arrayStart, arrayEnd, stringQuote, literalQuote);

6
include/CLI/impl/StringTools_inl.hpp
View File

@@ -86,7 +86,7 @@ CLI11_INLINE std::string &remove_outer(std::string &str, char key) {
CLI11_INLINE std::string fix_newlines(const std::string &leader, std::string input) {
std::string::size_type n = 0;
while(n != std::string::npos && n < input.size()) {
n = input.find('\n', n);
n = input.find_first_of("\r\n", n);
if(n != std::string::npos) {
input = input.substr(0, n + 1) + leader + input.substr(n + 1);
n += leader.size();
@@ -422,7 +422,7 @@ CLI11_INLINE std::size_t escape_detect(std::string &str, std::size_t offset) {
return offset + 1;
}
CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape) {
CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape, bool force) {
// s is our escaped output string
std::string escaped_string{};
// loop through all characters
@@ -449,7 +449,7 @@ CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escap
escaped_string.push_back(c);
}
}
if(escaped_string != string_to_escape) {
if(escaped_string != string_to_escape || force) {
auto sqLoc = escaped_string.find('\'');
while(sqLoc != std::string::npos) {
escaped_string[sqLoc] = '\\';

42
tests/FuzzFailTest.cpp
View File

@@ -6,6 +6,7 @@
#include "../fuzz/fuzzApp.hpp"
#include "app_helper.hpp"
#include <iostream>
#include <string>
#include <vector>
@@ -108,6 +109,7 @@ TEST_CASE("app_file_roundtrip") {
auto app = fuzzdata.generateApp();
auto app2 = fuzzdata2.generateApp();
int index = GENERATE(range(1, 41));
// int index = GENERATE(range(8, 9));
std::string optionString, flagString;
auto parseData = loadFailureFile("fuzz_app_file_fail", index);
if(parseData.size() > 25) {
@@ -147,6 +149,7 @@ TEST_CASE("app_file_roundtrip") {
result = fuzzdata2.compare(fuzzdata);
}
*/
INFO("Failure in test case " << index)
CHECK(result);
}
@@ -312,7 +315,7 @@ TEST_CASE("app_roundtrip_custom") {
CLI::FuzzApp fuzzdata2;
auto app = fuzzdata.generateApp();
auto app2 = fuzzdata2.generateApp();
int index = GENERATE(range(1, 5));
int index = GENERATE(range(1, 12));
auto parseData = loadFailureFile("round_trip_custom", index);
std::size_t pstring_start{0};
pstring_start = fuzzdata.add_custom_options(app.get(), parseData);
@@ -322,17 +325,23 @@ TEST_CASE("app_roundtrip_custom") {
} else {
app->parse(parseData);
}
// should be able to write the config to a file and read from it again
std::string configOut = app->config_to_str();
app->clear();
std::stringstream out(configOut);
if(pstring_start > 0) {
fuzzdata2.add_custom_options(app2.get(), parseData);
if(fuzzdata.support_config_file_only()) {
// should be able to write the config to a file and read from it again
std::string configOut = app->config_to_str();
std::stringstream out(configOut);
if(pstring_start > 0) {
fuzzdata2.add_custom_options(app2.get(), parseData);
}
app2->parse_from_stream(out);
auto result = fuzzdata2.compare(fuzzdata);
if(!result) {
result = fuzzdata.compare(fuzzdata2, true);
std::cout << "\n:parsed:\n" << parseData;
std::cout << "\n:config:\n" << configOut << '\n';
}
INFO("Failure in test case " << index)
CHECK(result);
}
app2->parse_from_stream(out);
auto result = fuzzdata2.compare(fuzzdata);
CHECK(result);
}
// this test
@@ -341,11 +350,16 @@ TEST_CASE("app_roundtrip_parse_normal_fail") {
// like HorribleErrors
CLI::FuzzApp fuzzdata;
auto app = fuzzdata.generateApp();
int index = GENERATE(range(1, 4));
int index = GENERATE(range(1, 7));
auto parseData = loadFailureFile("parse_fail_check", index);
std::size_t pstring_start{0};
pstring_start = fuzzdata.add_custom_options(app.get(), parseData);
try {
pstring_start = fuzzdata.add_custom_options(app.get(), parseData);
} catch(const CLI::ConstructionError & /*ce*/) {
CHECK(true);
return;
}
INFO("Failure in test case " << index)
try {
if(pstring_start > 0) {
app->parse(parseData.substr(pstring_start));

BIN
tests/fuzzFail/parse_fail_check4 Normal file
View File

Binary file not shown.

BIN
tests/fuzzFail/parse_fail_check5 Normal file
View File

Binary file not shown.

BIN
tests/fuzzFail/parse_fail_check6 Normal file
View File

Binary file not shown.

7
tests/fuzzFail/round_trip_custom10 Normal file
View File

@@ -0,0 +1,7 @@
<flago<o<modifiers=<<x-'o-a</o<<5ţ˙˙ůopt6=8
-'--R-(‹Í sÖlag=opt6=vJ8
-'-- -(‹5 --‹Í --=
--vC=opt6=8
-'-- -pt=on>a-'o-a˙˙ůO<x-lag>ÁÁoptggggggggggggggggggggggg'o-a˙˙ůO<x-lag>ÁÁopt4 » »h-
űűűEv˙
-.v˙˙'ÝflagtupĚ<70>oop2flag<<x-'g>t</flag>ttfapt2

BIN
tests/fuzzFail/round_trip_custom11 Normal file
View File

Binary file not shown.

BIN
tests/fuzzFail/round_trip_custom5 Normal file
View File

Binary file not shown.

109
tests/fuzzFail/round_trip_custom6 Normal file
View File

@@ -0,0 +1,109 @@
--vD=`
\
5=,,
pt

106
tests/fuzzFail/round_trip_custom7 Normal file
View File

@@ -0,0 +1,106 @@
--vD=`
'''
<vector>
5=,,
pt

BIN
tests/fuzzFail/round_trip_custom8 Normal file
View File

Binary file not shown.

1
tests/fuzzFail/round_trip_custom9 Normal file
View File

@@ -0,0 +1 @@
--flag=-Adlag=-AB-v,ag=-ABB-A-v""""""""""""""""""""""""""""""""""""""""""''''''''''''''''''''''''''''''''''''""""""""""""""",ag=-AB-"x