add option modifiers pathways to custom options in the fuzzer (#1128)

Next phase of the fuzzer, adds some mechanics for the fuzzer itself to
add modifiers on the options.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/fuzz.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - v*
   pull_request:
 
 concurrency:

.gitignore

@@ -1,12 +1,14 @@
 a.out*
 *.swp
 /*build*
+/*out*
 /test_package/build
 /Makefile
 /CMakeFiles/*
 /cmake_install.cmake
 /*.kdev4
 /.vscode
+/.vs
 /html/*
 !/meson.build
 /CMakeUserPresets.json

fuzz/CMakeLists.txt

@@ -20,10 +20,10 @@ if(CMAKE_CXX_STANDARD GREATER 16)
     endif()
 
     if(NOT CLI11_FUZZ_TIME_APP)
-      set(CLI11_FUZZ_TIME_APP 600)
+      set(CLI11_FUZZ_TIME_APP 660)
     endif()
     if(NOT CLI11_FUZZ_TIME_FILE)
-      set(CLI11_FUZZ_TIME_FILE 240)
+      set(CLI11_FUZZ_TIME_FILE 200)
     endif()
     add_custom_target(
       QUICK_CLI11_APP_FUZZ

fuzz/cli11_app_fuzz.cpp

@@ -33,7 +33,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
         } else {
             app->parse(parseString);
         }
-
+    } catch(const CLI::HorribleError &he) {
+        throw;
     } catch(const CLI::ParseError &e) {
         //(app)->exit(e);
         // this just indicates we caught an error known by CLI

fuzz/cli11_file_fuzz.cpp

@@ -28,7 +28,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
         app->clear();
         std::stringstream out(configOut);
         app->parse_from_stream(out);
-
+    } catch(const CLI::HorribleError &he) {
+        throw;
     } catch(const CLI::ParseError &e) {
         // (app)->exit(e);
         // this just indicates we caught an error known by CLI

fuzz/fuzzApp.cpp

@@ -76,6 +76,7 @@ std::shared_ptr<CLI::App> FuzzApp::generateApp() {
 
     fApp->add_option("--oopt1", od1);
     fApp->add_option("--oopt2", ods);
+    fApp->add_option("--ovopt", ovs);
 
     fApp->add_option("--tup1", p1);
     fApp->add_option("--tup2", t1);
@@ -210,6 +211,9 @@ bool FuzzApp::compare(const FuzzApp &other) const {
     if(ods != other.ods) {
         return false;
     }
+    if(ovs != other.ovs) {
+        return false;
+    }
     if(p1 != other.p1) {
         return false;
     }
@@ -312,11 +316,114 @@ bool FuzzApp::compare(const FuzzApp &other) const {
     return true;
 }
 
+void FuzzApp::modify_option(CLI::Option *opt, const std::string &modifier_string) {
+    auto mod_start = modifier_string.find("modifiers=");
+    if(mod_start == std::string::npos) {
+        return;
+    }
+    auto mod_end = modifier_string.find_first_of(' ', mod_start + 1);
+    std::string modifiers = modifier_string.substr(mod_start + 10, mod_end - mod_start - 10);
+    for(const auto mod : modifiers) {
+        switch(mod) {
+        case 'r':
+        case 'R':
+            opt->required(mod < '`');
+            break;
+        case '0':
+        case '1':
+        case '2':
+        case '3':
+        case '4':
+            opt->expected(mod - '0');
+            break;
+        case '5':
+        case '6':
+        case '7':
+        case '8':
+        case '9':
+            opt->expected(opt->get_expected_min(), mod - '5');
+            break;
+        case 'c':
+        case 'C':
+            opt->ignore_case(mod < '`');
+            break;
+        case 'u':
+        case 'U':
+            opt->ignore_underscore(mod < '`');
+            break;
+        case 'f':
+        case 'F':
+            opt->disable_flag_override(mod < '`');
+            break;
+        case 'e':
+        case 'E':
+            opt->allow_extra_args(mod < '`');
+            break;
+        case ',':
+        case ';':
+        case '%':
+        case '#':
+        case '|':
+        case '\\':
+        case '~':
+        case ':':
+            opt->delimiter(mod);
+            break;
+        case 'g':
+        case 'G':
+            opt->configurable(mod < '`');
+            break;
+        case 'p':
+        case 'P':
+            opt->trigger_on_parse(mod < '`');
+            break;
+        case 't':
+        case 'T':
+            opt->multi_option_policy(CLI::MultiOptionPolicy::Throw);
+            break;
+        case 'l':
+        case 'L':
+            opt->multi_option_policy(CLI::MultiOptionPolicy::TakeLast);
+            break;
+        case 's':
+        case 'S':
+            opt->multi_option_policy(CLI::MultiOptionPolicy::TakeFirst);
+            break;
+        case 'a':
+        case 'A':
+            opt->multi_option_policy(CLI::MultiOptionPolicy::TakeAll);
+            break;
+        case 'v':
+        case 'V':
+            opt->multi_option_policy(CLI::MultiOptionPolicy::Reverse);
+            break;
+        case 'j':
+        case 'J':
+            opt->multi_option_policy(CLI::MultiOptionPolicy::Join);
+            break;
+        case '+':
+            opt->multi_option_policy(CLI::MultiOptionPolicy::Sum);
+            break;
+        case 'i':
+            opt->check(CLI::Number);
+            break;
+        case 'I':
+            opt->check(CLI::NonNegativeNumber);
+            break;
+        case 'w':
+            opt->check(!CLI::Number);
+            break;
+        default:
+            break;
+        }
+    }
+}
+
 //<option>name_string</option>
 //<vector>name_string</vector>
 //<flag>name_string</flag>
 /** generate additional options based on a string config*/
-std::size_t FuzzApp::add_custom_options(CLI::App *app, std::string &description_string) {
+std::size_t FuzzApp::add_custom_options(CLI::App *app, const std::string &description_string) {
     std::size_t current_index{0};
     while(description_string.size() - 5 > current_index && description_string[current_index] == '<') {
         if(description_string.compare(current_index, 7, "<option") == 0) {
@@ -330,7 +437,12 @@ std::size_t FuzzApp::add_custom_options(CLI::App *app, std::string &description_
             }
             std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
             custom_string_options.push_back(std::make_shared<std::string>());
-            app->add_option(name, *(custom_string_options.back()));
+            auto *opt = app->add_option(name, *(custom_string_options.back()));
+            if(header_close > current_index + 19) {
+                std::string attributes = description_string.substr(current_index + 8, header_close - 8 - current_index);
+                modify_option(opt, attributes);
+            }
+
             current_index = end_option + 9;
         } else if(description_string.compare(current_index, 5, "<flag") == 0) {
             auto end_option = description_string.find("</flag>", current_index + 6);
@@ -343,7 +455,12 @@ std::size_t FuzzApp::add_custom_options(CLI::App *app, std::string &description_
             }
             std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
             custom_string_options.push_back(std::make_shared<std::string>());
-            app->add_option(name, *(custom_string_options.back()));
+            auto *opt = app->add_option(name, *(custom_string_options.back()));
+
+            if(header_close > current_index + 17) {
+                std::string attributes = description_string.substr(current_index + 6, header_close - 6 - current_index);
+                modify_option(opt, attributes);
+            }
             current_index = end_option + 7;
         } else if(description_string.compare(current_index, 7, "<vector") == 0) {
             auto end_option = description_string.find("</vector>", current_index + 8);
@@ -356,7 +473,11 @@ std::size_t FuzzApp::add_custom_options(CLI::App *app, std::string &description_
             }
             std::string name = description_string.substr(header_close + 1, end_option - header_close - 1);
             custom_vector_options.push_back(std::make_shared<std::vector<std::string>>());
-            app->add_option(name, *(custom_string_options.back()));
+            auto *opt = app->add_option(name, *(custom_vector_options.back()));
+            if(header_close > current_index + 19) {
+                std::string attributes = description_string.substr(current_index + 8, header_close - 8 - current_index);
+                modify_option(opt, attributes);
+            }
             current_index = end_option + 9;
         } else {
             if(isspace(description_string[current_index]) != 0) {

fuzz/fuzzApp.hpp

@@ -59,7 +59,10 @@ class FuzzApp {
     /** compare two fuzz apps for equality*/
     CLI11_NODISCARD bool compare(const FuzzApp &other) const;
     /** generate additional options based on a string config*/
-    std::size_t add_custom_options(CLI::App *app, std::string &description_string);
+    std::size_t add_custom_options(CLI::App *app, const std::string &description_string);
+    /** modify an option based on string*/
+    void modify_option(CLI::Option *opt, const std::string &modifier);
+
     int32_t val32{0};
     int16_t val16{0};
     int8_t val8{0};
@@ -83,6 +86,7 @@ class FuzzApp {
     std::vector<std::vector<std::string>> vvs{};
     std::optional<double> od1{};
     std::optional<std::string> ods{};
+    std::optional<std::vector<std::string>> ovs{};
     std::pair<double, std::string> p1{};
     std::pair<std::vector<double>, std::string> p2{};
     std::tuple<int64_t, uint16_t, std::optional<double>> t1{};

fuzz/fuzz_dictionary1.txt

@@ -183,3 +183,4 @@
 "</flag>"
 ">-"
 ">--"
+"modifiers="

fuzz/fuzz_dictionary2.txt

@@ -90,3 +90,4 @@
 "vS"
 "vM"
 "vE"
+"modifiers="

tests/FuzzFailTest.cpp

@@ -248,6 +248,7 @@ TEST_CASE("app_roundtrip_single") {
     CHECK(result);
 }
 
+/** test the fuzzer itself to support custom options*/
 TEST_CASE("fuzz_config_test1") {
     CLI::FuzzApp fuzzdata;
     auto app = fuzzdata.generateApp();
@@ -261,6 +262,50 @@ TEST_CASE("fuzz_config_test1") {
     CHECK(app->get_option_no_throw("--new_vector") != nullptr);
 }
 
+/** test the fuzzer itself to support custom options*/
+TEST_CASE("fuzz_config_test2") {
+    CLI::FuzzApp fuzzdata;
+    auto app = fuzzdata.generateApp();
+
+    std::string config_string =
+        "<option>--new_option</option><flag>--new_flag</flag><vector>--new_vector</vector> --new_flag --new_option 10";
+    auto loc = fuzzdata.add_custom_options(app.get(), config_string);
+    config_string = config_string.substr(loc);
+    CHECK(!config_string.empty());
+    CHECK(config_string == " --new_flag --new_option 10");
+    CHECK(app->get_option_no_throw("--new_option") != nullptr);
+    CHECK(app->get_option_no_throw("--new_flag") != nullptr);
+    CHECK(app->get_option_no_throw("--new_vector") != nullptr);
+}
+
+/** test the fuzzer itself to support custom option modifiers*/
+TEST_CASE("fuzz_config_modifier_test1") {
+    CLI::FuzzApp fuzzdata;
+    auto app = fuzzdata.generateApp();
+
+    std::string config_string = "<option  modifiers=R2CG>--new_option</option><flag "
+                                "modifiers=cFg>--new_flag</flag><vector modifiers=35s+>--new_vector</vector>";
+    auto loc = fuzzdata.add_custom_options(app.get(), config_string);
+    config_string = config_string.substr(loc);
+    CHECK(config_string.empty());
+    auto *opt1 = app->get_option_no_throw("--new_option");
+    REQUIRE(opt1 != nullptr);
+    CHECK(opt1->get_required());
+    CHECK(opt1->get_expected_min() == 2);
+    CHECK(opt1->get_configurable());
+    CHECK(opt1->get_ignore_case());
+    auto *opt2 = app->get_option_no_throw("--new_flag");
+    REQUIRE(opt2 != nullptr);
+    CHECK(opt2->get_disable_flag_override());
+    CHECK(!opt2->get_configurable());
+    CHECK(!opt2->get_ignore_case());
+    auto *opt3 = app->get_option_no_throw("--new_vector");
+    REQUIRE(opt3 != nullptr);
+    CHECK(opt3->get_expected_min() == 0);
+    CHECK(opt3->get_expected_max() == 3);
+    CHECK(opt3->get_multi_option_policy() == CLI::MultiOptionPolicy::Sum);
+}
+
 // this test uses the same tests as above just with a full roundtrip test
 TEST_CASE("app_roundtrip_custom") {
     CLI::FuzzApp fuzzdata;