add a round trip test to the fuzzer (#1060)

This is the next phase of the fuzzer. It runs a round trip and makes
sure that the config files generated by the app will load into the same
results, to test full round trip on the config files.

Issues fixed
- fix a bug in the string escape code caught by initial round trip tests
- resolve inconsistencies in handling of {} for empty vector indication
between config and cli parsing

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

fuzz/CMakeLists.txt

@@ -31,7 +31,7 @@ if(CMAKE_CXX_STANDARD GREATER 16)
       COMMAND
         cli11_app_fuzzer corp -max_total_time=${CLI11_FUZZ_TIME_APP} -max_len=2148
         -dict=${CMAKE_CURRENT_SOURCE_DIR}/fuzz_dictionary1.txt
-        -exact_artifact_path=${CLI11_FUZZ_ARTIFACT_PATH}/cli11_app_fail_artifact.txt)
+        -exact_artifact_path=${CLI11_FUZZ_ARTIFACT_PATH}/cli11_app_roundtrip_fail_artifact.txt)
 
     add_custom_target(
       QUICK_CLI11_FILE_FUZZ

fuzz/cli11_app_fuzz.cpp

@@ -26,13 +26,17 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
         parseString.erase(0, 25);
     }
     CLI::FuzzApp fuzzdata;
+    CLI::FuzzApp fuzzdata2;
     auto app = fuzzdata.generateApp();
+    auto app2 = fuzzdata2.generateApp();
     try {
         if(!optionString.empty()) {
             app->add_option(optionString, fuzzdata.buffer);
+            app2->add_option(optionString, fuzzdata2.buffer);
         }
         if(!flagString.empty()) {
             app->add_flag(flagString, fuzzdata.intbuffer);
+            app2->add_flag(flagString, fuzzdata2.intbuffer);
         }
     } catch(const CLI::ConstructionError &e) {
         return 0;  // Non-zero return values are reserved for future use.
@@ -50,6 +54,10 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     std::string configOut = app->config_to_str();
     app->clear();
     std::stringstream out(configOut);
-    app->parse_from_stream(out);
+    app2->parse_from_stream(out);
+    auto result = fuzzdata2.compare(fuzzdata);
+    if(!result) {
+        throw CLI::ValidationError("fuzzer", "file input results don't match parse results");
+    }
     return 0;
 }

fuzz/fuzzApp.cpp

@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: BSD-3-Clause
 
 #include "fuzzApp.hpp"
+#include <algorithm>
 
 namespace CLI {
 /*
@@ -148,4 +149,147 @@ std::shared_ptr<CLI::App> FuzzApp::generateApp() {
     return fApp;
 }
 
+bool FuzzApp::compare(const FuzzApp &other) const {
+    if(val32 != other.val32) {
+        return false;
+    }
+    if(val16 != other.val16) {
+        return false;
+    }
+    if(val8 != other.val8) {
+        return false;
+    }
+    if(val64 != other.val64) {
+        return false;
+    }
+
+    if(uval32 != other.uval32) {
+        return false;
+    }
+    if(uval16 != other.uval16) {
+        return false;
+    }
+    if(uval8 != other.uval8) {
+        return false;
+    }
+    if(uval64 != other.uval64) {
+        return false;
+    }
+
+    if(atomicval64 != other.atomicval64) {
+        return false;
+    }
+    if(atomicuval64 != other.atomicuval64) {
+        return false;
+    }
+
+    if(v1 != other.v1) {
+        return false;
+    }
+    if(v2 != other.v2) {
+        return false;
+    }
+
+    if(vv1 != other.vv1) {
+        return false;
+    }
+    if(vstr != other.vstr) {
+        return false;
+    }
+
+    if(vecvecd != other.vecvecd) {
+        return false;
+    }
+    if(vvs != other.vvs) {
+        return false;
+    }
+    if(od1 != other.od1) {
+        return false;
+    }
+    if(ods != other.ods) {
+        return false;
+    }
+    if(p1 != other.p1) {
+        return false;
+    }
+    if(p2 != other.p2) {
+        return false;
+    }
+    if(t1 != other.t1) {
+        return false;
+    }
+    if(tcomplex != other.tcomplex) {
+        return false;
+    }
+    if(tcomplex2 != other.tcomplex2) {
+        return false;
+    }
+    if(vectup != other.vectup) {
+        return false;
+    }
+    if(vstrv != other.vstrv) {
+        return false;
+    }
+
+    if(flag1 != other.flag1) {
+        return false;
+    }
+    if(flagCnt != other.flagCnt) {
+        return false;
+    }
+    if(flagAtomic != other.flagAtomic) {
+        return false;
+    }
+
+    if(iwrap.value() != other.iwrap.value()) {
+        return false;
+    }
+    if(dwrap.value() != other.dwrap.value()) {
+        return false;
+    }
+    if(swrap.value() != other.swrap.value()) {
+        return false;
+    }
+    if(buffer != other.buffer) {
+        return false;
+    }
+    if(intbuffer != other.intbuffer) {
+        return false;
+    }
+    if(doubleAtomic != other.doubleAtomic) {
+        return false;
+    }
+
+    // for testing restrictions and reduction methods
+    if(vstrA != other.vstrA) {
+        return false;
+    }
+    if(vstrB != other.vstrB) {
+        return false;
+    }
+    if(vstrC != other.vstrC) {
+        return false;
+    }
+    if(vstrD != other.vstrD) {
+        // the return result if reversed so it can alternate
+        std::vector<std::string> res = vstrD;
+        std::reverse(res.begin(), res.end());
+        if(res != other.vstrD) {
+            return false;
+        }
+    }
+    if(vstrE != other.vstrE) {
+        return false;
+    }
+    if(vstrF != other.vstrF) {
+        return false;
+    }
+    if(mergeBuffer != other.mergeBuffer) {
+        return false;
+    }
+    if(validator_strings != other.validator_strings) {
+        return false;
+    }
+    return true;
+}
 }  // namespace CLI

fuzz/fuzzApp.hpp

@@ -54,8 +54,10 @@ class stringWrapper {
 class FuzzApp {
   public:
     FuzzApp() = default;
-
+    /** generate a fuzzing application with a bunch of different interfaces*/
     std::shared_ptr<CLI::App> generateApp();
+    /** compare two fuzz apps for equality*/
+    CLI11_NODISCARD bool compare(const FuzzApp &other) const;
 
     int32_t val32{0};
     int16_t val16{0};