Implement web search.

cmake/tidy-cmake.patch

@@ -0,0 +1,39 @@
+diff --git i/CMakeLists.txt w/CMakeLists.txt
+index 8efec25..c1d101e 100644
+--- i/CMakeLists.txt
++++ w/CMakeLists.txt
+@@ -17,7 +17,7 @@
+ # @date    Consult git log.
+ ##############################################################################
+ 
+-cmake_minimum_required (VERSION 2.8.12)
++cmake_minimum_required (VERSION 3.10)
+ 
+ set(LIB_NAME tidy)
+ set(LIBTIDY_DESCRIPTION "${LIB_NAME} - HTML syntax checker")
+@@ -528,6 +528,7 @@ if (UNIX AND SUPPORT_CONSOLE_APP)
+  
+         # Run the built EXE to generate xml output .
+         add_custom_command(
++            POST_BUILD
+             TARGET man
+             COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME} -xml-help > ${TIDYHELP}
+             COMMENT "Generate ${TIDYHELP}"
+@@ -536,6 +537,7 @@ if (UNIX AND SUPPORT_CONSOLE_APP)
+ 
+         # Run the built EXE to generate more xml output.
+         add_custom_command(
++            POST_BUILD
+             TARGET man
+             COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME} -xml-config > ${TIDYCONFIG}
+             COMMENT "Generate ${TIDYCONFIG}"
+@@ -544,8 +546,8 @@ if (UNIX AND SUPPORT_CONSOLE_APP)
+ 
+         # Run xsltproc to generate the install files.
+         add_custom_command(
++            POST_BUILD
+             TARGET man
+-            DEPENDS ${TIDYHELP}
+             COMMAND xsltproc ARGS ${TIDY1XSL} ${TIDYHELP} > ${CMAKE_CURRENT_BINARY_DIR}/${TIDY_MANFILE}
+             COMMENT "Generate ${TIDY_MANFILE}"
+             VERBATIM

cmake/vendor.cmake

@@ -12,6 +12,7 @@ option(DOWNLOAD_CAPSTONE "Force download capstone" ON)
 option(DOWNLOAD_GLFW "Force download glfw" OFF)
 option(DOWNLOAD_FREETYPE "Force download freetype" OFF)
 option(DOWNLOAD_LIBCURL "Force download libcURL" OFF)
+option(DOWNLOAD_PUGIXML "Force download pugixml" OFF)
 
 # capstone
 
@@ -137,6 +138,24 @@ else()
     target_link_libraries(TracyLibcurl INTERFACE curl)
 endif()
 
+# pugixml
+
+pkg_check_modules(PUGIXML pugixml)
+if (PUGIXML_FOUND AND NOT DOWNLOAD_PUGIXML)
+    add_library(TracyPugixml INTERFACE)
+    target_include_directories(TracyPugixml INTERFACE ${PUGIXML_INCLUDE_DIRS})
+    target_link_libraries(TracyPugixml INTERFACE ${PUGIXML_LINK_LIBRARIES})
+else()
+    CPMAddPackage(
+        NAME pugixml
+        GITHUB_REPOSITORY zeux/pugixml
+        GIT_TAG v1.15
+        EXCLUDE_FROM_ALL TRUE
+    )
+    add_library(TracyPugixml INTERFACE)
+    target_link_libraries(TracyPugixml INTERFACE pugixml)
+endif()
+
 # Diff Template Library
 
 set(DTL_DIR "${ROOT_DIR}/dtl")
@@ -239,3 +258,14 @@ CPMAddPackage(
     EXCLUDE_FROM_ALL TRUE
 )
 set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_SAVE})
+
+# tidy
+
+CPMAddPackage(
+    NAME tidy
+    GITHUB_REPOSITORY htacg/tidy-html5
+    GIT_TAG 5.8.0
+    PATCHES
+        "${CMAKE_CURRENT_LIST_DIR}/tidy-cmake.patch"
+    EXCLUDE_FROM_ALL TRUE
+)

profiler/CMakeLists.txt

@@ -206,8 +206,8 @@ else()
 endif()
 
 find_package(Threads REQUIRED)
-target_link_libraries(${PROJECT_NAME} PRIVATE TracyServer TracyImGui Threads::Threads TracyLibcurl base64)
-target_include_directories(${PROJECT_NAME} PRIVATE ${ollama-hpp_SOURCE_DIR}/include)
+target_link_libraries(${PROJECT_NAME} PRIVATE TracyServer TracyImGui Threads::Threads TracyLibcurl base64 tidy-static TracyPugixml)
+target_include_directories(${PROJECT_NAME} PRIVATE ${ollama-hpp_SOURCE_DIR}/include ${tidy_SOURCE_DIR}/include)
 
 if(NOT DEFINED GIT_REV)
     set(GIT_REV "HEAD")

profiler/src/llm/system.prompt

@@ -51,6 +51,15 @@ These are the tools available to you. There are no other tools, and you do not h
         "description": "The language code."
       }
     ]
+  },
+  {
+    "name": "search_web",
+    "description": "Search the web with given query.",
+    "parameters": [
+      {
+        "description": "Search query."
+      }
+    ]
   }
 ]
 

profiler/src/profiler/TracyLlm.cpp

@@ -3,7 +3,10 @@
 #include <ollama.hpp>
 #include <stdint.h>
 #include <stdlib.h>
+#include <pugixml.hpp>
 #include <ranges>
+#include <tidy.h>
+#include <tidybuffio.h>
 #include <time.h>
 
 #include "TracyConfig.hpp"
@@ -840,6 +843,11 @@ TracyLlm::ToolReply TracyLlm::HandleToolCalls( const std::string& name, const st
         if( args.size() < 2 ) return { .reply = "Missing language argument" };
         return { .reply = GetWikipedia( args[0], args[1] ) };
     }
+    if( name == "search_web" )
+    {
+        if( args.empty() ) return { .reply = "Missing search term argument" };
+        return { .reply = SearchWeb( args[0] ) };
+    }
     return { .reply = "Unknown tool call: " + name };
 }
 
@@ -971,4 +979,80 @@ std::string TracyLlm::GetWikipedia( std::string page, const std::string& lang )
     return res;
 }
 
+static std::string RemoveNewline( std::string str )
+{
+    std::erase( str, '\r' );
+    std::ranges::replace( str, '\n', ' ' );
+    return str;
+}
+
+std::string TracyLlm::SearchWeb( std::string query )
+{
+    std::ranges::replace( query, ' ', '+' );
+    const auto response = FetchWebPage( "https://lite.duckduckgo.com/lite?q=" + UrlEncode( query ) );
+
+    TidyBuffer err = {};
+    tidyBufInit( &err );
+
+    TidyDoc td = tidyCreate();
+    tidyOptSetBool( td, TidyXhtmlOut, yes );
+    tidyOptSetBool( td, TidyLowerLiterals, yes );
+    tidyOptSetBool( td, TidyMark, no );
+    tidyOptSetBool( td, TidyHideComments, yes );
+    tidySetErrorBuffer( td, &err );
+
+    if( tidyParseString( td, response.c_str() ) == 2 )
+    {
+        auto out = std::string( (const char*)err.bp );
+        tidyBufFree( &err );
+        tidyRelease( td );
+        return out;
+    }
+
+    TidyBuffer buf = {};
+    tidyBufInit( &buf );
+    tidyCleanAndRepair( td );
+    tidySaveBuffer( td, &buf );
+
+    auto tidy = std::string( (const char*)buf.bp );
+
+    tidyBufFree( &buf );
+    tidyBufFree( &err );
+    tidyRelease( td );
+
+    auto doc = std::make_unique<pugi::xml_document>();
+    if( !doc->load_string( tidy.c_str() ) )
+    {
+        return "Error: Failed to parse HTML";
+    }
+
+    const auto titles = doc->select_nodes( "//a[@class='result-link']" );
+    const auto snippets = doc->select_nodes( "//td[@class='result-snippet']" );
+    const auto urls = doc->select_nodes( "//span[@class='link-text']" );
+
+    const auto sz = titles.size();
+    if( sz != snippets.size() || sz != urls.size() )
+    {
+        return "Error: Failed to parse HTML";
+    }
+
+    nlohmann::json json;
+
+    for( size_t i = 0; i < sz; i++ )
+    {
+        auto title = titles[i].node();
+        auto snippet = snippets[i].node();
+        auto url = urls[i].node();
+
+        nlohmann::json result;
+        result["title"] = RemoveNewline( title.text().as_string() );
+        result["snippet"] = RemoveNewline( snippet.text().as_string() );
+        result["url"] = RemoveNewline( url.text().as_string() );
+
+        json[i] = result;
+    }
+
+    return json.dump( 2 );
+}
+
 }

profiler/src/profiler/TracyLlm.hpp

@@ -103,6 +103,7 @@ private:
     std::string FetchWebPage( const std::string& url );
     ToolReply SearchWikipedia( std::string query, const std::string& lang );
     std::string GetWikipedia( std::string page, const std::string& lang );
+    std::string SearchWeb( std::string query );
 
     std::unique_ptr<Ollama> m_ollama;