threecode/web

Web helpers: fetch a URL and return readable text, or run a Startpage search and return a compact list of hits. Exposed to the agent as native web_search and web_fetch tool calls (dispatched in actions.nim).

No external binaries, no scripting runtimes, pure Nim httpclient + a hand-rolled HTML-to-text pass.

Types

SearchHit = object
  title*, url*, snippet*: string

Consts

DefaultSearchUrl = "https://www.startpage.com/do/search?cat=web&q="

Procs

proc capText(s: string; cap = DefaultFetchCap): string {....raises: [], tags: [],
    forbids: [].}
proc decodeEntities(s: string): string {....raises: [KeyError], tags: [],
    forbids: [].}
proc fetchUrl(url: string): string {....raises: [LibraryError, SslError, Exception,
    IOError, KeyError, OSError, ValueError, HttpRequestError, TimeoutError,
    ProtocolError], tags: [RootEffect, ReadDirEffect, ReadEnvEffect,
                           ReadIOEffect, WriteIOEffect, TimeEffect], forbids: [].}
proc formatHits(hits: seq[SearchHit]): string {....raises: [], tags: [],
    forbids: [].}
proc parseSearchHits(html: string): seq[SearchHit] {....raises: [KeyError],
    tags: [], forbids: [].}
Extract Startpage organic web results. Each hit's anchor carries data-testid="gl-title-link" and the title sits inside an <h2>; the snippet follows in a <p class="description ...">. Class names are stable identifiers; the surrounding emotion-css hashes are not, so we never match on them.
proc stripHtml(html: string): string {....raises: [KeyError], tags: [], forbids: [].}
proc webSearch(query: string; searchUrl = DefaultSearchUrl): seq[SearchHit] {....raises: [
    LibraryError, SslError, Exception, IOError, KeyError, OSError, ValueError,
    HttpRequestError, TimeoutError, ProtocolError], tags: [RootEffect,
    ReadDirEffect, ReadEnvEffect, ReadIOEffect, WriteIOEffect, TimeEffect],
    forbids: [].}