mirror of
https://github.com/binaricat/Netcatty.git
synced 2026-11-21 23:50:45 +00:00
* feat: add web search and URL fetch tools for AI agent Add web_search and url_fetch tools to Catty Agent, allowing the AI to search the internet for current information and fetch webpage content. - Support 5 search providers: Tavily, Exa, Bocha, Zhipu, SearXNG - Settings UI with provider selection, API key encryption, and config - web_search is conditional on config; url_fetch is always available - Both tools are read-only and work in all permission modes (incl. observer) - aiFetch skipHostCheck for AI tool requests to arbitrary URLs - System prompt guidelines for when to use search/fetch - i18n support (en + zh-CN) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address code review findings (SSRF, key exposure, state race) - P1: Restore SSRF protection when skipHostCheck is true — still block localhost, RFC1918, link-local, and cloud metadata endpoints; only skip the domain allowlist for public HTTPS hosts - P2: Move web search API key decryption to main process via dedicated IPC handler, matching the existing provider key security model - P2: Use configRef to avoid stale closure in async settings callbacks that could overwrite newer user changes Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address second review — DNS rebinding, url_fetch approval, maxResults - P1: url_fetch now requires approval in confirm mode (outbound GET is a side effect that could exfiltrate data via query strings) - P1: Add DNS resolution check when skipHostCheck is set — resolve hostname and reject if any IP is private/loopback/link-local, blocking DNS rebinding attacks against internal services - P2: Slice search results after provider call to enforce maxResults consistently (Zhipu and SearXNG ignore the limit parameter) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address third review — localhost/IPv6 SSRF, API key blur race - P1: Block localhost/loopback when skipHostCheck is enabled — restructure isAllowedFetchUrl to check private hosts first in the skipHostCheck path, preventing access to local services on allowlisted ports - P1: Handle IPv6 private ranges (fc00::/7, fe80::/10, ::ffff: mapped), strip brackets from URL.hostname, block [::1] and fd00:: addresses - P2: Guard handleApiKeyBlur against provider change during async encryption — skip stale write if provider switched while encrypting Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address fourth review — main-process key isolation, SearXNG compat - P1: Replace aiWebSearchDecryptKey IPC with __WEB_SEARCH_KEY__ placeholder pattern — renderer never sees plaintext keys; main process replaces placeholder in headers before HTTP request, matching provider key flow - P1: Search API requests use normal allowlist path (not skipHostCheck), so SearXNG on localhost/HTTP/private networks works via aiSyncWebSearch; only url_fetch uses skipHostCheck for arbitrary public HTTPS URLs - P2: Remove needsApproval from url_fetch — treat as read-only like sftp_read_file, consistent with observer mode allowlist Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address fifth review — private LAN providers, maxResults default - P1: Allow private-IP hosts that are explicitly in the provider/search allowlist (e.g. https://192.168.x.x model providers or SearXNG) - P2: Remove .default(5) from web_search maxResults schema so the user's configured maxResults setting is used when the model omits the param Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address sixth review — HTTPS scope, config gate, redirects - P2: Scope HTTP exception to private/LAN IPs only — remote allowlisted hosts still require HTTPS to protect API keys in transit - P2: Gate web_search tool on complete config (API key for providers that require it, apiHost for SearXNG) to avoid advertising a broken tool - P2: Add redirect following (up to 5 hops) to aiFetch for url_fetch — handles 301/302/307 for short links, www canonicalization, etc. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address seventh review — redirect SSRF, decrypt race, HTTPS-only - P1: Revalidate each redirect hop against SSRF guards (allowlist check + DNS resolution) before following, preventing open-redirect SSRF - P2: Add sequence counter to API key decryption effect — stale promise results from a previous provider are discarded on provider switch - P3: Restrict url_fetch to HTTPS-only URLs, matching the skipHostCheck policy that already rejects HTTP in the bridge Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address eighth review — OS resolver, allowlisted HTTP hosts - P1: Use dns.lookup (OS resolver) instead of dns.resolve4/6 for private IP checks — matches what http.request actually connects to, respects /etc/hosts, mDNS, and other local resolver sources - P2: Allow HTTP for any explicitly allowlisted host (not just literal private IPs), so self-hosted SearXNG at http://searxng.lan works Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address ninth review — HTTP scope, blur ordering, decrypt flag - P1: Narrow HTTP exception to web search apiHost only — AI provider endpoints remain HTTPS-only to protect credentials in transit - P2: Add blur sequence counter to prevent out-of-order encryption results from overwriting newer API key saves - P2: Reset isDecrypting flag when cancelling decrypt on provider switch, preventing permanently disabled API key input Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address tenth review — DNS pinning, prompt/tool alignment - P1: Pin validated DNS result to the HTTP request via custom lookup function, preventing TOCTOU/DNS-rebinding between validation and actual connection - P2: Extract isWebSearchReady() helper and use it consistently in both tool registration and system prompt, so the model isn't told web search is available when config is incomplete Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address eleventh review — single DNS lookup, redirect pinning, CGNAT - P1: Combine DNS validation and pinning into a single lookup call, eliminating the TOCTOU window between hasPrivateResolution and pinnedLookup - P1: Pin DNS for redirect targets too — resolve/validate/pin in one step before following each redirect hop - P2: Add 100.64.0.0/10 (CGNAT) to private IP ranges for Tailscale and similar CGNAT-addressed internal services Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: address twelfth review — apiHost validation, sync on enable - P2: Validate apiHost is a well-formed URL in isWebSearchReady(), preventing tool exposure when user enters a malformed host - P2: Add webSearchConfig.enabled to sync effect deps so the main process gets updated immediately when the toggle changes Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: remove DNS-level SSRF checks that break fakedns/proxy environments DNS resolution validation (dns.lookup + IP pinning) breaks in proxy environments where fakedns resolves all domains to LAN addresses. Revert to hostname-level checks only (blocking localhost, 127.0.0.1, metadata endpoints, etc.) which are sufficient without false positives. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: resolve empty catch block lint warning Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
215 lines
6.8 KiB
TypeScript
215 lines
6.8 KiB
TypeScript
/**
|
|
* Web search provider implementations.
|
|
*
|
|
* Each provider function normalises its API response into a common
|
|
* `{ results: Array<{ title, url, content }> }` shape so callers don't need
|
|
* to know about provider-specific quirks.
|
|
*
|
|
* All HTTP requests go through `bridge.aiFetch()` to avoid CORS issues in the
|
|
* renderer process.
|
|
*/
|
|
|
|
import type { NetcattyBridge } from '../cattyAgent/executor';
|
|
import type { WebSearchConfig } from '../types';
|
|
import { WEB_SEARCH_PROVIDER_PRESETS } from '../types';
|
|
|
|
export interface WebSearchResult {
|
|
title: string;
|
|
url: string;
|
|
content: string;
|
|
}
|
|
|
|
interface BridgeFetchResponse {
|
|
ok: boolean;
|
|
status?: number;
|
|
data?: string;
|
|
error?: string;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helper
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function resolveApiHost(config: WebSearchConfig): string {
|
|
return config.apiHost || WEB_SEARCH_PROVIDER_PRESETS[config.providerId].defaultApiHost;
|
|
}
|
|
|
|
async function fetchJson(
|
|
bridge: NetcattyBridge,
|
|
url: string,
|
|
method: string,
|
|
headers: Record<string, string>,
|
|
body?: string,
|
|
): Promise<unknown> {
|
|
const aiFetch = (bridge as unknown as Record<string, (...args: unknown[]) => Promise<unknown>>).aiFetch;
|
|
if (!aiFetch) throw new Error('aiFetch is not available on the bridge');
|
|
// Search API hosts are added to the allowlist via aiSyncWebSearch, no skipHostCheck needed
|
|
const resp = await aiFetch(url, method, headers, body) as BridgeFetchResponse;
|
|
if (!resp.ok) throw new Error(resp.error || `HTTP ${resp.status}`);
|
|
return JSON.parse(resp.data || '{}');
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Tavily
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function searchTavily(
|
|
bridge: NetcattyBridge,
|
|
config: WebSearchConfig,
|
|
query: string,
|
|
maxResults: number,
|
|
): Promise<WebSearchResult[]> {
|
|
const host = resolveApiHost(config);
|
|
const data = await fetchJson(bridge, `${host}/search`, 'POST', {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': `Bearer ${config.apiKey}`,
|
|
}, JSON.stringify({
|
|
query,
|
|
max_results: maxResults,
|
|
search_depth: 'basic',
|
|
})) as { results?: Array<{ title?: string; url?: string; content?: string }> };
|
|
|
|
return (data.results || []).map(r => ({
|
|
title: r.title || '',
|
|
url: r.url || '',
|
|
content: r.content || '',
|
|
}));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Exa
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function searchExa(
|
|
bridge: NetcattyBridge,
|
|
config: WebSearchConfig,
|
|
query: string,
|
|
maxResults: number,
|
|
): Promise<WebSearchResult[]> {
|
|
const host = resolveApiHost(config);
|
|
const data = await fetchJson(bridge, `${host}/search`, 'POST', {
|
|
'Content-Type': 'application/json',
|
|
'x-api-key': config.apiKey || '',
|
|
}, JSON.stringify({
|
|
query,
|
|
numResults: maxResults,
|
|
contents: { text: true },
|
|
})) as { results?: Array<{ title?: string; url?: string; text?: string }> };
|
|
|
|
return (data.results || []).map(r => ({
|
|
title: r.title || '',
|
|
url: r.url || '',
|
|
content: r.text || '',
|
|
}));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Bocha
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function searchBocha(
|
|
bridge: NetcattyBridge,
|
|
config: WebSearchConfig,
|
|
query: string,
|
|
maxResults: number,
|
|
): Promise<WebSearchResult[]> {
|
|
const host = resolveApiHost(config);
|
|
const data = await fetchJson(bridge, `${host}/v1/web-search`, 'POST', {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': `Bearer ${config.apiKey}`,
|
|
}, JSON.stringify({
|
|
query,
|
|
count: maxResults,
|
|
summary: true,
|
|
})) as { webPages?: { value?: Array<{ name?: string; url?: string; snippet?: string; summary?: string }> } };
|
|
|
|
return (data.webPages?.value || []).map(r => ({
|
|
title: r.name || '',
|
|
url: r.url || '',
|
|
content: r.summary || r.snippet || '',
|
|
}));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Zhipu
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function searchZhipu(
|
|
bridge: NetcattyBridge,
|
|
config: WebSearchConfig,
|
|
query: string,
|
|
_maxResults: number,
|
|
): Promise<WebSearchResult[]> {
|
|
const host = resolveApiHost(config);
|
|
const data = await fetchJson(bridge, `${host}/web_search`, 'POST', {
|
|
'Content-Type': 'application/json',
|
|
'Authorization': `Bearer ${config.apiKey}`,
|
|
}, JSON.stringify({
|
|
search_query: query,
|
|
search_engine: 'search_std',
|
|
})) as { search_result?: Array<{ title?: string; link?: string; content?: string }> };
|
|
|
|
return (data.search_result || []).map(r => ({
|
|
title: r.title || '',
|
|
url: r.link || '',
|
|
content: r.content || '',
|
|
}));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// SearXNG
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function searchSearxng(
|
|
bridge: NetcattyBridge,
|
|
config: WebSearchConfig,
|
|
query: string,
|
|
_maxResults: number,
|
|
): Promise<WebSearchResult[]> {
|
|
const host = resolveApiHost(config);
|
|
if (!host) throw new Error('SearXNG requires an API Host to be configured');
|
|
const url = `${host}/search?q=${encodeURIComponent(query)}&format=json`;
|
|
const data = await fetchJson(bridge, url, 'GET', {}) as {
|
|
results?: Array<{ title?: string; url?: string; content?: string }>;
|
|
};
|
|
|
|
return (data.results || []).map(r => ({
|
|
title: r.title || '',
|
|
url: r.url || '',
|
|
content: r.content || '',
|
|
}));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Dispatcher
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const PROVIDER_SEARCH_FNS: Record<string, typeof searchTavily> = {
|
|
tavily: searchTavily,
|
|
exa: searchExa,
|
|
bocha: searchBocha,
|
|
zhipu: searchZhipu,
|
|
searxng: searchSearxng,
|
|
};
|
|
|
|
/**
|
|
* Placeholder token for the web search API key.
|
|
* The renderer sends this in HTTP headers; the main process replaces it
|
|
* with the real decrypted key before the request is sent, so plaintext
|
|
* keys never enter the renderer.
|
|
*/
|
|
const WEB_SEARCH_KEY_PLACEHOLDER = '__WEB_SEARCH_KEY__';
|
|
|
|
export async function executeWebSearchProvider(
|
|
bridge: NetcattyBridge,
|
|
config: WebSearchConfig,
|
|
query: string,
|
|
maxResults: number,
|
|
): Promise<WebSearchResult[]> {
|
|
const fn = PROVIDER_SEARCH_FNS[config.providerId];
|
|
if (!fn) throw new Error(`Unsupported web search provider: ${config.providerId}`);
|
|
// Use placeholder — main process replaces with real decrypted key before HTTP request
|
|
const safeConfig = { ...config, apiKey: WEB_SEARCH_KEY_PLACEHOLDER };
|
|
return fn(bridge, safeConfig, query, maxResults);
|
|
}
|