web_search: add searxng provider

This commit is contained in:
He-Xun committed 2026-01-30 15:33:37 +08:00
1 parent 613724c26e
commit 8d4270bcd6
12 files changed
+453 -40

No files matched your search

+40 -2
View File
@@ -2,8 +2,13 @@ import { describe, expect, it } from "vitest";
import { __testing } from "./web-search.js";
const { inferPerplexityBaseUrlFromApiKey, resolvePerplexityBaseUrl, normalizeFreshness } =
__testing;
const {
inferPerplexityBaseUrlFromApiKey,
resolvePerplexityBaseUrl,
normalizeFreshness,
resolveSearxngBaseUrl,
resolveSearxngHeaders,
} = __testing;
describe("web_search perplexity baseUrl defaults", () => {
it("detects a Perplexity key prefix", () => {
@@ -69,3 +74,36 @@ describe("web_search freshness normalization", () => {
expect(normalizeFreshness("2024-03-10to2024-03-01")).toBeUndefined();
});
});
describe("web_search searxng config resolution", () => {
it("prefers config baseUrl and trims trailing slash", () => {
expect(resolveSearxngBaseUrl({ baseUrl: "http://localhost:8080/" })).toBe(
"http://localhost:8080",
);
});
it("falls back to SEARXNG_BASE_URL env var", () => {
const prev = process.env.SEARXNG_BASE_URL;
process.env.SEARXNG_BASE_URL = "http://searxng.local:8888/";
try {
expect(resolveSearxngBaseUrl(undefined)).toBe("http://searxng.local:8888");
} finally {
if (prev === undefined) {
delete process.env.SEARXNG_BASE_URL;
} else {
process.env.SEARXNG_BASE_URL = prev;
}
}
});
it("builds headers with optional auth and extras", () => {
expect(resolveSearxngHeaders(undefined)).toEqual({ Accept: "application/json" });
expect(resolveSearxngHeaders({ apiKey: "token-123" }).Authorization).toBe("Bearer token-123");
expect(resolveSearxngHeaders({ apiKey: "Basic abc" }).Authorization).toBe("Basic abc");
const headers = resolveSearxngHeaders({ headers: { "X-Test": "ok" }, apiKey: "token-123" });
expect(headers["X-Test"]).toBe("ok");
expect(headers.Accept).toBe("application/json");
});
});
+234 -5
View File
@@ -17,7 +17,7 @@ import {
writeCache,
} from "./web-shared.js";
const SEARCH_PROVIDERS = ["brave", "perplexity"] as const;
const SEARCH_PROVIDERS = ["brave", "perplexity", "searxng"] as const;
const DEFAULT_SEARCH_COUNT = 5;
const MAX_SEARCH_COUNT = 10;
@@ -41,6 +41,39 @@ const WebSearchSchema = Type.Object({
maximum: MAX_SEARCH_COUNT,
}),
),
categories: Type.Optional(
Type.String({
description: "Comma-separated categories to search (SearXNG only), e.g. 'general,news'.",
}),
),
engines: Type.Optional(
Type.String({
description: "Comma-separated engines to search (SearXNG only), e.g. 'duckduckgo,bing'.",
}),
),
language: Type.Optional(
Type.String({
description: "Language code (SearXNG), e.g. 'en', 'de'.",
}),
),
time_range: Type.Optional(
Type.String({
description: "Time range (SearXNG only). Values: day, month, year.",
}),
),
safesearch: Type.Optional(
Type.Number({
description: "Safe search level (SearXNG only). Values: 0, 1, 2.",
minimum: 0,
maximum: 2,
}),
),
pageno: Type.Optional(
Type.Number({
description: "Search page number (SearXNG only). Default: 1.",
minimum: 1,
}),
),
country: Type.Optional(
Type.String({
description:
@@ -90,6 +123,13 @@ type PerplexityConfig = {
model?: string;
};
type SearxngConfig = {
baseUrl?: string;
apiKey?: string;
headers?: Record<string, string>;
params?: Record<string, string | number | boolean>;
};
type PerplexityApiKeySource = "config" | "perplexity_env" | "openrouter_env" | "none";
type PerplexitySearchResponse = {
@@ -103,6 +143,17 @@ type PerplexitySearchResponse = {
type PerplexityBaseUrlHint = "direct" | "openrouter";
type SearxngSearchResult = {
title?: string;
url?: string;
content?: string;
publishedDate?: string;
};
type SearxngSearchResponse = {
results?: SearxngSearchResult[];
};
function resolveSearchConfig(cfg?: OpenClawConfig): WebSearchConfig {
const search = cfg?.tools?.web?.search;
if (!search || typeof search !== "object") return undefined;
@@ -131,6 +182,14 @@ function missingSearchKeyPayload(provider: (typeof SEARCH_PROVIDERS)[number]) {
docs: "https://docs.openclaw.ai/tools/web",
};
}
if (provider === "searxng") {
return {
error: "missing_searxng_base_url",
message:
"web_search (searxng) needs a base URL. Configure tools.web.search.searxng.baseUrl or set SEARXNG_BASE_URL in the Gateway environment.",
docs: "https://docs.openclaw.ai/tools/web",
};
}
return {
error: "missing_brave_api_key",
message: `web_search needs a Brave Search API key. Run \`${formatCliCommand("openclaw configure --section web")}\` to store it, or set BRAVE_API_KEY in the Gateway environment.`,
@@ -145,6 +204,7 @@ function resolveSearchProvider(search?: WebSearchConfig): (typeof SEARCH_PROVIDE
: "";
if (raw === "perplexity") return "perplexity";
if (raw === "brave") return "brave";
if (raw === "searxng") return "searxng";
return "brave";
}
@@ -155,6 +215,13 @@ function resolvePerplexityConfig(search?: WebSearchConfig): PerplexityConfig {
return perplexity as PerplexityConfig;
}
function resolveSearxngConfig(search?: WebSearchConfig): SearxngConfig {
if (!search || typeof search !== "object") return {};
const searxng = "searxng" in search ? (search as Record<string, unknown>).searxng : undefined;
if (!searxng || typeof searxng !== "object") return {};
return searxng as SearxngConfig;
}
function resolvePerplexityApiKey(perplexity?: PerplexityConfig): {
apiKey?: string;
source: PerplexityApiKeySource;
@@ -221,6 +288,38 @@ function resolvePerplexityModel(perplexity?: PerplexityConfig): string {
return fromConfig || DEFAULT_PERPLEXITY_MODEL;
}
function resolveSearxngBaseUrl(searxng?: SearxngConfig): string | undefined {
const fromConfig =
searxng && "baseUrl" in searxng && typeof searxng.baseUrl === "string" ? searxng.baseUrl : "";
const fromEnv = (process.env.SEARXNG_BASE_URL ?? "").trim();
const baseUrl = (fromConfig ?? "").trim() || fromEnv;
if (!baseUrl) return undefined;
return baseUrl.replace(/\/$/, "");
}
function resolveSearxngHeaders(searxng?: SearxngConfig): Record<string, string> {
const headers: Record<string, string> = {
Accept: "application/json",
};
if (!searxng || typeof searxng !== "object") return headers;
if (searxng.headers && typeof searxng.headers === "object") {
for (const [key, value] of Object.entries(searxng.headers)) {
if (typeof value === "string" && value.trim()) {
headers[key] = value;
}
}
}
const apiKey = typeof searxng.apiKey === "string" ? searxng.apiKey.trim() : "";
if (apiKey) {
headers.Authorization = apiKey.includes(" ") ? apiKey : `Bearer ${apiKey}`;
}
return headers;
}
function resolveSearchCount(value: unknown, fallback: number): number {
const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
const clamped = Math.max(1, Math.min(MAX_SEARCH_COUNT, Math.floor(parsed)));
@@ -306,6 +405,72 @@ async function runPerplexitySearch(params: {
return { content, citations };
}
function addSearxngParams(
searchParams: URLSearchParams,
params?: Record<string, string | number | boolean>,
): void {
if (!params) return;
for (const [key, value] of Object.entries(params)) {
if (!key.trim()) continue;
if (value === undefined) continue;
searchParams.set(key, String(value));
}
}
async function runSearxngSearch(params: {
query: string;
baseUrl: string;
count: number;
timeoutSeconds: number;
searchParams?: Record<string, string | number | boolean>;
headers?: Record<string, string>;
categories?: string;
engines?: string;
language?: string;
time_range?: string;
safesearch?: number;
pageno?: number;
}): Promise<{ results: Array<Record<string, unknown>> }> {
const url = new URL(`${params.baseUrl.replace(/\/$/, "")}/search`);
addSearxngParams(url.searchParams, params.searchParams);
url.searchParams.set("q", params.query);
url.searchParams.set("format", "json");
if (params.categories) url.searchParams.set("categories", params.categories);
if (params.engines) url.searchParams.set("engines", params.engines);
if (params.language) url.searchParams.set("language", params.language);
if (params.time_range) url.searchParams.set("time_range", params.time_range);
if (typeof params.safesearch === "number" && Number.isFinite(params.safesearch)) {
url.searchParams.set("safesearch", String(params.safesearch));
}
if (typeof params.pageno === "number" && Number.isFinite(params.pageno) && params.pageno > 0) {
url.searchParams.set("pageno", String(Math.floor(params.pageno)));
}
const res = await fetch(url.toString(), {
method: "GET",
headers: params.headers,
signal: withTimeout(undefined, params.timeoutSeconds * 1000),
});
if (!res.ok) {
const detail = await readResponseText(res);
throw new Error(`SearXNG API error (${res.status}): ${detail || res.statusText}`);
}
const data = (await res.json()) as SearxngSearchResponse;
const results = Array.isArray(data.results) ? data.results : [];
const mapped = results.slice(0, params.count).map((entry) => ({
title: entry.title ?? "",
url: entry.url ?? "",
description: entry.content ?? "",
published: entry.publishedDate ?? undefined,
siteName: resolveSiteName(entry.url ?? ""),
}));
return { results: mapped };
}
async function runWebSearch(params: {
query: string;
count: number;
@@ -319,11 +484,22 @@ async function runWebSearch(params: {
freshness?: string;
perplexityBaseUrl?: string;
perplexityModel?: string;
searxngBaseUrl?: string;
searxngHeaders?: Record<string, string>;
searxngParams?: Record<string, string | number | boolean>;
categories?: string;
engines?: string;
language?: string;
time_range?: string;
safesearch?: number;
pageno?: number;
}): Promise<Record<string, unknown>> {
const cacheKey = normalizeCacheKey(
params.provider === "brave"
? `${params.provider}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || "default"}:${params.ui_lang || "default"}:${params.freshness || "default"}`
: `${params.provider}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || "default"}:${params.ui_lang || "default"}`,
: params.provider === "searxng"
? `${params.provider}:${params.query}:${params.count}:${params.categories || "default"}:${params.engines || "default"}:${params.language || "default"}:${params.time_range || "default"}:${String(params.safesearch ?? "default")}:${String(params.pageno ?? "default")}`
: `${params.provider}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || "default"}:${params.ui_lang || "default"}`,
);
const cached = readCache(SEARCH_CACHE, cacheKey);
if (cached) return { ...cached.value, cached: true };
@@ -351,6 +527,33 @@ async function runWebSearch(params: {
return payload;
}
if (params.provider === "searxng") {
const { results } = await runSearxngSearch({
query: params.query,
baseUrl: params.searxngBaseUrl ?? "",
count: params.count,
timeoutSeconds: params.timeoutSeconds,
searchParams: params.searxngParams,
headers: params.searxngHeaders,
categories: params.categories,
engines: params.engines,
language: params.language,
time_range: params.time_range,
safesearch: params.safesearch,
pageno: params.pageno,
});
const payload = {
query: params.query,
provider: params.provider,
count: results.length,
tookMs: Date.now() - start,
results,
};
writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs);
return payload;
}
if (params.provider !== "brave") {
throw new Error("Unsupported web search provider.");
}
@@ -415,11 +618,14 @@ export function createWebSearchTool(options?: {
const provider = resolveSearchProvider(search);
const perplexityConfig = resolvePerplexityConfig(search);
const searxngConfig = resolveSearxngConfig(search);
const description =
provider === "perplexity"
? "Search the web using Perplexity Sonar (direct or via OpenRouter). Returns AI-synthesized answers with citations from real-time web search."
: "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research.";
: provider === "searxng"
? "Search the web using a self-hosted SearXNG instance. Returns titles, URLs, and snippets for fast research."
: "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research.";
return {
label: "Web Search",
@@ -431,14 +637,26 @@ export function createWebSearchTool(options?: {
provider === "perplexity" ? resolvePerplexityApiKey(perplexityConfig) : undefined;
const apiKey =
provider === "perplexity" ? perplexityAuth?.apiKey : resolveSearchApiKey(search);
const searxngBaseUrl =
provider === "searxng" ? resolveSearxngBaseUrl(searxngConfig) : undefined;
if (!apiKey) {
if (provider === "searxng") {
if (!searxngBaseUrl) return jsonResult(missingSearchKeyPayload(provider));
} else if (!apiKey) {
return jsonResult(missingSearchKeyPayload(provider));
}
const params = args as Record<string, unknown>;
const query = readStringParam(params, "query", { required: true });
const count =
readNumberParam(params, "count", { integer: true }) ?? search?.maxResults ?? undefined;
const categories = readStringParam(params, "categories");
const engines = readStringParam(params, "engines");
const language =
readStringParam(params, "language") ??
(provider === "searxng" ? readStringParam(params, "search_lang") : undefined);
const time_range = readStringParam(params, "time_range");
const safesearch = readNumberParam(params, "safesearch", { integer: true });
const pageno = readNumberParam(params, "pageno", { integer: true });
const country = readStringParam(params, "country");
const search_lang = readStringParam(params, "search_lang");
const ui_lang = readStringParam(params, "ui_lang");
@@ -462,7 +680,7 @@ export function createWebSearchTool(options?: {
const result = await runWebSearch({
query,
count: resolveSearchCount(count, DEFAULT_SEARCH_COUNT),
apiKey,
apiKey: apiKey ?? "",
timeoutSeconds: resolveTimeoutSeconds(search?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS),
cacheTtlMs: resolveCacheTtlMs(search?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),
provider,
@@ -476,6 +694,15 @@ export function createWebSearchTool(options?: {
perplexityAuth?.apiKey,
),
perplexityModel: resolvePerplexityModel(perplexityConfig),
searxngBaseUrl,
searxngHeaders: provider === "searxng" ? resolveSearxngHeaders(searxngConfig) : undefined,
searxngParams: provider === "searxng" ? searxngConfig.params : undefined,
categories,
engines,
language,
time_range,
safesearch: typeof safesearch === "number" ? safesearch : undefined,
pageno: typeof pageno === "number" ? pageno : undefined,
});
return jsonResult(result);
},
@@ -486,4 +713,6 @@ export const __testing = {
inferPerplexityBaseUrlFromApiKey,
resolvePerplexityBaseUrl,
normalizeFreshness,
resolveSearxngBaseUrl,
resolveSearxngHeaders,
} as const;
+15 -2
View File
@@ -193,6 +193,10 @@ const FIELD_LABELS: Record<string, string> = {
"tools.web.search.maxResults": "Web Search Max Results",
"tools.web.search.timeoutSeconds": "Web Search Timeout (sec)",
"tools.web.search.cacheTtlMinutes": "Web Search Cache TTL (min)",
"tools.web.search.searxng.baseUrl": "SearXNG Base URL",
"tools.web.search.searxng.apiKey": "SearXNG API Key",
"tools.web.search.searxng.headers": "SearXNG Extra Headers",
"tools.web.search.searxng.params": "SearXNG Default Params",
"tools.web.fetch.enabled": "Enable Web Fetch Tool",
"tools.web.fetch.maxChars": "Web Fetch Max Chars",
"tools.web.fetch.timeoutSeconds": "Web Fetch Timeout (sec)",
@@ -434,8 +438,9 @@ const FIELD_HELP: Record<string, string> = {
"tools.message.crossContext.marker.suffix":
'Text suffix for cross-context markers (supports "{channel}").',
"tools.message.broadcast.enabled": "Enable broadcast action (default: true).",
"tools.web.search.enabled": "Enable the web_search tool (requires a provider API key).",
"tools.web.search.provider": 'Search provider ("brave" or "perplexity").',
"tools.web.search.enabled":
"Enable the web_search tool (requires provider configuration: API key or base URL).",
"tools.web.search.provider": 'Search provider ("brave", "perplexity", or "searxng").',
"tools.web.search.apiKey": "Brave Search API key (fallback: BRAVE_API_KEY env var).",
"tools.web.search.maxResults": "Default number of results to return (1-10).",
"tools.web.search.timeoutSeconds": "Timeout in seconds for web_search requests.",
@@ -446,6 +451,14 @@ const FIELD_HELP: Record<string, string> = {
"Perplexity base URL override (default: https://openrouter.ai/api/v1 or https://api.perplexity.ai).",
"tools.web.search.perplexity.model":
'Perplexity model override (default: "perplexity/sonar-pro").',
"tools.web.search.searxng.baseUrl":
"SearXNG instance base URL (e.g. http://localhost:8080). Fallback: SEARXNG_BASE_URL env var.",
"tools.web.search.searxng.apiKey":
"Optional SearXNG auth token; sent as Authorization header (Bearer by default).",
"tools.web.search.searxng.headers":
"Optional extra headers for SearXNG requests (e.g. for reverse-proxy auth).",
"tools.web.search.searxng.params":
"Optional default query params for SearXNG /search (merged into each request).",
"tools.web.fetch.enabled": "Enable the web_fetch tool (lightweight HTTP fetch).",
"tools.web.fetch.maxChars": "Max characters returned by web_fetch (truncated).",
"tools.web.fetch.timeoutSeconds": "Timeout in seconds for web_fetch requests.",
+12 -1
View File
@@ -165,7 +165,9 @@ export const ToolPolicySchema = ToolPolicyBaseSchema.superRefine((value, ctx) =>
export const ToolsWebSearchSchema = z
.object({
enabled: z.boolean().optional(),
provider: z.union([z.literal("brave"), z.literal("perplexity")]).optional(),
provider: z
.union([z.literal("brave"), z.literal("perplexity"), z.literal("searxng")])
.optional(),
apiKey: z.string().optional(),
maxResults: z.number().int().positive().optional(),
timeoutSeconds: z.number().int().positive().optional(),
@@ -178,6 +180,15 @@ export const ToolsWebSearchSchema = z
})
.strict()
.optional(),
searxng: z
.object({
baseUrl: z.string().optional(),
apiKey: z.string().optional(),
headers: z.record(z.string(), z.string()).optional(),
params: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(),
})
.strict()
.optional(),
})
.strict()
.optional();
+83 -11
View File
@@ -433,29 +433,101 @@ export async function finalizeOnboardingWizard(options: FinalizeOnboardingOption
);
}
const webSearchKey = (nextConfig.tools?.web?.search?.apiKey ?? "").trim();
const webSearchEnv = (process.env.BRAVE_API_KEY ?? "").trim();
const hasWebSearchKey = Boolean(webSearchKey || webSearchEnv);
const webSearch = nextConfig.tools?.web?.search;
const webSearchProviderRaw =
webSearch &&
typeof webSearch === "object" &&
"provider" in webSearch &&
typeof webSearch.provider === "string"
? webSearch.provider.trim().toLowerCase()
: "";
const webSearchProvider =
webSearchProviderRaw === "perplexity"
? "perplexity"
: webSearchProviderRaw === "searxng"
? "searxng"
: "brave";
const readTrimmedString = (value: unknown): string =>
typeof value === "string" ? value.trim() : "";
const braveKey =
webSearch && typeof webSearch === "object" && "apiKey" in webSearch
? readTrimmedString(webSearch.apiKey)
: "";
const braveEnv = (process.env.BRAVE_API_KEY ?? "").trim();
const hasBraveKey = Boolean(braveKey || braveEnv);
const perplexityKey =
webSearch &&
typeof webSearch === "object" &&
"perplexity" in webSearch &&
webSearch.perplexity &&
typeof webSearch.perplexity === "object" &&
"apiKey" in webSearch.perplexity
? readTrimmedString((webSearch.perplexity as Record<string, unknown>).apiKey)
: "";
const perplexityEnv = (process.env.PERPLEXITY_API_KEY ?? "").trim();
const openrouterEnv = (process.env.OPENROUTER_API_KEY ?? "").trim();
const hasPerplexityKey = Boolean(perplexityKey || perplexityEnv || openrouterEnv);
const searxngBaseUrl =
webSearch &&
typeof webSearch === "object" &&
"searxng" in webSearch &&
webSearch.searxng &&
typeof webSearch.searxng === "object" &&
"baseUrl" in webSearch.searxng
? readTrimmedString((webSearch.searxng as Record<string, unknown>).baseUrl)
: "";
const searxngEnv = (process.env.SEARXNG_BASE_URL ?? "").trim();
const hasSearxngBaseUrl = Boolean(searxngBaseUrl || searxngEnv);
const hasWebSearchSetup =
webSearchProvider === "perplexity"
? hasPerplexityKey
: webSearchProvider === "searxng"
? hasSearxngBaseUrl
: hasBraveKey;
await prompter.note(
hasWebSearchKey
hasWebSearchSetup
? [
"Web search is enabled, so your agent can look things up online when needed.",
"",
webSearchKey
? "API key: stored in config (tools.web.search.apiKey)."
: "API key: provided via BRAVE_API_KEY env var (Gateway environment).",
`Provider: ${webSearchProvider}`,
webSearchProvider === "searxng"
? searxngBaseUrl
? "Base URL: stored in config (tools.web.search.searxng.baseUrl)."
: "Base URL: provided via SEARXNG_BASE_URL env var (Gateway environment)."
: webSearchProvider === "perplexity"
? perplexityKey
? "API key: stored in config (tools.web.search.perplexity.apiKey)."
: perplexityEnv
? "API key: provided via PERPLEXITY_API_KEY env var (Gateway environment)."
: "API key: provided via OPENROUTER_API_KEY env var (Gateway environment)."
: braveKey
? "API key: stored in config (tools.web.search.apiKey)."
: "API key: provided via BRAVE_API_KEY env var (Gateway environment).",
"Docs: https://docs.openclaw.ai/tools/web",
].join("\n")
: [
"If you want your agent to be able to search the web, you’ll need an API key.",
"If you want your agent to be able to search the web, you’ll need to configure a provider.",
"",
"OpenClaw uses Brave Search for the `web_search` tool. Without a Brave Search API key, web search won’t work.",
`OpenClaw uses ${webSearchProvider} for the \`web_search\` tool.`,
"",
"Set it up interactively:",
`- Run: ${formatCliCommand("openclaw configure --section web")}`,
"- Enable web_search and paste your Brave Search API key",
webSearchProvider === "searxng"
? "- Enable web_search and set your SearXNG base URL"
: webSearchProvider === "perplexity"
? "- Enable web_search and set your Perplexity/OpenRouter API key"
: "- Enable web_search and paste your Brave Search API key",
"",
"Alternative: set BRAVE_API_KEY in the Gateway environment (no config changes).",
webSearchProvider === "searxng"
? "Alternative: set SEARXNG_BASE_URL in the Gateway environment (no config changes)."
: webSearchProvider === "perplexity"
? "Alternative: set PERPLEXITY_API_KEY or OPENROUTER_API_KEY in the Gateway environment (no config changes)."
: "Alternative: set BRAVE_API_KEY in the Gateway environment (no config changes).",
"Docs: https://docs.openclaw.ai/tools/web",
].join("\n"),
"Web search (optional)",