From f5189305c7a5a9fe6cf6b428192b85b02b3643b8 Mon Sep 17 00:00:00 2001 From: jules Date: Sun, 3 May 2026 21:41:13 +1000 Subject: [PATCH] ai: wire arcadia-search backend (search_kb + read_chunk + reindex button) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the agent-facing surface for the new Tantivy lexical search service (arcadia-search). Sits alongside the existing search_docs (browser MiniSearch) — agent picks based on tool description. - admin-tools.ts: new search_kb(query, corpus, limit?, tags?) and read_chunk(chunk_id, corpus) tools. KB_BASE_URL honors window.__ARCADIA_SEARCH_URL runtime override + VITE_ARCADIA_SEARCH_URL build env, defaults to localhost:7800. Token resolved per-call from sessionStorage.arcadia_access_token (matching lib-arcadia-client's storage convention) with "dev" fallback for unauthenticated dev. - assistant.tsx: system-prompt section telling the agent when to pick search_docs (browser, bundled) vs search_kb (server, dynamic + expandable via read_chunk). - ai.tsx: reindexKB() helper + "reindex kb (docs)" button on the empty state, next to the existing block-preview button. Toasts on start/success/failure. Wired with data-action="kb-reindex-docs" so the agent can also trigger via the command bus. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/lib/admin-tools.ts | 138 +++++++++++++++++++++++++++++++++++++++ app/routes/ai.tsx | 52 ++++++++++++++- app/routes/assistant.tsx | 1 + 3 files changed, 190 insertions(+), 1 deletion(-) diff --git a/app/lib/admin-tools.ts b/app/lib/admin-tools.ts index 6260fd2..03e75d6 100644 --- a/app/lib/admin-tools.ts +++ b/app/lib/admin-tools.ts @@ -22,6 +22,77 @@ import { BLOCK_INDEX, getBlockSchema } from "~/lib/block-schemas" // calls reuse the parsed MiniSearch instance. const docsClient = createRAGClient("/docs-index.json") +// Server-side Tantivy backend (arcadia-search). +// +// URL: comes from window.__ARCADIA_SEARCH_URL (override hook) or +// VITE_ARCADIA_SEARCH_URL build-time env, defaulting to localhost. +// +// Token: prefer the real arcadia access token (sessionStorage — +// matches lib-arcadia-client's storage convention). Fall back to "dev" +// when missing, which only works against AUTH_MODE=dev backends. In +// production, arcadia-search runs in JWT mode and the dev fallback +// gets rejected with 401 — surfacing the missing-login as a clear +// error rather than silently using the wrong identity. +const KB_BASE_URL: string = + (typeof window !== "undefined" && + (window as unknown as { __ARCADIA_SEARCH_URL?: string }).__ARCADIA_SEARCH_URL) || + (typeof import.meta !== "undefined" && + (import.meta as unknown as { env?: { VITE_ARCADIA_SEARCH_URL?: string } }).env + ?.VITE_ARCADIA_SEARCH_URL) || + "http://127.0.0.1:7800" + +function kbAuthToken(): string { + if (typeof window === "undefined") return "dev" + try { + return window.sessionStorage.getItem("arcadia_access_token") ?? "dev" + } catch { + return "dev" + } +} + +type KBHit = { + chunk_id: string + title: string + source_path: string + heading_path: string + tags: string[] + snippet: string + score: number + mtime: string +} + +async function kbSearch( + query: string, + corpus: string, + limit: number, + tags?: string[], +): Promise<{ count: number; hits: KBHit[] }> { + const res = await fetch(`${KB_BASE_URL}/search`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${kbAuthToken()}`, + }, + body: JSON.stringify({ query, corpus, limit, tags }), + }) + if (!res.ok) { + throw new Error(`arcadia-search ${res.status}: ${await res.text()}`) + } + return (await res.json()) as { count: number; hits: KBHit[] } +} + +async function kbRead(chunkId: string, corpus: string): Promise { + const url = `${KB_BASE_URL}/chunks/${encodeURIComponent(chunkId)}?corpus=${encodeURIComponent(corpus)}` + const res = await fetch(url, { + headers: { Authorization: `Bearer ${kbAuthToken()}` }, + }) + if (res.status === 404) return null + if (!res.ok) { + throw new Error(`arcadia-search ${res.status}: ${await res.text()}`) + } + return await res.json() +} + export type ToolCall = { name: string args: Record @@ -275,6 +346,73 @@ const TOOLS: ToolDef[] = [ } }, }, + { + name: "search_kb", + description: + "Lexical (BM25) search over the arcadia-search Tantivy backend. Use for the LARGER, server-hosted knowledge corpora — the same arcadia docs the browser RAG serves are indexed here as `corpus=docs` for parity, and additional corpora (uploaded files, runbooks, etc.) will land here as they're added. Returns chunks with snippets + chunk_ids that can be passed to `read_chunk` to expand. Prefer this over `search_docs` (browser) when you need richer hits or when the user is asking about content that wouldn't be in the bundled docs (e.g. uploaded files).", + parameters: { + type: "object", + properties: { + query: { type: "string", description: "Lexical search query." }, + corpus: { + type: "string", + description: + "Which indexed corpus to search. `docs` is the parity corpus (arcadia documentation). New corpora are added by the operator.", + }, + limit: { + type: "integer", + description: "Max hits. Default 5, cap 20.", + minimum: 1, + maximum: 20, + }, + tags: { + type: "array", + items: { type: "string" }, + description: + "Optional tag filter — return only hits whose chunk has at least one matching tag.", + }, + }, + required: ["query", "corpus"], + additionalProperties: false, + }, + isWrite: false, + run: async (args) => { + const query = typeof args.query === "string" ? args.query.trim() : "" + const corpus = typeof args.corpus === "string" ? args.corpus.trim() : "" + if (!query) throw new Error("search_kb requires a non-empty { query }") + if (!corpus) throw new Error("search_kb requires a { corpus } name") + const limit = Math.min(20, Math.max(1, typeof args.limit === "number" ? args.limit : 5)) + const tags = Array.isArray(args.tags) ? (args.tags as string[]) : undefined + return await kbSearch(query, corpus, limit, tags) + }, + }, + { + name: "read_chunk", + description: + "Fetch the full body of one chunk by id from the arcadia-search backend, after `search_kb` returned it as a snippet. Use this to expand a hit when the snippet looked promising but you need more context to answer.", + parameters: { + type: "object", + properties: { + chunk_id: { type: "string", description: "The chunk_id from a prior search_kb hit." }, + corpus: { type: "string", description: "Same corpus the chunk came from." }, + }, + required: ["chunk_id", "corpus"], + additionalProperties: false, + }, + isWrite: false, + run: async (args) => { + const chunkId = typeof args.chunk_id === "string" ? args.chunk_id : "" + const corpus = typeof args.corpus === "string" ? args.corpus : "" + if (!chunkId || !corpus) { + throw new Error("read_chunk requires { chunk_id, corpus }") + } + const result = await kbRead(chunkId, corpus) + if (result === null) { + return { error: "chunk not found", chunk_id: chunkId, corpus } + } + return result + }, + }, { name: "get_block_schema", description: `Fetch the full JSON schema + example for a rich-output block kind so you can emit it correctly in your reply. Call this the first time in a thread that you intend to render a particular kind. Available kinds: ${Object.entries( diff --git a/app/routes/ai.tsx b/app/routes/ai.tsx index ca08361..707ceeb 100644 --- a/app/routes/ai.tsx +++ b/app/routes/ai.tsx @@ -111,6 +111,48 @@ function ToolResultBlock({ name, result }: { name: string; result: unknown }) { return
{rich}
} +// Trigger a server-side rebuild of an arcadia-search corpus. Reads the +// same KB URL + token resolution as the search_kb tool (see admin-tools.ts). +// Surfaces success/failure via the existing toast provider. +async function reindexKB( + corpus: string, + toast: ReturnType, +): Promise { + const baseUrl = + (typeof window !== "undefined" && + (window as unknown as { __ARCADIA_SEARCH_URL?: string }).__ARCADIA_SEARCH_URL) || + "http://127.0.0.1:7800" + const token = + (typeof window !== "undefined" && + window.sessionStorage.getItem("arcadia_access_token")) || + "dev" + const url = `${baseUrl}/index/${encodeURIComponent(corpus)}/build` + toast.show?.({ + title: "Reindexing…", + description: `Rebuilding corpus '${corpus}'.`, + }) + try { + const res = await fetch(url, { + method: "POST", + headers: { Authorization: `Bearer ${token}` }, + }) + if (!res.ok) { + throw new Error(`HTTP ${res.status}: ${await res.text()}`) + } + const out = (await res.json()) as { chunk_count: number; built_at: string } + toast.show?.({ + title: "Reindex complete", + description: `${out.chunk_count} chunks indexed for '${corpus}'.`, + }) + } catch (err) { + toast.show?.({ + title: "Reindex failed", + description: err instanceof Error ? err.message : String(err), + tone: "error", + }) + } +} + // Synthetic assistant message that exercises every typed rich-output block. // Wired to the "preview rich-output blocks" button in the empty state — used // to eyeball renderer + theme without driving a live model. Safe to delete @@ -1179,7 +1221,7 @@ function ChatSurface({ Issue an instruction. Read tools run automatically. Writes pause for confirmation. Tab ⇥ for command palette.

-
+
+
diff --git a/app/routes/assistant.tsx b/app/routes/assistant.tsx index 3e59e52..2936357 100644 --- a/app/routes/assistant.tsx +++ b/app/routes/assistant.tsx @@ -113,6 +113,7 @@ function buildAdminPreface(activeAgent: Agent | undefined, uiControl: boolean): const ctx = formatAdminContextForPrompt() const parts = [ "You are the operator's assistant inside Arcadia Admin. Be precise and direct. You have native function tools attached to this conversation — call them whenever the user asks about live platform state (counts, statuses, listings, lookups). Never invent tenant slugs, user counts, or statuses; if you need data, call a tool.", + "Two retrieval surfaces exist for documentation/knowledge: `search_docs` (browser-side, BM25 over the bundled arcadia docs — fast, always available, small corpus) and `search_kb` (server-side, BM25 over arcadia-search — same docs as `corpus=docs` for parity, plus larger and additional corpora as the operator adds them). For questions about the bundled arcadia docs either is fine; prefer `search_kb` when you want richer hits or when the user is asking about content that wouldn't be in the bundled docs (uploaded files, tenant-specific knowledge). When `search_kb` returns a chunk_id you want to expand, call `read_chunk(chunk_id, corpus)`.", RICH_OUTPUT_PREFACE, ARCADIA_KNOWLEDGE, persona,