From e5cd85fff3fd0474c474242972bb23acac72457e Mon Sep 17 00:00:00 2001 From: jules Date: Fri, 1 May 2026 20:16:41 +1000 Subject: [PATCH] Add 5 more admin tools + inline write confirmation flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New tools in admin-tools.ts: - list_audit_log({limit?}) — recent audit entries (terse: actor, action, target, timestamp). Hits /api/v1/admin/audit-log. - get_platform_stats() — aggregate counts (tenants by status + by plan), composed locally from list_tenants until arcadia exposes a real stats endpoint. - list_users({limit?}) — users in the currently-selected tenant via /api/v1/users. - suspend_tenant({slug}) — write tool, suspends a tenant by slug. - activate_tenant({slug}) — write tool, restores a suspended/deactivated tenant. Inline write confirmation: - New ConfirmCard component renders below the assistant message that proposed a write. Shows tool(args) and Confirm/Deny buttons. - classifyCalls() splits LLM tool calls into reads/writes. Auto-loop runs reads immediately; for any writes, holds them in pendingConfirm state instead of dispatching. - On Confirm: runs writes with allowWrites:true, prepends prior read results, continueChat to produce the final answer. - On Deny: synthesises tool-result messages telling the model the user declined; continueChat so it can acknowledge. - Arcadia-knowledge primer updated to tell the model the user sees an inline confirm card automatically — it shouldn't ask in prose first. Wired into both /ai and /assistant. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/components/assistant/confirm-card.tsx | 52 ++++++ app/lib/admin-tools.ts | 207 +++++++++++++++++++++- app/lib/arcadia-knowledge.ts | 2 +- app/routes/ai.tsx | 82 +++++++-- app/routes/assistant.tsx | 76 +++++++- 5 files changed, 398 insertions(+), 21 deletions(-) create mode 100644 app/components/assistant/confirm-card.tsx diff --git a/app/components/assistant/confirm-card.tsx b/app/components/assistant/confirm-card.tsx new file mode 100644 index 0000000..bfe25a2 --- /dev/null +++ b/app/components/assistant/confirm-card.tsx @@ -0,0 +1,52 @@ +// Inline transcript card asking the operator to confirm or deny one or more +// write tool calls the model has proposed. Renders below the assistant +// message that emitted them; the auto-loop is paused until the user picks. + +import { ShieldAlert } from "lucide-react" + +import type { ToolCall } from "@crema/llm-ui" +import { formatToolCallArgs } from "~/lib/admin-tools" + +import { Button } from "~/components/ui/button" + +export function ConfirmCard({ + calls, + onConfirm, + onDeny, + busy, +}: { + calls: ToolCall[] + onConfirm: () => void + onDeny: () => void + busy?: boolean +}) { + return ( +
+
+ + Confirm write{calls.length > 1 ? "s" : ""} +
+
    + {calls.map((c) => ( +
  • + {c.name}({formatToolCallArgs(c)}) +
    • + ))} +
+
+ + +
+
+ ) +} diff --git a/app/lib/admin-tools.ts b/app/lib/admin-tools.ts index d19cec3..c6c63ed 100644 --- a/app/lib/admin-tools.ts +++ b/app/lib/admin-tools.ts @@ -8,7 +8,13 @@ import type { ArcadiaClient } from "@crema/arcadia-client" import type { Tool, ToolCall as LLMToolCall } from "@crema/llm-ui" -import { getTenant, listTenants, type Tenant } from "~/lib/arcadia/tenants" +import { + activateTenant, + getTenant, + listTenants, + suspendTenant, + type Tenant, +} from "~/lib/arcadia/tenants" export type ToolCall = { name: string @@ -78,8 +84,164 @@ const TOOLS: ToolDef[] = [ return found ? summarize(found) : null }, }, + { + name: "get_platform_stats", + description: + "Aggregate platform stats. Returns total tenant count and a breakdown by status (active / suspended / deactivated / etc). Call this for big-picture questions like 'how is the platform doing'.", + parameters: { + type: "object", + properties: {}, + additionalProperties: false, + }, + isWrite: false, + run: async (_args, { arcadia }) => { + const tenants = await listTenants(arcadia) + const byStatus: Record = {} + for (const t of tenants) { + byStatus[t.status] = (byStatus[t.status] ?? 0) + 1 + } + const byPlan: Record = {} + for (const t of tenants) { + const plan = t.plan?.name ?? "(no plan)" + byPlan[plan] = (byPlan[plan] ?? 0) + 1 + } + return { + tenants_total: tenants.length, + tenants_by_status: byStatus, + tenants_by_plan: byPlan, + } + }, + }, + { + name: "list_audit_log", + description: + "Recent audit log entries from the platform. Returns a terse summary per entry: actor_type, actor_id, action, target, inserted_at. Call this for 'who did what' questions or to investigate a recent change.", + parameters: { + type: "object", + properties: { + limit: { + type: "integer", + description: "Max entries to return (default 25, max 100).", + minimum: 1, + maximum: 100, + }, + }, + additionalProperties: false, + }, + isWrite: false, + run: async (args, { arcadia }) => { + const limit = typeof args.limit === "number" ? Math.min(100, Math.max(1, args.limit)) : 25 + const res = await arcadia.GET<{ data: AuditEntry[] }>( + `/api/v1/admin/audit-log?limit=${limit}`, + ) + const entries = res.data ?? [] + return entries.map((e) => ({ + actor_type: e.actor_type, + actor_id: e.actor_id ?? null, + action: e.action, + target: e.target_type ? `${e.target_type}#${e.target_id ?? "?"}` : null, + inserted_at: e.inserted_at, + })) + }, + }, + { + name: "list_users", + description: + "List users in the currently-selected tenant context. Returns email, name, roles, status. Note: this is scoped to whichever tenant the assistant is currently logged into; use get_tenant first if you need to confirm which tenant.", + parameters: { + type: "object", + properties: { + limit: { + type: "integer", + description: "Max users to return (default 50, max 200).", + minimum: 1, + maximum: 200, + }, + }, + additionalProperties: false, + }, + isWrite: false, + run: async (args, { arcadia }) => { + const limit = typeof args.limit === "number" ? Math.min(200, Math.max(1, args.limit)) : 50 + const res = await arcadia.GET<{ data: UserEntry[] }>(`/api/v1/users?per_page=${limit}`) + const users = res.data ?? [] + return users.map((u) => ({ + id: u.id, + email: u.email, + name: [u.first_name, u.last_name].filter(Boolean).join(" ") || null, + roles: u.roles?.map((r) => r.slug ?? r.name).filter(Boolean) ?? [], + verified: u.email_verified ?? null, + inserted_at: u.inserted_at, + })) + }, + }, + { + name: "suspend_tenant", + description: + "Suspend a tenant by slug. Members can no longer sign in until reactivated. Use for temporary holds (overdue invoice, abuse investigation). REVERSIBLE via activate_tenant. Requires user confirmation before executing.", + parameters: { + type: "object", + properties: { + slug: { type: "string", description: "The tenant's slug." }, + }, + required: ["slug"], + additionalProperties: false, + }, + isWrite: true, + run: async (args, { arcadia }) => { + const slug = typeof args.slug === "string" ? args.slug : null + if (!slug) throw new Error("suspend_tenant requires { slug }") + const tenants = await listTenants(arcadia) + const target = tenants.find((t) => t.slug === slug) + if (!target) throw new Error(`No tenant with slug "${slug}"`) + const updated = await suspendTenant(arcadia, target.id) + return summarize(updated) + }, + }, + { + name: "activate_tenant", + description: + "Re-activate a previously suspended (or deactivated) tenant by slug. Restores member sign-in. Requires user confirmation before executing.", + parameters: { + type: "object", + properties: { + slug: { type: "string", description: "The tenant's slug." }, + }, + required: ["slug"], + additionalProperties: false, + }, + isWrite: true, + run: async (args, { arcadia }) => { + const slug = typeof args.slug === "string" ? args.slug : null + if (!slug) throw new Error("activate_tenant requires { slug }") + const tenants = await listTenants(arcadia) + const target = tenants.find((t) => t.slug === slug) + if (!target) throw new Error(`No tenant with slug "${slug}"`) + const updated = await activateTenant(arcadia, target.id) + return summarize(updated) + }, + }, ] +interface AuditEntry { + actor_type: string + actor_id?: string + action: string + target_type?: string + target_id?: string + inserted_at: string +} + +interface UserEntry { + id: string + email: string + first_name?: string + last_name?: string + email_verified?: boolean + inserted_at: string + roles?: { slug?: string; name?: string }[] +} + /** OpenAI-format tool list to pass into ChatRequest.tools. */ export function getOpenAITools(): Tool[] { return TOOLS.map((t) => ({ @@ -89,6 +251,49 @@ export function getOpenAITools(): Tool[] { })) } +/** Split an LLM tool-call list into reads (run automatically) and writes + * (held for user confirmation). Unknown tools fall into reads so the runner + * can surface a structured "unknown tool" error to the model. */ +export function classifyCalls(calls: LLMToolCall[]): { + reads: LLMToolCall[] + writes: LLMToolCall[] +} { + const reads: LLMToolCall[] = [] + const writes: LLMToolCall[] = [] + for (const c of calls) { + const def = TOOL_BY_NAME.get(c.name) + if (def?.isWrite) writes.push(c) + else reads.push(c) + } + return { reads, writes } +} + +/** Synthesise tool-result messages saying the user denied a write call. */ +export function buildDenialMessages( + calls: LLMToolCall[], +): { role: "tool"; content: string; toolCallId: string; name: string }[] { + return calls.map((c) => ({ + role: "tool", + content: JSON.stringify({ + error: "User denied this write. Do not retry without re-asking the user.", + }), + toolCallId: c.id, + name: c.name, + })) +} + +/** Pretty-print args for the confirm UI. */ +export function formatToolCallArgs(c: LLMToolCall): string { + try { + const parsed = c.arguments ? JSON.parse(c.arguments) : {} + const keys = Object.keys(parsed) + if (keys.length === 0) return "" + return keys.map((k) => `${k}=${JSON.stringify(parsed[k])}`).join(", ") + } catch { + return c.arguments + } +} + function summarize(t: Tenant) { return { id: t.id, diff --git a/app/lib/arcadia-knowledge.ts b/app/lib/arcadia-knowledge.ts index 59abf23..d2bad56 100644 --- a/app/lib/arcadia-knowledge.ts +++ b/app/lib/arcadia-knowledge.ts @@ -32,4 +32,4 @@ Things to keep in mind when assisting: - Quotas / rate cards / billing config errors usually surface as 402/403 from /api/v1 endpoints — diagnose by checking the tenant's billing-config and api-metering quotas. - The reference Phoenix app lives at \`reference/arcadia-app/\` in the workspace; its OpenAPI spec is at /api/openapi (sync via \`node ../lib-arcadia-client/scripts/sync-spec.mjs\`). -When the user asks something that maps to a tool, call it. When they ask about a concept, explain it from this primer in plain language. When they ask to do something destructive, summarise the impact in one sentence and ask for confirmation before suggesting a tool call.` +When the user asks something that maps to a tool, call it. When they ask about a concept, explain it from this primer in plain language. Write tools (suspend_tenant, activate_tenant) prompt the operator with an inline confirm card before they actually run — you do not need to ask in prose first; just call the tool and the user will see the confirmation UI. If the user denies a write, do not retry it; ask what they'd like to do differently.` diff --git a/app/routes/ai.tsx b/app/routes/ai.tsx index 4a2c240..91865a6 100644 --- a/app/routes/ai.tsx +++ b/app/routes/ai.tsx @@ -63,9 +63,15 @@ import { Avatar, AvatarFallback } from "~/components/ui/avatar" import { pageTitle } from "~/lib/page-meta" import { useArcadiaClient } from "@crema/arcadia-client" import type { ToolCall } from "@crema/llm-ui" -import { getOpenAITools, runLLMToolCalls } from "~/lib/admin-tools" +import { + buildDenialMessages, + classifyCalls, + getOpenAITools, + runLLMToolCalls, +} from "~/lib/admin-tools" import { ARCADIA_KNOWLEDGE } from "~/lib/arcadia-knowledge" import { formatAdminContextForPrompt } from "~/lib/admin-context" +import { ConfirmCard } from "~/components/assistant/confirm-card" const SNAPSHOT_KEY = "crema.ai.snapshot" type StoredMessage = { role: "user" | "assistant"; content: string } @@ -241,11 +247,21 @@ function ChatSurface({ system: systemPrompt, }) - // Auto tool-loop using native function calls. + // Auto tool-loop using native function calls. Reads run automatically; + // writes are held in `pendingConfirm` until the operator clicks Confirm + // or Deny in the inline ConfirmCard. const toolIterationsRef = useRef(0) const processedTurnRef = useRef(-1) const prevStreamingRef = useRef(isStreaming) const MAX_TOOL_ITERATIONS = 3 + const [pendingConfirm, setPendingConfirm] = useState<{ + /** Message index that emitted the write calls. */ + afterIndex: number + writes: ToolCall[] + readMessages: { role: "tool"; content: string; toolCallId: string; name: string }[] + } | null>(null) + const [confirmBusy, setConfirmBusy] = useState(false) + useEffect(() => { const justFinished = prevStreamingRef.current && !isStreaming prevStreamingRef.current = isStreaming @@ -264,13 +280,50 @@ function ChatSurface({ if (toolIterationsRef.current >= MAX_TOOL_ITERATIONS) return toolIterationsRef.current += 1 void (async () => { - const { toolMessages } = await runLLMToolCalls(calls, { arcadia }) - void continueChat(toolMessages, { + const { reads, writes } = classifyCalls(calls) + const { toolMessages: readMsgs } = + reads.length > 0 + ? await runLLMToolCalls(reads, { arcadia }) + : { toolMessages: [] } + if (writes.length > 0) { + setPendingConfirm({ afterIndex: lastIdx, writes, readMessages: readMsgs }) + return + } + void continueChat(readMsgs, { system: systemPrompt, tools: getOpenAITools(), }) })() }, [messages, isStreaming, arcadia, continueChat, systemPrompt]) + + const onConfirmWrites = useCallback(async () => { + if (!pendingConfirm) return + setConfirmBusy(true) + try { + const { toolMessages: writeMsgs } = await runLLMToolCalls( + pendingConfirm.writes, + { arcadia }, + { allowWrites: true }, + ) + void continueChat([...pendingConfirm.readMessages, ...writeMsgs], { + system: systemPrompt, + tools: getOpenAITools(), + }) + } finally { + setPendingConfirm(null) + setConfirmBusy(false) + } + }, [pendingConfirm, arcadia, continueChat, systemPrompt]) + + const onDenyWrites = useCallback(() => { + if (!pendingConfirm) return + const denials = buildDenialMessages(pendingConfirm.writes) + void continueChat([...pendingConfirm.readMessages, ...denials], { + system: systemPrompt, + tools: getOpenAITools(), + }) + setPendingConfirm(null) + }, [pendingConfirm, continueChat, systemPrompt]) const { complete: completeOneShot, isLoading: compacting } = useCompletion() const [input, setInput] = useState("") const [showPromptOpen, setShowPromptOpen] = useState(false) @@ -473,12 +526,21 @@ function ChatSurface({ {messages .filter((m) => m.role !== "system") .map((m, i) => ( - +
+ + {pendingConfirm?.afterIndex === i && ( + + )} +
))} {isStreaming && messages.at(-1)?.role !== "assistant" && (
diff --git a/app/routes/assistant.tsx b/app/routes/assistant.tsx index 606562d..df584bc 100644 --- a/app/routes/assistant.tsx +++ b/app/routes/assistant.tsx @@ -81,9 +81,16 @@ wait_for assistant-ui-control import { formatAdminContextForPrompt } from "~/lib/admin-context" -import { getOpenAITools, runLLMToolCalls } from "~/lib/admin-tools" +import { + buildDenialMessages, + classifyCalls, + getOpenAITools, + runLLMToolCalls, +} from "~/lib/admin-tools" import { ARCADIA_KNOWLEDGE } from "~/lib/arcadia-knowledge" import { useArcadiaClient } from "@crema/arcadia-client" +import { ConfirmCard } from "~/components/assistant/confirm-card" +import type { ToolCall } from "@crema/llm-ui" /** * Always includes domain knowledge + tools + admin context + persona. @@ -579,9 +586,16 @@ function AssistantSurface({ const prevStreamingRef = useRef(isStreaming) const MAX_TOOL_ITERATIONS = 3 - // Auto tool-loop using native function calls. When the streaming assistant - // turn ends with toolCalls attached, run them, push tool result messages, - // and continue the chat so the model produces its final answer. + // Auto tool-loop using native function calls. Reads run automatically; + // writes are held in `pendingConfirm` until the operator clicks Confirm + // or Deny in the inline ConfirmCard. + const [pendingConfirm, setPendingConfirm] = useState<{ + afterIndex: number + writes: ToolCall[] + readMessages: { role: "tool"; content: string; toolCallId: string; name: string }[] + } | null>(null) + const [confirmBusy, setConfirmBusy] = useState(false) + useEffect(() => { const justFinished = prevStreamingRef.current && !isStreaming prevStreamingRef.current = isStreaming @@ -595,7 +609,6 @@ function AssistantSurface({ processedTurnRef.current = lastIdx const calls = last.toolCalls ?? [] - console.debug("[admin-tools] assistant turn finished, tool_calls:", calls) if (calls.length === 0) { toolIterationsRef.current = 0 return @@ -607,16 +620,53 @@ function AssistantSurface({ toolIterationsRef.current += 1 void (async () => { - console.debug("[admin-tools] running tool calls", calls) - const { results, toolMessages } = await runLLMToolCalls(calls, { arcadia }) - console.debug("[admin-tools] tool results", results) - void continueChat(toolMessages, { + const { reads, writes } = classifyCalls(calls) + const { toolMessages: readMsgs } = + reads.length > 0 + ? await runLLMToolCalls(reads, { arcadia }) + : { toolMessages: [] } + if (writes.length > 0) { + setPendingConfirm({ afterIndex: lastIdx, writes, readMessages: readMsgs }) + return + } + void continueChat(readMsgs, { system: constructorSystemPrompt, tools: getOpenAITools(), maxTokens: responseBudget, }) })() }, [messages, isStreaming, arcadia, continueChat, constructorSystemPrompt, responseBudget]) + + const onConfirmWrites = useCallback(async () => { + if (!pendingConfirm) return + setConfirmBusy(true) + try { + const { toolMessages: writeMsgs } = await runLLMToolCalls( + pendingConfirm.writes, + { arcadia }, + { allowWrites: true }, + ) + void continueChat([...pendingConfirm.readMessages, ...writeMsgs], { + system: constructorSystemPrompt, + tools: getOpenAITools(), + maxTokens: responseBudget, + }) + } finally { + setPendingConfirm(null) + setConfirmBusy(false) + } + }, [pendingConfirm, arcadia, continueChat, constructorSystemPrompt, responseBudget]) + + const onDenyWrites = useCallback(() => { + if (!pendingConfirm) return + const denials = buildDenialMessages(pendingConfirm.writes) + void continueChat([...pendingConfirm.readMessages, ...denials], { + system: constructorSystemPrompt, + tools: getOpenAITools(), + maxTokens: responseBudget, + }) + setPendingConfirm(null) + }, [pendingConfirm, continueChat, constructorSystemPrompt, responseBudget]) const [editingIndex, setEditingIndex] = useState(null) const [editingDraft, setEditingDraft] = useState("") const [speakingIndex, setSpeakingIndex] = useState(null) @@ -1109,6 +1159,14 @@ function AssistantSurface({
)} + {pendingConfirm?.afterIndex === i && ( + + )} ) })}