feat: initial commit — extracted from comfy-cloud

Anything-can-drive-the-UI command bus. Single dispatch point for LLM tool
calls, scripts, and (optional) WebSocket remote control. JSON canonical
form plus a plain-text DSL sugar layer. Includes a visible virtual cursor
that animates to elements before commands act on them.

Modules:
- bus.ts        — CommandBus class, dispatch, handlers, vars, history,
                  listActions, readState. Built-in handlers: navigate, click,
                  fill, submit, select, wait, wait_for, scroll, read, expect,
                  set
- parser.ts     — DSL ↔ JSON, with quoted values, # comments, # speed:
                  directive, $name = <cmd> assignment, $var interpolation,
                  expect assertions
- script.ts     — script runner; loads /scripts/<name>.script, supports
                  sub-scripts via `run` command
- cursor.ts     — virtual cursor + ripple, speed-controlled, aria-hidden
- provider.tsx  — React glue: registers `navigate` via react-router, mounts
                  cursor, exposes window.commandBus / runScript /
                  runScriptText for console use
- ws.ts         — optional WebSocket producer (connectCommandSocket)
- llm-bridge.ts — buildSystemPrompt (DSL ref + live action snapshot),
                  extractActionBlocks, runActionBlocks, estimateTokens,
                  trimMessages
- index.tsx     — barrel re-exports

Peer deps: react, react-router (consuming app provides).

Originally inline in `comfy-cloud/app/lib/`. Extracted as part of the
crema-app-aifirst-template scaffolding.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jules
2026-04-27 18:32:22 +10:00
commit f28038abec
9 changed files with 1178 additions and 0 deletions

102
src/llm-bridge.ts Normal file
View File

@@ -0,0 +1,102 @@
// LLM ↔ command bus glue.
// - buildSystemPrompt() teaches the model the DSL and lists the actions
// currently visible on screen.
// - extractActionBlocks() pulls ```action ... ``` blocks from assistant text.
// - runActionBlocks() executes them through the script runner.
import { commandBus } from "./bus"
import { runScriptText } from "./script"
const DSL_REFERENCE = `To act on the UI, emit a fenced \`\`\`action ... \`\`\` block. One command per line.
Commands: navigate <path> | click <target> | fill <target> "<value>" | submit <target> | select <target> <value> | wait <ms> | wait_for <target> | scroll [<target>] | read [<target>] | expect <target> to_contain "<text>" | expect <target> to_be_visible | expect <target> to_have_value "<text>"
Rules: only emit a block when asked to do something. Use only target ids from "Available actions". Short sentence + block. Quote values with spaces. Comments start with #.
Example — User: "go to resources" → "On it.\n\n\`\`\`action\nnavigate /resources\n\`\`\`"`
export type SystemPromptContext = {
/** Optional preface specific to the app/persona. */
preface?: string
/** Current route pathname. */
path?: string
/** Whether to inject the live action snapshot. */
includeActions?: boolean
}
export function buildSystemPrompt(ctx: SystemPromptContext = {}): string {
const parts: string[] = []
parts.push(
ctx.preface ??
"You are the assistant in Comfy Cloud. Answer concisely and drive the UI when asked.",
)
parts.push(DSL_REFERENCE)
if (ctx.includeActions !== false) {
const actions = commandBus.listActions().filter((a) => a.visible)
const path = ctx.path ?? (typeof window !== "undefined" ? window.location.pathname : "")
parts.push(`Route: ${path || "?"}\nAvailable actions:\n${
actions.length === 0
? "(none)"
: actions.map((a) => `- ${a.id}${a.label ? `: ${a.label}` : ""}`).join("\n")
}`)
}
return parts.join("\n\n")
}
/** Rough token estimate: ~4 chars per token. Good enough for budgeting. */
export function estimateTokens(text: string): number {
return Math.ceil(text.length / 4)
}
/** Trim a message list to fit a token budget, preserving the most recent turns. */
export function trimMessages<T extends { content: string }>(
messages: T[],
budgetTokens: number,
): T[] {
let used = 0
const kept: T[] = []
for (let i = messages.length - 1; i >= 0; i--) {
const t = estimateTokens(messages[i].content)
if (used + t > budgetTokens) break
kept.unshift(messages[i])
used += t
}
return kept
}
const ACTION_BLOCK_RE = /```action\s*\n([\s\S]*?)```/g
export function extractActionBlocks(text: string): string[] {
const blocks: string[] = []
let m: RegExpExecArray | null
ACTION_BLOCK_RE.lastIndex = 0
while ((m = ACTION_BLOCK_RE.exec(text)) !== null) {
blocks.push(m[1].trim())
}
return blocks
}
export type RunActionBlocksResult = {
ran: number
errors: string[]
}
export async function runActionBlocks(
text: string,
opts: { signal?: AbortSignal } = {},
): Promise<RunActionBlocksResult> {
const blocks = extractActionBlocks(text)
const errors: string[] = []
let ran = 0
for (const block of blocks) {
try {
await runScriptText(block, { signal: opts.signal })
ran++
} catch (e) {
errors.push(e instanceof Error ? e.message : String(e))
}
}
return { ran, errors }
}