From 483609509d58c3a09f77ff10d1b2acfca42977d2 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 9 Jun 2026 18:02:31 +0200 Subject: [PATCH] ui: add opt-in run_javascript frontend tool (#24244) * ui: add opt-in run_javascript frontend tool Expose a run_javascript tool to the model, executed entirely in the browser through the existing agentic loop. Code runs in a Web Worker inside a sandboxed iframe with an opaque origin, isolated from the WebUI and its API. Console output, errors and the return value are fed back as the tool result. The parent enforces a hard timeout by removing the iframe, which terminates the worker. Disabled by default, toggle in Settings > Developer. * ui: address review feedback from allozaur Use the JsonSchemaType enum for the tool definition parameter types instead of raw string literals, extending it with STRING and NUMBER. Move the worker shim and the iframe harness html into their own files so the service no longer carries inline source blobs. Replace the remaining magic strings with constants: SANDBOX_EMPTY_OUTPUT and SANDBOX_TRUNCATION_NOTICE, and reuse NEWLINE_SEPARATOR for joins. * ui: move sandbox worker shim to a raw imported file Replace the inline worker template string with a real sandbox-worker.js imported as raw text, and build the iframe harness from it in sandbox-harness.ts. The raw worker ships as a string, not a module, so it is excluded from eslint and the typecheck program. --- tools/ui/eslint.config.js | 9 +- tools/ui/src/lib/constants/index.ts | 1 + tools/ui/src/lib/constants/sandbox.ts | 39 ++++++ tools/ui/src/lib/constants/settings-keys.ts | 1 + .../ui/src/lib/constants/settings-registry.ts | 8 ++ tools/ui/src/lib/constants/tools.ts | 6 +- tools/ui/src/lib/enums/mcp.enums.ts | 4 +- tools/ui/src/lib/enums/tools.enums.ts | 3 +- tools/ui/src/lib/services/index.ts | 20 ++++ tools/ui/src/lib/services/sandbox-harness.ts | 25 ++++ tools/ui/src/lib/services/sandbox-worker.js | 30 +++++ tools/ui/src/lib/services/sandbox.service.ts | 112 ++++++++++++++++++ tools/ui/src/lib/stores/agentic.svelte.ts | 8 ++ tools/ui/src/lib/stores/tools.svelte.ts | 20 ++++ tools/ui/tsconfig.json | 3 +- 15 files changed, 283 insertions(+), 6 deletions(-) create mode 100644 tools/ui/src/lib/constants/sandbox.ts create mode 100644 tools/ui/src/lib/services/sandbox-harness.ts create mode 100644 tools/ui/src/lib/services/sandbox-worker.js create mode 100644 tools/ui/src/lib/services/sandbox.service.ts diff --git a/tools/ui/eslint.config.js b/tools/ui/eslint.config.js index 6b3b1b5c04..b90376b6c1 100644 --- a/tools/ui/eslint.config.js +++ b/tools/ui/eslint.config.js @@ -46,7 +46,14 @@ export default ts.config( }, { // Exclude generated build output and Storybook files from ESLint - ignores: ['dist/**', 'build/**', '.svelte-kit/**', 'test-results/**', '.storybook/**/*'] + ignores: [ + 'dist/**', + 'build/**', + '.svelte-kit/**', + 'test-results/**', + '.storybook/**/*', + 'src/lib/services/sandbox-worker.js' + ] }, storybook.configs['flat/recommended'] ); diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index c4334132c9..9ab864cd0e 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -37,6 +37,7 @@ export * from './model-id'; export * from './precision'; export * from './processing-info'; export * from './routes'; +export * from './sandbox'; export * from './settings-keys'; export * from './settings-registry'; export * from './supported-file-types'; diff --git a/tools/ui/src/lib/constants/sandbox.ts b/tools/ui/src/lib/constants/sandbox.ts new file mode 100644 index 0000000000..30e769509e --- /dev/null +++ b/tools/ui/src/lib/constants/sandbox.ts @@ -0,0 +1,39 @@ +import { JsonSchemaType, ToolCallType } from '$lib/enums'; +import type { OpenAIToolDefinition } from '$lib/types'; + +export const SANDBOX_TOOL_NAME = 'run_javascript'; + +export const SANDBOX_TIMEOUT_MS_DEFAULT = 10000; + +export const SANDBOX_TIMEOUT_MS_MAX = 30000; + +export const SANDBOX_OUTPUT_MAX_CHARS = 8192; + +export const SANDBOX_EMPTY_OUTPUT = '(no output)'; + +export const SANDBOX_TRUNCATION_NOTICE = '[output truncated]'; + +export const SANDBOX_TOOL_DEFINITION: OpenAIToolDefinition = { + type: ToolCallType.FUNCTION, + function: { + name: SANDBOX_TOOL_NAME, + description: + 'Execute JavaScript in a sandboxed browser worker (no DOM, no page access). ' + + 'Top level await is supported. Use console.log to print intermediate values; ' + + 'a top level return statement is captured as the result.', + parameters: { + type: JsonSchemaType.OBJECT, + properties: { + code: { + type: JsonSchemaType.STRING, + description: 'JavaScript source to execute' + }, + timeout_ms: { + type: JsonSchemaType.NUMBER, + description: `Execution timeout in milliseconds, default ${SANDBOX_TIMEOUT_MS_DEFAULT}, max ${SANDBOX_TIMEOUT_MS_MAX}` + } + }, + required: ['code'] + } + } +}; diff --git a/tools/ui/src/lib/constants/settings-keys.ts b/tools/ui/src/lib/constants/settings-keys.ts index 5fff9f94c2..3e19166dfd 100644 --- a/tools/ui/src/lib/constants/settings-keys.ts +++ b/tools/ui/src/lib/constants/settings-keys.ts @@ -69,6 +69,7 @@ export const SETTINGS_KEYS = { ENABLE_THINKING: 'enableThinking', SHOW_RAW_OUTPUT_SWITCH: 'showRawOutputSwitch', // PY_INTERPRETER_ENABLED: 'pyInterpreterEnabled', + JS_SANDBOX_ENABLED: 'jsSandboxEnabled', CUSTOM_JSON: 'customJson', CUSTOM_CSS: 'customCss' } as const; diff --git a/tools/ui/src/lib/constants/settings-registry.ts b/tools/ui/src/lib/constants/settings-registry.ts index 9246b97033..910e5c2df0 100644 --- a/tools/ui/src/lib/constants/settings-registry.ts +++ b/tools/ui/src/lib/constants/settings-registry.ts @@ -690,6 +690,14 @@ const SETTINGS_REGISTRY: Record = { paramType: SyncableParameterType.BOOLEAN } }, + { + key: SETTINGS_KEYS.JS_SANDBOX_ENABLED, + label: 'JavaScript sandbox tool', + help: 'Expose a run_javascript tool to the model. Code runs in a Web Worker inside a sandboxed iframe with an opaque origin, isolated from the WebUI and its API, with a hard timeout.', + defaultValue: false, + type: SettingsFieldType.CHECKBOX, + section: SETTINGS_SECTION_SLUGS.DEVELOPER + }, { key: SETTINGS_KEYS.CUSTOM_JSON, label: 'Custom JSON', diff --git a/tools/ui/src/lib/constants/tools.ts b/tools/ui/src/lib/constants/tools.ts index efc3476cd7..4d9385f9f5 100644 --- a/tools/ui/src/lib/constants/tools.ts +++ b/tools/ui/src/lib/constants/tools.ts @@ -2,10 +2,12 @@ import { ToolSource } from '$lib/enums/tools.enums'; export const TOOL_GROUP_LABELS = { [ToolSource.BUILTIN]: 'Built-in', - [ToolSource.CUSTOM]: 'JSON Schema' + [ToolSource.CUSTOM]: 'JSON Schema', + [ToolSource.FRONTEND]: 'Browser' } as const; export const TOOL_SERVER_LABELS = { [ToolSource.BUILTIN]: 'Built-in Tools', - [ToolSource.CUSTOM]: 'Custom Tools' + [ToolSource.CUSTOM]: 'Custom Tools', + [ToolSource.FRONTEND]: 'Browser Tools' } as const; diff --git a/tools/ui/src/lib/enums/mcp.enums.ts b/tools/ui/src/lib/enums/mcp.enums.ts index d2c27e1a0c..3d9a2070dc 100644 --- a/tools/ui/src/lib/enums/mcp.enums.ts +++ b/tools/ui/src/lib/enums/mcp.enums.ts @@ -54,7 +54,9 @@ export enum MCPContentType { * JSON Schema types used in MCP tool definitions */ export enum JsonSchemaType { - OBJECT = 'object' + OBJECT = 'object', + STRING = 'string', + NUMBER = 'number' } /** diff --git a/tools/ui/src/lib/enums/tools.enums.ts b/tools/ui/src/lib/enums/tools.enums.ts index 4b2cdab320..120c1d471d 100644 --- a/tools/ui/src/lib/enums/tools.enums.ts +++ b/tools/ui/src/lib/enums/tools.enums.ts @@ -1,7 +1,8 @@ export enum ToolSource { BUILTIN = 'builtin', MCP = 'mcp', - CUSTOM = 'custom' + CUSTOM = 'custom', + FRONTEND = 'frontend' } export enum ToolPermissionDecision { diff --git a/tools/ui/src/lib/services/index.ts b/tools/ui/src/lib/services/index.ts index 475e6419bb..386f740b8f 100644 --- a/tools/ui/src/lib/services/index.ts +++ b/tools/ui/src/lib/services/index.ts @@ -261,6 +261,26 @@ export { ParameterSyncService } from './parameter-sync.service'; */ export { MCPService } from './mcp.service'; +/** + * **SandboxService** - Frontend JavaScript execution in a browser sandbox + * + * Stateless executor for the run_javascript frontend tool. Model generated + * code runs in a Web Worker spawned inside a sandboxed iframe with an opaque + * origin: no access to the app origin, its storage or its API, and outgoing + * requests carry a null origin. The code never touches a main thread, so the + * parent enforces the timeout by removing the iframe, which terminates the + * worker at the browser level. + * + * **Architecture & Relationships:** + * - **SandboxService** (this class): Stateless sandbox execution + * - **toolsStore**: Exposes the tool definition when the sandbox is enabled + * - **agenticStore**: Dispatches ToolSource.FRONTEND calls here + * + * @see SANDBOX_TOOL_DEFINITION in constants/sandbox.ts - tool schema sent to the LLM + * @see agenticStore in stores/agentic.svelte.ts - tool dispatch + */ +export { SandboxService } from './sandbox.service'; + /** * **RouterService** — Dynamic route URL construction utility * diff --git a/tools/ui/src/lib/services/sandbox-harness.ts b/tools/ui/src/lib/services/sandbox-harness.ts new file mode 100644 index 0000000000..27b05e24b4 --- /dev/null +++ b/tools/ui/src/lib/services/sandbox-harness.ts @@ -0,0 +1,25 @@ +import WORKER_SHIM from './sandbox-worker.js?raw'; + +/** + * Harness loaded as srcdoc into a sandboxed iframe (allow-scripts only). + * The opaque origin is the security boundary: no access to the app origin, + * its storage or its API. The harness spawns a worker so model code never + * runs on a main thread, which makes the parent timeout enforceable by + * removing the iframe. + */ +export const SANDBOX_HARNESS_HTML = ``; diff --git a/tools/ui/src/lib/services/sandbox-worker.js b/tools/ui/src/lib/services/sandbox-worker.js new file mode 100644 index 0000000000..689b9211eb --- /dev/null +++ b/tools/ui/src/lib/services/sandbox-worker.js @@ -0,0 +1,30 @@ +const logs = []; +const fmt = (value) => { + if (typeof value === 'string') return value; + try { + return JSON.stringify(value); + } catch { + return String(value); + } +}; +const capture = + (level, prefix) => + (...args) => { + logs.push(prefix + args.map(fmt).join(' ')); + }; +console.log = capture('log', ''); +console.info = capture('info', ''); +console.debug = capture('debug', ''); +console.warn = capture('warn', 'warn: '); +console.error = capture('error', 'error: '); +self.onmessage = async (event) => { + const reply = { logs, result: null, error: null }; + try { + const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; + const value = await new AsyncFunction(event.data.code)(); + if (value !== undefined) reply.result = fmt(value); + } catch (err) { + reply.error = err instanceof Error ? err.stack || err.message : String(err); + } + self.postMessage(reply); +}; diff --git a/tools/ui/src/lib/services/sandbox.service.ts b/tools/ui/src/lib/services/sandbox.service.ts new file mode 100644 index 0000000000..ef0e6cf4eb --- /dev/null +++ b/tools/ui/src/lib/services/sandbox.service.ts @@ -0,0 +1,112 @@ +import { + NEWLINE_SEPARATOR, + SANDBOX_EMPTY_OUTPUT, + SANDBOX_OUTPUT_MAX_CHARS, + SANDBOX_TIMEOUT_MS_DEFAULT, + SANDBOX_TIMEOUT_MS_MAX, + SANDBOX_TOOL_NAME, + SANDBOX_TRUNCATION_NOTICE +} from '$lib/constants'; +import { SANDBOX_HARNESS_HTML } from './sandbox-harness'; +import type { ToolExecutionResult } from '$lib/types'; + +interface SandboxReply { + logs?: unknown; + result?: unknown; + error?: unknown; +} + +function formatReply(reply: SandboxReply): ToolExecutionResult { + const lines: string[] = []; + + if (Array.isArray(reply.logs)) { + for (const line of reply.logs) lines.push(String(line)); + } + + if (reply.error != null) { + lines.push(`Error: ${String(reply.error)}`); + } else if (reply.result != null) { + lines.push(`=> ${String(reply.result)}`); + } + + let content = lines.join(NEWLINE_SEPARATOR); + if (!content) content = SANDBOX_EMPTY_OUTPUT; + if (content.length > SANDBOX_OUTPUT_MAX_CHARS) { + content = `${content.slice(0, SANDBOX_OUTPUT_MAX_CHARS)}${NEWLINE_SEPARATOR}${SANDBOX_TRUNCATION_NOTICE}`; + } + + return { content, isError: reply.error != null }; +} + +export class SandboxService { + /** + * Execute a frontend sandbox tool call and return its output. + * One disposable iframe per execution, removed on completion, + * timeout or abort. Removing the iframe terminates the worker + * at the browser level, so runaway code cannot outlive it. + */ + static executeTool( + toolName: string, + params: Record, + signal?: AbortSignal + ): Promise { + if (toolName !== SANDBOX_TOOL_NAME) { + return Promise.resolve({ content: `Unknown frontend tool: ${toolName}`, isError: true }); + } + + const code = typeof params.code === 'string' ? params.code : ''; + if (!code) { + return Promise.resolve({ content: 'Missing required parameter: code', isError: true }); + } + + const requested = Number(params.timeout_ms); + const timeoutMs = + Number.isFinite(requested) && requested > 0 + ? Math.min(requested, SANDBOX_TIMEOUT_MS_MAX) + : SANDBOX_TIMEOUT_MS_DEFAULT; + + return new Promise((resolve, reject) => { + const iframe = document.createElement('iframe'); + iframe.setAttribute('sandbox', 'allow-scripts'); + iframe.style.display = 'none'; + iframe.srcdoc = SANDBOX_HARNESS_HTML; + + let settled = false; + + const cleanup = () => { + settled = true; + clearTimeout(timer); + window.removeEventListener('message', onMessage); + signal?.removeEventListener('abort', onAbort); + iframe.remove(); + }; + + const finish = (result: ToolExecutionResult) => { + if (settled) return; + cleanup(); + resolve(result); + }; + + const onAbort = () => { + if (settled) return; + cleanup(); + reject(new DOMException('Sandbox execution aborted', 'AbortError')); + }; + + const onMessage = (event: MessageEvent) => { + if (event.source !== iframe.contentWindow) return; + finish(formatReply((event.data ?? {}) as SandboxReply)); + }; + + const timer = setTimeout( + () => finish({ content: `Execution timed out after ${timeoutMs} ms`, isError: true }), + timeoutMs + ); + + window.addEventListener('message', onMessage); + signal?.addEventListener('abort', onAbort); + iframe.onload = () => iframe.contentWindow?.postMessage({ code }, '*'); + document.body.appendChild(iframe); + }); + } +} diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 947737d7c1..5579cc1e5a 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -29,6 +29,7 @@ import { permissionsStore } from '$lib/stores/permissions.svelte'; import { ToolSource, ToolPermissionDecision } from '$lib/enums'; import { SvelteMap } from 'svelte/reactivity'; import { ToolsService } from '$lib/services/tools.service'; +import { SandboxService } from '$lib/services/sandbox.service'; import { isAbortError } from '$lib/utils'; import { DEFAULT_AGENTIC_CONFIG, NEWLINE_SEPARATOR } from '$lib/constants'; import { @@ -784,6 +785,13 @@ class AgenticStore { result = executionResult.content; + if (executionResult.isError) toolSuccess = false; + } else if (toolSource === ToolSource.FRONTEND) { + const args = this.parseToolArguments(toolCall.function.arguments); + const executionResult = await SandboxService.executeTool(toolName, args, signal); + + result = executionResult.content; + if (executionResult.isError) toolSuccess = false; } else { const mcpCall: MCPToolCall = { diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts index 82e41f0bf5..9f0101a82e 100644 --- a/tools/ui/src/lib/stores/tools.svelte.ts +++ b/tools/ui/src/lib/stores/tools.svelte.ts @@ -5,6 +5,7 @@ import { HealthCheckStatus, JsonSchemaType, ToolCallType, ToolSource } from '$li import { config } from '$lib/stores/settings.svelte'; import { DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY, + SANDBOX_TOOL_DEFINITION, TOOL_GROUP_LABELS, TOOL_SERVER_LABELS } from '$lib/constants'; @@ -18,6 +19,8 @@ function toolKey(source: ToolSource, name: string, serverId?: string): string { return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`; case ToolSource.CUSTOM: return `custom:${name}`; + case ToolSource.FRONTEND: + return `frontend:${name}`; default: return `builtin:${name}`; } @@ -82,6 +85,10 @@ class ToolsStore { return mcpStore.getToolDefinitionsForLLM(); } + get frontendTools(): OpenAIToolDefinition[] { + return config().jsSandboxEnabled ? [SANDBOX_TOOL_DEFINITION] : []; + } + get customTools(): OpenAIToolDefinition[] { const raw = config().customJson; if (!raw || typeof raw !== 'string') return []; @@ -156,6 +163,15 @@ class ToolsStore { push({ source: ToolSource.BUILTIN, key: toolKey(ToolSource.BUILTIN, name), definition: def }); } + for (const def of this.frontendTools) { + const name = def.function.name; + push({ + source: ToolSource.FRONTEND, + key: toolKey(ToolSource.FRONTEND, name), + definition: def + }); + } + for (const { serverId, serverName, definition } of this.mcpEntries()) { const name = definition.function.name; push({ @@ -208,6 +224,8 @@ class ToolsStore { return entry.serverName ?? ''; case ToolSource.CUSTOM: return TOOL_GROUP_LABELS[ToolSource.CUSTOM]; + case ToolSource.FRONTEND: + return TOOL_GROUP_LABELS[ToolSource.FRONTEND]; default: return TOOL_GROUP_LABELS[ToolSource.BUILTIN]; } @@ -237,6 +255,7 @@ class ToolsStore { }; for (const def of this._builtinTools) take(def); + for (const def of this.frontendTools) take(def); for (const def of mcpStore.getToolDefinitionsForLLM()) take(def); for (const def of this.customTools) take(def); @@ -346,6 +365,7 @@ class ToolsStore { if (entry.serverName) return mcpStore.getServerDisplayName(entry.serverName); if (entry.source === ToolSource.BUILTIN) return TOOL_SERVER_LABELS[ToolSource.BUILTIN]; if (entry.source === ToolSource.CUSTOM) return TOOL_SERVER_LABELS[ToolSource.CUSTOM]; + if (entry.source === ToolSource.FRONTEND) return TOOL_SERVER_LABELS[ToolSource.FRONTEND]; return ''; } diff --git a/tools/ui/tsconfig.json b/tools/ui/tsconfig.json index 7c585f4db2..51f55971e9 100644 --- a/tools/ui/tsconfig.json +++ b/tools/ui/tsconfig.json @@ -24,7 +24,8 @@ "tests/**/*.svelte", ".storybook/**/*.ts", ".storybook/**/*.svelte" - ] + ], + "exclude": ["src/lib/services/sandbox-worker.js"] // Path aliases are handled by https://svelte.dev/docs/kit/configuration#alias // except $lib which is handled by https://svelte.dev/docs/kit/configuration#files //