diff --git a/tools/ui/package-lock.json b/tools/ui/package-lock.json index 61fa529d20..9d0cdfea6c 100644 --- a/tools/ui/package-lock.json +++ b/tools/ui/package-lock.json @@ -40,6 +40,7 @@ "eslint-config-prettier": "10.1.8", "eslint-plugin-storybook": "10.4.2", "eslint-plugin-svelte": "3.19.0", + "fflate": "0.8.3", "globals": "16.5.0", "highlight.js": "11.11.1", "http-server": "14.1.1", @@ -9454,6 +9455,13 @@ } } }, + "node_modules/fflate": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", + "dev": true, + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", diff --git a/tools/ui/package.json b/tools/ui/package.json index 2f47992e1e..4803922889 100644 --- a/tools/ui/package.json +++ b/tools/ui/package.json @@ -59,6 +59,7 @@ "eslint-config-prettier": "10.1.8", "eslint-plugin-storybook": "10.4.2", "eslint-plugin-svelte": "3.19.0", + "fflate": "0.8.3", "globals": "16.5.0", "highlight.js": "11.11.1", "http-server": "14.1.1", diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte index b7b91d65b6..a86d68584c 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte @@ -132,14 +132,18 @@ async function handleExportConfirm(selectedConversations: DatabaseConversation[]) { try { - const allData: ExportedConversations = await Promise.all( + const allData: ExportedConversation[] = await Promise.all( selectedConversations.map(async (conv) => { const messages = await conversationsStore.getConversationMessages(conv.id); return { conv: $state.snapshot(conv), messages: $state.snapshot(messages) }; }) ); - conversationsStore.downloadConversationFile(allData); + if (allData.length === 1) { + conversationsStore.downloadConversationFile(allData[0]); + } else { + conversationsStore.downloadConversationsArchive(allData); + } exportedConversations = selectedConversations; showExportSummary = true; @@ -156,37 +160,21 @@ const input = document.createElement('input'); input.type = HtmlInputType.FILE; - input.accept = FileExtensionText.JSON; + input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`; input.onchange = async (e) => { const file = (e.target as HTMLInputElement)?.files?.[0]; if (!file) return; try { - const text = await file.text(); - const parsedData = JSON.parse(text); - let importedData: ExportedConversations; + const importedData = await conversationsStore.parseImportFile(file); - if (Array.isArray(parsedData)) { - importedData = parsedData; - } else if ( - parsedData && - typeof parsedData === 'object' && - 'conv' in parsedData && - 'messages' in parsedData - ) { - // Single conversation object - importedData = [parsedData]; - } else { - throw new Error( - 'Invalid file format: expected array of conversations or single conversation object' - ); + if (importedData.length === 0) { + throw new Error('No conversations found in file'); } fullImportData = importedData; - availableConversations = importedData.map( - (item: { conv: DatabaseConversation; messages: DatabaseMessage[] }) => item.conv - ); + availableConversations = importedData.map((item) => item.conv); messageCountMap = createMessageCountMap(importedData); showImportDialog = true; } catch (err: unknown) { @@ -258,7 +246,7 @@ { + // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it. + const { toolCalls, ...rest } = message; + const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest; + + return JSON.stringify({ type: 'message', message: normalized }); + }); + + return [sessionLine, ...messageLines].join('\n'); + } + + /** + * Parses the JSONL session format produced by {@link serializeSessionToJsonl}. + * A `type: 'session'` line starts a new session; following `type: 'message'` + * lines are appended to it. Supports multiple sessions in a single file. + * @param text - The JSONL file contents + * @returns The parsed conversations with their messages + */ + parseSessionsJsonl(text: string): ExportedConversation[] { + const sessions: ExportedConversation[] = []; + let current: ExportedConversation | null = null; + + for (const line of text.split('\n')) { + const trimmed = line.trim(); + if (!trimmed) continue; + + const record = JSON.parse(trimmed); + + if (record.type === 'session') { + // Drop the discriminator and harness marker; the rest is the conversation. + const conv = { ...record }; + delete conv.type; + delete conv.harness; + current = { conv: conv as DatabaseConversation, messages: [] }; + sessions.push(current); + } else if (record.type === 'message') { + if (!current) { + throw new Error('Invalid JSONL: message record before any session record'); + } + + const message = record.message as DatabaseMessage; + // `toolCalls` is parsed to an array on export; the DB stores it as a string. + if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') { + message.toolCalls = JSON.stringify(message.toolCalls); + } + current.messages.push(message); + } + // Ignore unknown record types for forward compatibility. + } + + return sessions; + } + + /** + * Parses an import file into conversations, accepting the current `.jsonl` and + * `.zip` formats as well as the legacy `.json` format. + * @param file - The user-selected file + * @returns The parsed conversations with their messages + */ + async parseImportFile(file: File): Promise { + const name = file.name.toLowerCase(); + + if (name.endsWith(FileExtensionText.ZIP)) { + const entries = unzipSync(new Uint8Array(await file.arrayBuffer())); + const sessions: ExportedConversation[] = []; + for (const [entryName, bytes] of Object.entries(entries)) { + if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue; + sessions.push(...this.parseSessionsJsonl(strFromU8(bytes))); + } + return sessions; + } + + const text = await file.text(); + + if (name.endsWith(FileExtensionText.JSONL)) { + return this.parseSessionsJsonl(text); + } + + // Legacy JSON format: an array of conversations or a single conversation object. + const parsed = JSON.parse(text); + if (Array.isArray(parsed)) { + return parsed; + } + if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) { + return [parsed]; + } + throw new Error( + 'Invalid file format: expected array of conversations or single conversation object' + ); } /** * Triggers a browser download of the provided exported conversation data - * @param data - The exported conversation payload (either a single conversation or array of them) + * @param data - The exported conversation payload (a single conversation with its messages) * @param filename - Filename; if omitted, a deterministic name is generated */ - downloadConversationFile(data: ExportedConversations, filename?: string): void { - // Choose the first conversation or message - const conversation = - 'conv' in data ? data.conv : Array.isArray(data) ? data[0]?.conv : undefined; - const msgs = - 'messages' in data ? data.messages : Array.isArray(data) ? data[0]?.messages : undefined; + downloadConversationFile(data: ExportedConversation, filename?: string): void { + const { conv: conversation, messages: msgs } = data; if (!conversation) { console.error('Invalid data: missing conversation'); return; } - let downloadFilename: string; + const downloadFilename = filename ?? this.generateConversationFilename(conversation, msgs); - if (filename) { - downloadFilename = filename; - } else if (Array.isArray(data) && data.length > 1) { - downloadFilename = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations.json`; - } else { - downloadFilename = this.generateConversationFilename(conversation, msgs); + const jsonl = this.serializeSessionToJsonl(data); + const blob = new Blob([jsonl], { type: MimeTypeText.JSONL }); + this.triggerDownload(blob, downloadFilename); + } + + /** + * Triggers a browser download of multiple conversations as a `.zip`, one + * `.jsonl` file per conversation. + * @param data - The conversations to export + */ + downloadConversationsArchive(data: ExportedConversation[]): void { + if (data.length === 0) { + console.error('Invalid data: no conversations to export'); + return; } - const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' }); + const usedNames = new SvelteSet(); + const files: Record = {}; + + for (const session of data) { + const baseName = this.generateConversationFilename(session.conv, session.messages); + + // Disambiguate any duplicate filenames within the archive. + let entryName = baseName; + let suffix = 1; + while (usedNames.has(entryName)) { + entryName = baseName.replace( + new RegExp(`${FileExtensionText.JSONL}$`), + `_${suffix++}${FileExtensionText.JSONL}` + ); + } + usedNames.add(entryName); + + files[entryName] = strToU8(this.serializeSessionToJsonl(session)); + } + + const archiveName = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations${FileExtensionText.ZIP}`; + + const zipped = zipSync(files); + const blob = new Blob([zipped], { type: MimeTypeApplication.ZIP }); + this.triggerDownload(blob, archiveName); + } + + /** + * Triggers a browser download of a blob under the given filename. + */ + private triggerDownload(blob: Blob, filename: string): void { const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; - a.download = downloadFilename; + a.download = filename; document.body.appendChild(a); a.click(); document.body.removeChild(a);