diff --git a/packages/backend/server/src/plugins/copilot/providers/utils.ts b/packages/backend/server/src/plugins/copilot/providers/utils.ts index 8a6df94312..3f60da68f9 100644 --- a/packages/backend/server/src/plugins/copilot/providers/utils.ts +++ b/packages/backend/server/src/plugins/copilot/providers/utils.ts @@ -539,6 +539,20 @@ export class StreamObjectParser { } break; } + case 'tool-result': { + const index = acc.findIndex( + item => + item.type === 'tool-call' && + item.toolCallId === curr.toolCallId && + item.toolName === curr.toolName + ); + if (index !== -1) { + acc[index] = curr; + } else { + acc.push(curr); + } + break; + } default: { acc.push(curr); break; diff --git a/packages/frontend/core/src/blocksuite/ai/actions/doc-handler.ts b/packages/frontend/core/src/blocksuite/ai/actions/doc-handler.ts index 90bded5394..8066afbbe0 100644 --- a/packages/frontend/core/src/blocksuite/ai/actions/doc-handler.ts +++ b/packages/frontend/core/src/blocksuite/ai/actions/doc-handler.ts @@ -10,6 +10,7 @@ import { buildFinishConfig, buildGeneratingConfig, } from '../ai-panel'; +import { StreamObjectSchema } from '../components/ai-chat-messages'; import { type AIItemGroupConfig } from '../components/ai-item/types'; import { type AIError, AIProvider } from '../provider'; import { reportResponse } from '../utils/action-reporter'; @@ -21,8 +22,12 @@ import { getSelections, selectAboveBlocks, } from '../utils/selection-utils'; +import { mergeStreamObjects } from '../utils/stream-objects'; import type { AffineAIPanelWidget } from '../widgets/ai-panel/ai-panel'; -import type { AINetworkSearchConfig } from '../widgets/ai-panel/type'; +import type { + AIActionAnswer, + AINetworkSearchConfig, +} from '../widgets/ai-panel/type'; import { actionToAnswerRenderer } from './answer-renderer'; export function bindTextStream( @@ -32,13 +37,15 @@ export function bindTextStream( finish, signal, }: { - update: (text: string) => void; + update: (answer: AIActionAnswer) => void; finish: (state: 'success' | 'error' | 'aborted', err?: AIError) => void; signal?: AbortSignal; } ) { (async () => { - let answer = ''; + const answer: AIActionAnswer = { + content: '', + }; signal?.addEventListener('abort', () => { finish('aborted'); reportResponse('aborted:stop'); @@ -47,7 +54,19 @@ export function bindTextStream( if (signal?.aborted) { return; } - answer += data; + try { + const parsed = StreamObjectSchema.safeParse(JSON.parse(data)); + if (parsed.success) { + answer.streamObjects = mergeStreamObjects([ + ...(answer.streamObjects ?? []), + parsed.data, + ]); + } else { + answer.content += data; + } + } catch { + answer.content += data; + } update(answer); } finish('success'); @@ -137,7 +156,7 @@ function actionToGenerateAnswer( }: { input: string; signal?: AbortSignal; - update: (text: string) => void; + update: (answer: AIActionAnswer) => void; finish: (state: 'success' | 'error' | 'aborted', err?: AIError) => void; }) => { const { selectedBlocks: blocks } = getSelections(host); diff --git a/packages/frontend/core/src/blocksuite/ai/actions/edgeless-handler.ts b/packages/frontend/core/src/blocksuite/ai/actions/edgeless-handler.ts index a4e3c22f6e..27057213df 100644 --- a/packages/frontend/core/src/blocksuite/ai/actions/edgeless-handler.ts +++ b/packages/frontend/core/src/blocksuite/ai/actions/edgeless-handler.ts @@ -37,7 +37,10 @@ import { getSelections, } from '../utils/selection-utils'; import type { AffineAIPanelWidget } from '../widgets/ai-panel/ai-panel'; -import type { AINetworkSearchConfig } from '../widgets/ai-panel/type'; +import type { + AIActionAnswer, + AINetworkSearchConfig, +} from '../widgets/ai-panel/type'; import type { EdgelessCopilotWidget } from '../widgets/edgeless-copilot'; import { actionToAnswerRenderer } from './answer-renderer'; import { EXCLUDING_COPY_ACTIONS } from './consts'; @@ -282,7 +285,7 @@ function actionToGeneration( }: { input: string; signal?: AbortSignal; - update: (text: string) => void; + update: (answer: AIActionAnswer) => void; finish: (state: 'success' | 'error' | 'aborted', err?: AIError) => void; }) => { if (!extract) { diff --git a/packages/frontend/core/src/blocksuite/ai/chat-panel/chat-panel-messages.ts b/packages/frontend/core/src/blocksuite/ai/chat-panel/chat-panel-messages.ts index 31cdb81ddd..faddaade51 100644 --- a/packages/frontend/core/src/blocksuite/ai/chat-panel/chat-panel-messages.ts +++ b/packages/frontend/core/src/blocksuite/ai/chat-panel/chat-panel-messages.ts @@ -21,8 +21,10 @@ import { type ChatMessage, isChatAction, isChatMessage, + StreamObjectSchema, } from '../components/ai-chat-messages'; import { type AIError, AIProvider, UnauthorizedError } from '../provider'; +import { mergeStreamObjects } from '../utils/stream-objects'; import { type ChatContextValue } from './chat-context'; import { HISTORY_IMAGE_ACTIONS } from './const'; import { AIPreloadConfig } from './preload-config'; @@ -387,7 +389,12 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) { last.content = ''; last.createdAt = new Date().toISOString(); } - this.updateContext({ messages, status: 'loading', error: null }); + this.updateContext({ + messages, + status: 'loading', + error: null, + abortController, + }); const { store } = this.host; const stream = await AIProvider.actions.chat({ @@ -404,11 +411,23 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) { reasoning: this._isReasoningActive, webSearch: this._isNetworkActive, }); - this.updateContext({ abortController }); + for await (const text of stream) { const messages = [...this.chatContextValue.messages]; const last = messages[messages.length - 1] as ChatMessage; - last.content += text; + try { + const parsed = StreamObjectSchema.safeParse(JSON.parse(text)); + if (parsed.success) { + last.streamObjects = mergeStreamObjects([ + ...(last.streamObjects ?? []), + parsed.data, + ]); + } else { + last.content += text; + } + } catch { + last.content += text; + } this.updateContext({ messages, status: 'transmitting' }); } diff --git a/packages/frontend/core/src/blocksuite/ai/chat-panel/message/assistant.ts b/packages/frontend/core/src/blocksuite/ai/chat-panel/message/assistant.ts index e50f1b068a..1b469875aa 100644 --- a/packages/frontend/core/src/blocksuite/ai/chat-panel/message/assistant.ts +++ b/packages/frontend/core/src/blocksuite/ai/chat-panel/message/assistant.ts @@ -3,6 +3,7 @@ import '../content/rich-text'; import type { FeatureFlagService } from '@affine/core/modules/feature-flag'; import { WithDisposable } from '@blocksuite/affine/global/lit'; +import { unsafeCSSVarV2 } from '@blocksuite/affine/shared/theme'; import { isInsidePageEditor } from '@blocksuite/affine/shared/utils'; import type { EditorHost } from '@blocksuite/affine/std'; import { ShadowlessElement } from '@blocksuite/affine/std'; @@ -18,9 +19,11 @@ import { type ChatMessage, type ChatStatus, isChatMessage, + type StreamObject, } from '../../components/ai-chat-messages'; import { AIChatErrorRenderer } from '../../messages/error'; import { type AIError } from '../../provider'; +import { mergeStreamContent } from '../../utils/stream-objects'; export class ChatMessageAssistant extends WithDisposable(ShadowlessElement) { static override styles = css` @@ -29,6 +32,20 @@ export class ChatMessageAssistant extends WithDisposable(ShadowlessElement) { font-size: var(--affine-font-xs); font-weight: 400; } + + .reasoning-wrapper { + padding: 16px 20px; + margin: 8px 0; + border-radius: 8px; + background-color: rgba(0, 0, 0, 0.05); + } + + .tool-wrapper { + padding: 12px; + margin: 8px 0; + border-radius: 8px; + border: 0.5px solid ${unsafeCSSVarV2('layer/insideBorder/border')}; + } `; @property({ attribute: false }) @@ -78,33 +95,127 @@ export class ChatMessageAssistant extends WithDisposable(ShadowlessElement) { renderContent() { const { host, item, isLast, status, error } = this; - - const state = isLast - ? status !== 'loading' && status !== 'transmitting' - ? 'finished' - : 'generating' - : 'finished'; + const { streamObjects, content } = item; const shouldRenderError = isLast && status === 'error' && !!error; return html` - ${item.attachments - ? html`` - : nothing} - + ${this.renderImages()} + ${streamObjects?.length + ? this.renderStreamObjects(streamObjects) + : this.renderRichText(content)} ${shouldRenderError ? AIChatErrorRenderer(host, error) : nothing} ${this.renderEditorActions()} `; } - renderEditorActions() { + private renderImages() { + const { item } = this; + if (!item.attachments) return nothing; + + return html``; + } + + private renderStreamObjects(answer: StreamObject[]) { + return html`
+ ${answer.map(data => { + switch (data.type) { + case 'text-delta': + return this.renderRichText(data.textDelta); + case 'reasoning': + return html` +
+ ${this.renderRichText(data.textDelta)} +
+ `; + case 'tool-call': + return this.renderToolCall(data); + case 'tool-result': + return this.renderToolResult(data); + default: + return nothing; + } + })} +
`; + } + + private renderToolCall(streamObject: StreamObject) { + if (streamObject.type !== 'tool-call') { + return nothing; + } + + switch (streamObject.toolName) { + case 'web_crawl_exa': + return html` + + `; + case 'web_search_exa': + return html` + + `; + default: + return html` +
+ ${streamObject.toolName} tool calling... +
+ `; + } + } + + private renderToolResult(streamObject: StreamObject) { + if (streamObject.type !== 'tool-result') { + return nothing; + } + + switch (streamObject.toolName) { + case 'web_crawl_exa': + return html` + + `; + case 'web_search_exa': + return html` + + `; + default: + return html` +
+ ${streamObject.toolName} tool result... +
+ `; + } + } + + private renderRichText(text: string) { + const { host, isLast, status } = this; + const state = isLast + ? status !== 'loading' && status !== 'transmitting' + ? 'finished' + : 'generating' + : 'finished'; + + return html``; + } + + private renderEditorActions() { const { item, isLast, status } = this; if (!isChatMessage(item) || item.role !== 'assistant') return nothing; @@ -118,7 +229,10 @@ export class ChatMessageAssistant extends WithDisposable(ShadowlessElement) { return nothing; const { host } = this; - const { content, id: messageId } = item; + const { content, streamObjects, id: messageId } = item; + const markdown = streamObjects?.length + ? mergeStreamContent(streamObjects) + : content; const actions = isInsidePageEditor(host) ? PageEditorActions @@ -128,18 +242,18 @@ export class ChatMessageAssistant extends WithDisposable(ShadowlessElement) { this.retry()} > - ${isLast && !!content + ${isLast && !!markdown ? html`; + + @state() + private accessor dotsText = '.'; + + private animationTimer?: number; + + override connectedCallback() { + super.connectedCallback(); + this.startDotsAnimation(); + } + + override disconnectedCallback() { + super.disconnectedCallback(); + this.stopDotsAnimation(); + } + + private startDotsAnimation() { + let dotCount = 1; + this.animationTimer = window.setInterval(() => { + dotCount = (dotCount % 3) + 1; + this.dotsText = '.'.repeat(dotCount); + }, 750); + } + + private stopDotsAnimation() { + if (this.animationTimer) { + clearInterval(this.animationTimer); + this.animationTimer = undefined; + } + } + + protected override render() { + return html` +
+
+
${this.icon}
+
+ ${this.name}${this.dotsText} +
+
+
+ `; + } +} diff --git a/packages/frontend/core/src/blocksuite/ai/chat-panel/tools/tool-result-card.ts b/packages/frontend/core/src/blocksuite/ai/chat-panel/tools/tool-result-card.ts new file mode 100644 index 0000000000..bde2f302e2 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/ai/chat-panel/tools/tool-result-card.ts @@ -0,0 +1,160 @@ +import { WithDisposable } from '@blocksuite/affine/global/lit'; +import { ImageProxyService } from '@blocksuite/affine/shared/adapters'; +import { unsafeCSSVarV2 } from '@blocksuite/affine/shared/theme'; +import { type EditorHost, ShadowlessElement } from '@blocksuite/affine/std'; +import { ToggleDownIcon } from '@blocksuite/icons/lit'; +import { css, html, nothing, type TemplateResult } from 'lit'; +import { property } from 'lit/decorators.js'; + +interface ToolResult { + title?: string; + icon?: string | TemplateResult<1>; + content?: string; +} + +export class ToolResultCard extends WithDisposable(ShadowlessElement) { + static override styles = css` + .ai-tool-wrapper { + padding: 12px; + margin: 8px 0; + border-radius: 8px; + border: 0.5px solid ${unsafeCSSVarV2('layer/insideBorder/border')}; + + .ai-tool-header { + display: flex; + justify-content: space-between; + align-items: center; + gap: 8px; + } + + .ai-icon { + width: 24px; + height: 24px; + + svg { + width: 24px; + height: 24px; + color: ${unsafeCSSVarV2('icon/primary')}; + } + } + + .ai-tool-name { + font-size: 14px; + font-weight: 500; + line-height: 24px; + margin-left: 0px; + margin-right: auto; + color: ${unsafeCSSVarV2('icon/primary')}; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .ai-tool-results { + display: flex; + flex-direction: column; + gap: 4px; + margin: 4px 2px 4px 12px; + padding-left: 20px; + border-left: 1px solid ${unsafeCSSVarV2('layer/insideBorder/border')}; + } + + .result-header { + display: flex; + justify-content: space-between; + align-items: center; + } + + .result-title { + font-size: 12px; + font-weight: 400; + line-height: 20px; + color: ${unsafeCSSVarV2('icon/primary')}; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + flex: 1; + } + + .result-icon { + width: 24px; + height: 24px; + + img { + width: 24px; + height: 24px; + border-radius: 100%; + border: 1px solid ${unsafeCSSVarV2('layer/insideBorder/border')}; + } + } + + .result-content { + font-size: 12px; + line-height: 20px; + color: ${unsafeCSSVarV2('text/secondary')}; + margin-top: 8px; + display: -webkit-box; + -webkit-line-clamp: 4; + -webkit-box-orient: vertical; + overflow: hidden; + text-overflow: ellipsis; + } + } + `; + + @property({ attribute: false }) + accessor host!: EditorHost; + + @property({ attribute: false }) + accessor name!: string; + + @property({ attribute: false }) + accessor icon!: TemplateResult<1> | string; + + @property({ attribute: false }) + accessor results!: ToolResult[]; + + protected override render() { + const imageProxyService = this.host.store.get(ImageProxyService); + + return html` +
+
+
${this.icon}
+
${this.name}
+
${ToggleDownIcon()}
+
+
+ ${this.results.map( + result => html` +
+
+
${result.title || 'Untitled'}
+ ${result.icon + ? html` +
+ ${typeof result.icon === 'string' + ? html`icon { + const target = e.target as HTMLImageElement; + target.style.display = 'none'; + }} + />` + : result.icon} +
+ ` + : nothing} +
+ ${result.content + ? html`
${result.content}
` + : nothing} +
+ ` + )} +
+
+ `; + } +} diff --git a/packages/frontend/core/src/blocksuite/ai/chat-panel/tools/web-crawl.ts b/packages/frontend/core/src/blocksuite/ai/chat-panel/tools/web-crawl.ts new file mode 100644 index 0000000000..b2fd4efc32 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/ai/chat-panel/tools/web-crawl.ts @@ -0,0 +1,79 @@ +import { WithDisposable } from '@blocksuite/affine/global/lit'; +import { type EditorHost, ShadowlessElement } from '@blocksuite/affine/std'; +import { WebIcon } from '@blocksuite/icons/lit'; +import { html, nothing } from 'lit'; +import { property } from 'lit/decorators.js'; + +interface WebCrawlToolCall { + type: 'tool-call'; + toolCallId: string; + toolName: string; + args: { url: string }; +} + +interface WebCrawlToolResult { + type: 'tool-result'; + toolCallId: string; + toolName: string; + args: { url: string }; + result: Array<{ + title: string; + url: string; + content: string; + favicon: string; + publishedDate: string; + author: string; + }>; +} + +export class WebCrawlTool extends WithDisposable(ShadowlessElement) { + @property({ attribute: false }) + accessor data!: WebCrawlToolCall | WebCrawlToolResult; + + @property({ attribute: false }) + accessor host!: EditorHost; + + renderToolCall() { + return html` + + `; + } + + renderToolResult() { + if (this.data.type !== 'tool-result') { + return nothing; + } + + const { favicon, title, content } = this.data.result[0]; + + return html` + + `; + } + + protected override render() { + const { data } = this; + + if (data.type === 'tool-call') { + return this.renderToolCall(); + } + if (data.type === 'tool-result') { + return this.renderToolResult(); + } + return nothing; + } +} diff --git a/packages/frontend/core/src/blocksuite/ai/chat-panel/tools/web-search.ts b/packages/frontend/core/src/blocksuite/ai/chat-panel/tools/web-search.ts new file mode 100644 index 0000000000..564786379f --- /dev/null +++ b/packages/frontend/core/src/blocksuite/ai/chat-panel/tools/web-search.ts @@ -0,0 +1,79 @@ +import { WithDisposable } from '@blocksuite/affine/global/lit'; +import { type EditorHost, ShadowlessElement } from '@blocksuite/affine/std'; +import { WebIcon } from '@blocksuite/icons/lit'; +import { html, nothing } from 'lit'; +import { property } from 'lit/decorators.js'; + +interface WebSearchToolCall { + type: 'tool-call'; + toolCallId: string; + toolName: string; + args: { url: string }; +} + +interface WebSearchToolResult { + type: 'tool-result'; + toolCallId: string; + toolName: string; + args: { url: string }; + result: Array<{ + title: string; + url: string; + content: string; + favicon: string; + publishedDate: string; + author: string; + }>; +} + +export class WebSearchTool extends WithDisposable(ShadowlessElement) { + @property({ attribute: false }) + accessor data!: WebSearchToolCall | WebSearchToolResult; + + @property({ attribute: false }) + accessor host!: EditorHost; + + renderToolCall() { + return html` + + `; + } + renderToolResult() { + if (this.data.type !== 'tool-result') { + return nothing; + } + + const results = this.data.result.map(item => { + const { favicon, title, content } = item; + return { + title: title, + icon: favicon, + content: content, + }; + }); + + return html` + + `; + } + + protected override render() { + const { data } = this; + + if (data.type === 'tool-call') { + return this.renderToolCall(); + } + if (data.type === 'tool-result') { + return this.renderToolResult(); + } + return nothing; + } +} diff --git a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/ai-chat-input.ts b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/ai-chat-input.ts index b6d3b40773..bae888ae7d 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/ai-chat-input.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-input/ai-chat-input.ts @@ -16,9 +16,10 @@ import { ChatAbortIcon } from '../../_common/icons'; import { type AIError, AIProvider } from '../../provider'; import { reportResponse } from '../../utils/action-reporter'; import { readBlobAsURL } from '../../utils/image'; +import { mergeStreamObjects } from '../../utils/stream-objects'; import type { ChatChip, DocDisplayConfig } from '../ai-chat-chips/type'; import { isDocChip } from '../ai-chat-chips/utils'; -import type { ChatMessage } from '../ai-chat-messages'; +import { type ChatMessage, StreamObjectSchema } from '../ai-chat-messages'; import { MAX_IMAGE_COUNT } from './const'; import type { AIChatInputContext, @@ -610,7 +611,19 @@ export class AIChatInput extends SignalWatcher( for await (const text of stream) { const messages = [...this.chatContextValue.messages]; const last = messages[messages.length - 1] as ChatMessage; - last.content += text; + try { + const parsed = StreamObjectSchema.safeParse(JSON.parse(text)); + if (parsed.success) { + last.streamObjects = mergeStreamObjects([ + ...(last.streamObjects ?? []), + parsed.data, + ]); + } else { + last.content += text; + } + } catch { + last.content += text; + } this.updateContext({ messages, status: 'transmitting' }); } diff --git a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-messages/type.ts b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-messages/type.ts index d15751e4af..a3c5f35987 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/ai-chat-messages/type.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/ai-chat-messages/type.ts @@ -1,6 +1,6 @@ import { z } from 'zod'; -const StreamObjectSchema = z.discriminatedUnion('type', [ +export const StreamObjectSchema = z.discriminatedUnion('type', [ z.object({ type: z.literal('text-delta'), textDelta: z.string(), diff --git a/packages/frontend/core/src/blocksuite/ai/components/ai-scrollable-text-renderer.ts b/packages/frontend/core/src/blocksuite/ai/components/ai-scrollable-text-renderer.ts index 8dab3b0ec7..492a1030b3 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/ai-scrollable-text-renderer.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/ai-scrollable-text-renderer.ts @@ -112,7 +112,7 @@ export const createAIScrollableTextRenderer: ( maxHeight, autoScroll ) => { - return (answer, state) => { + return (answer: string, state: AffineAIPanelState | undefined) => { return html` ExtensionType[] = () => { const manager = getViewManager().config.init().value; @@ -430,11 +427,11 @@ export class TextRenderer extends WithDisposable(ShadowlessElement) { accessor state: AffineAIPanelState | undefined = undefined; } -export const createTextRenderer: ( +export const createTextRenderer = ( host: EditorHost, options: TextRendererOptions -) => AffineAIPanelWidgetConfig['answerRenderer'] = (host, options) => { - return (answer, state) => { +) => { + return (answer: string, state?: AffineAIPanelState) => { return html` { + const prev = acc.at(-1); + switch (curr.type) { + case 'reasoning': + case 'text-delta': { + if (prev && prev.type === curr.type) { + prev.textDelta += curr.textDelta; + } else { + acc.push(curr); + } + break; + } + case 'tool-result': { + const index = acc.findIndex( + item => + item.type === 'tool-call' && + item.toolCallId === curr.toolCallId && + item.toolName === curr.toolName + ); + if (index !== -1) { + acc[index] = curr; + } else { + acc.push(curr); + } + break; + } + default: { + acc.push(curr); + break; + } + } + return acc; + }, [] as StreamObject[]); +} + +export function mergeStreamContent(chunks: StreamObject[]): string { + return chunks.reduce((acc, curr) => { + if (curr.type === 'text-delta') { + acc += curr.textDelta; + } + return acc; + }, ''); +} diff --git a/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/ai-panel.ts b/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/ai-panel.ts index 7555cdd84b..7f014b51e2 100644 --- a/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/ai-panel.ts +++ b/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/ai-panel.ts @@ -37,7 +37,12 @@ import { literal, unsafeStatic } from 'lit/static-html.js'; import { type AIError } from '../../provider'; import type { AIPanelGenerating } from './components/index.js'; -import type { AffineAIPanelState, AffineAIPanelWidgetConfig } from './type.js'; +import type { + AffineAIPanelState, + AffineAIPanelWidgetConfig, + AIActionAnswer, +} from './type.js'; +import { mergeAIActionAnswer } from './utils'; export const AFFINE_AI_PANEL_WIDGET = 'affine-ai-panel-widget'; export class AffineAIPanelWidget extends WidgetComponent { @@ -103,7 +108,7 @@ export class AffineAIPanelWidget extends WidgetComponent { private _abortController = new AbortController(); - private _answer: string | null = null; + private _answer: AIActionAnswer | null = null; private readonly _clearDiscardModal = () => { if (this._discardModalAbort) { @@ -226,7 +231,7 @@ export class AffineAIPanelWidget extends WidgetComponent { // reset answer this._answer = null; - const update = (answer: string) => { + const update = (answer: AIActionAnswer) => { this._answer = answer; this.requestUpdate(); }; @@ -345,7 +350,7 @@ export class AffineAIPanelWidget extends WidgetComponent { }; get answer() { - return this._answer; + return this._answer ? mergeAIActionAnswer(this._answer) : null; } get inputText() { diff --git a/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/type.ts b/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/type.ts index 0dc21c3d87..72c6e188e7 100644 --- a/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/type.ts +++ b/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/type.ts @@ -1,6 +1,7 @@ import type { Signal } from '@preact/signals-core'; import type { nothing, TemplateResult } from 'lit'; +import type { StreamObject } from '../../components/ai-chat-messages'; import type { AIItemGroupConfig } from '../../components/ai-item/types'; import type { AIError } from '../../provider'; @@ -34,6 +35,11 @@ export interface AINetworkSearchConfig { setEnabled: (state: boolean) => void; } +export type AIActionAnswer = { + content: string; + streamObjects?: StreamObject[]; +}; + export interface AffineAIPanelWidgetConfig { answerRenderer: ( answer: string, @@ -41,7 +47,7 @@ export interface AffineAIPanelWidgetConfig { ) => TemplateResult<1> | typeof nothing; generateAnswer?: (props: { input: string; - update: (answer: string) => void; + update: (answer: AIActionAnswer) => void; finish: (type: 'success' | 'error' | 'aborted', err?: AIError) => void; // Used to allow users to stop actively when generating signal: AbortSignal; diff --git a/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/utils.ts b/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/utils.ts index 809a82b155..db69578881 100644 --- a/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/utils.ts +++ b/packages/frontend/core/src/blocksuite/ai/widgets/ai-panel/utils.ts @@ -2,6 +2,8 @@ import { isInsidePageEditor } from '@blocksuite/affine/shared/utils'; import type { EditorHost } from '@blocksuite/affine/std'; import type { AIItemGroupConfig } from '../../components/ai-item/types'; +import { mergeStreamContent } from '../../utils/stream-objects'; +import type { AIActionAnswer } from './type'; export function filterAIItemGroup( host: EditorHost, @@ -19,3 +21,10 @@ export function filterAIItemGroup( })) .filter(group => group.items.length > 0); } + +export function mergeAIActionAnswer(answer: AIActionAnswer): string { + if (answer.streamObjects?.length) { + return mergeStreamContent(answer.streamObjects); + } + return answer.content; +}