Connect chat to llm

src/games/storywriter/assets/chat-sidebar.module.css

@@ -57,6 +57,23 @@
     word-wrap: break-word;
 }
 
+.loading {
+    color: var(--text-muted);
+    font-style: italic;
+}
+
+.errorMessage {
+    background: rgba(244, 67, 54, 0.1);
+    border-left: 2px solid var(--error, #f44336);
+}
+
+.errorText {
+    font-size: 13px;
+    color: var(--error, #f44336);
+    white-space: pre-wrap;
+    word-wrap: break-word;
+}
+
 .inputContainer {
     display: flex;
     flex-direction: column;

src/games/storywriter/components/chat-sidebar.tsx

@@ -2,11 +2,19 @@ import { Sidebar } from "./sidebar";
 import { useAppState } from "../contexts/state";
 import styles from '../assets/chat-sidebar.module.css';
 import { useState, useRef, useEffect } from "preact/hooks";
+import LLM from "../utils/llm";
+import { highlight } from "../utils/highlight";
+import Prompt from "../utils/prompt";
+import clsx from "clsx";
 
 export const ChatSidebar = () => {
-    const { currentStory, dispatch } = useAppState();
+    const appState = useAppState();
+    const { currentStory, dispatch, connection, model } = appState;
     const [input, setInput] = useState('');
+    const [isLoading, setIsLoading] = useState(false);
+    const [error, setError] = useState<string | null>(null);
     const messagesRef = useRef<HTMLDivElement>(null);
+    const abortControllerRef = useRef<AbortController | null>(null);
 
     useEffect(() => {
         if (messagesRef.current) {
@@ -17,29 +25,81 @@ export const ChatSidebar = () => {
         }
     }, [currentStory?.chatMessages.length]);
 
-    const sendMessage = () => {
+    useEffect(() => {
-        if (!currentStory || !input.trim()) return;
+        return () => {
+            abortControllerRef.current?.abort();
+        };
+    }, []);
+
+    const sendMessage = async () => {
+        if (!currentStory || !input.trim() || !connection || !model || isLoading) return;
+
+        const userMessage = {
+            id: crypto.randomUUID(),
+            role: 'user' as const,
+            content: input.trim(),
+        };
 
         dispatch({
             type: 'ADD_CHAT_MESSAGE',
             storyId: currentStory.id,
-            message: {
+            message: userMessage,
-                id: crypto.randomUUID(),
-                role: 'user',
-                content: input.trim(),
-            },
         });
+
+        const assistantMessageId = crypto.randomUUID();
         dispatch({
             type: 'ADD_CHAT_MESSAGE',
             storyId: currentStory.id,
             message: {
-                id: crypto.randomUUID(),
+                id: assistantMessageId,
                 role: 'assistant',
-                content: 'Assistant message goes here...',
+                content: 'Generating...',
             },
         });
 
         setInput('');
+        setIsLoading(true);
+        setError(null);
+
+        const request = Prompt.compilePrompt(appState, userMessage);
+
+        if (!request) {
+            setError('Failed to compile prompt');
+            setIsLoading(false);
+            return;
+        }
+
+        try {
+            let accumulatedContent = '';
+
+            for await (const chunk of LLM.generateStream(connection, request)) {
+                const delta = chunk.choices[0]?.delta?.content;
+                if (delta) {
+                    accumulatedContent += delta;
+                    dispatch({
+                        type: 'ADD_CHAT_MESSAGE',
+                        storyId: currentStory.id,
+                        message: {
+                            id: assistantMessageId,
+                            role: 'assistant',
+                            content: accumulatedContent,
+                        },
+                    });
+                }
+                if (abortControllerRef.current?.signal.aborted) {
+                    break;
+                }
+            }
+        } catch (err) {
+            const errorMessage = err instanceof Error
+                ? err.message
+                : 'Failed to generate response';
+
+            setError(errorMessage);
+        } finally {
+            setIsLoading(false);
+            abortControllerRef.current = null;
+        }
     };
 
     const handleKeyDown = (e: KeyboardEvent) => {
@@ -57,6 +117,8 @@ export const ChatSidebar = () => {
         });
     };
 
+    const isDisabled = !currentStory || !connection || !model || isLoading;
+
     return (
         <Sidebar side="right">
             <div class={styles.chat}>
@@ -64,6 +126,10 @@ export const ChatSidebar = () => {
                     <div class={styles.placeholder}>
                         Select a story to start chatting
                     </div>
+                ) : !connection || !model ? (
+                    <div class={styles.placeholder}>
+                        {!connection ? 'Connect to an LLM server' : 'Select a model'} to start chatting
+                    </div>
                 ) : currentStory.chatMessages.length === 0 ? (
                     <div class={styles.placeholder}>
                         No messages yet
@@ -73,9 +139,18 @@ export const ChatSidebar = () => {
                         {currentStory.chatMessages.map((message) => (
                             <div key={message.id} class={styles.message} data-role={message.role}>
                                 <div class={styles.role}>{message.role}</div>
-                                <div class={styles.content}>{message.content}</div>
+                                <div
+                                    class={styles.content}
+                                    dangerouslySetInnerHTML={{ __html: highlight(message.content) }}
+                                />
                             </div>
                         ))}
+                        {error && (
+                            <div class={clsx(styles.message, styles.errorMessage)} data-role="assistant">
+                                <div class={styles.role}>error</div>
+                                <div class={styles.errorText}>{error}</div>
+                            </div>
+                        )}
                         <button class={styles.clearButton} onClick={handleClear}>
                             Clear chat
                         </button>
@@ -88,11 +163,16 @@ export const ChatSidebar = () => {
                             value={input}
                             onInput={(e) => setInput((e.target as HTMLTextAreaElement).value)}
                             onKeyDown={handleKeyDown}
-                            placeholder="Type a message..."
+                            placeholder={isDisabled ? 'Connect to an LLM server to chat' : 'Type a message...'}
                             rows={3}
+                            disabled={isDisabled}
                         />
-                        <button class={styles.sendButton} onClick={sendMessage}>
+                        <button
-                            Send
+                            class={styles.sendButton}
+                            onClick={sendMessage}
+                            disabled={isDisabled || !input.trim()}
+                        >
+                            {isLoading ? 'Sending...' : 'Send'}
                         </button>
                     </div>
                 )}

src/games/storywriter/contexts/state.tsx

@@ -101,9 +101,18 @@ function reducer(state: IState, action: Action): IState {
         case 'ADD_CHAT_MESSAGE': {
             return {
                 ...state,
-                stories: state.stories.map(s =>
+                stories: state.stories.map(s => {
-                    s.id === action.storyId ? { ...s, chatMessages: [...s.chatMessages, action.message] } : s
+                    if (s.id !== action.storyId) return s;
-                ),
+                    const existingIndex = s.chatMessages.findIndex(m => m.id === action.message.id);
+                    if (existingIndex !== -1) {
+                        // Overwrite existing message with same id
+                        const updatedMessages = [...s.chatMessages];
+                        updatedMessages[existingIndex] = action.message;
+                        return { ...s, chatMessages: updatedMessages };
+                    }
+                    // Append new message
+                    return { ...s, chatMessages: [...s.chatMessages, action.message] };
+                }),
             };
         }
         case 'CLEAR_CHAT': {
@@ -131,7 +140,7 @@ function reducer(state: IState, action: Action): IState {
 
 // ─── Context ─────────────────────────────────────────────────────────────────
 
-interface IStateContext {
+export interface AppState {
     stories: Story[];
     currentStory: Story | null;
     connection: LLM.Connection | null;
@@ -139,7 +148,7 @@ interface IStateContext {
     dispatch: (action: Action) => void;
 }
 
-const StateContext = createContext<IStateContext>({} as IStateContext);
+const StateContext = createContext<AppState>({} as AppState);
 
 export const useAppState = () => useContext(StateContext);
 
@@ -148,7 +157,7 @@ export const useAppState = () => useContext(StateContext);
 export const StateContextProvider = ({ children }: { children?: any }) => {
     const [state, dispatch] = useStoredReducer('storywriter.state', reducer, DEFAULT_STATE);
 
-    const value = useMemo<IStateContext>(() => ({
+    const value = useMemo<AppState>(() => ({
         stories: state.stories,
         currentStory: state.stories.find(s => s.id === state.currentStoryId) ?? null,
         connection: state.connection,

src/games/storywriter/utils/llm.ts

@@ -1,4 +1,5 @@
 import { formatError } from '@common/errors';
+import Lock from '@common/lock';
 import SSE, { type SSEEvent } from '@common/sse';
 
 namespace LLM {
@@ -116,56 +117,51 @@ namespace LLM {
             payload: body ? JSON.stringify(body) : undefined,
         });
 
-        while (true) {
+        const readable = new ReadableStream<string>({
-            const event = await new Promise<{ data: string } | null>((resolve, reject) => {
+            start: async (controller) => {
-                const onMessage = (e: SSEEvent) => {
+                sse.addEventListener('message', (e) => {
-                    cleanup();
                     if (isMessageEvent(e)) {
-                        resolve(e);
+                        if (e.data === '[DONE]') {
-                    } else {
+                            controller.close();
-                        resolve(null);
+                        } else {
-                    }
+                            controller.enqueue(e.data);
-                };
+                        }
-                const onError = (e: SSEEvent) => {
-                    cleanup();
-                    reject(new Error(formatError(e, 'SSE connection error')));
-                };
-                const onAbort = () => {
-                    cleanup();
-                    resolve(null);
-                };
-                const onReadyStateChange = (e: SSEEvent) => {
-                    if (e != null && typeof e === 'object' && 'readyState' in e && e.readyState === SSE.CLOSED) {
-                        cleanup();
-                        resolve(null);
                     }
+                });
+
+                let closed = false;
+                const handleEnd = (e?: unknown) => {
+                    if (closed) return;
+                    closed = true;
+                    controller.close();
+
+                    console.log(formatError(e));
                 };
 
-                const cleanup = () => {
+                sse.addEventListener('error', handleEnd);
-                    sse.removeEventListener('message', onMessage);
+                sse.addEventListener('abort', handleEnd);
-                    sse.removeEventListener('error', onError);
+                sse.addEventListener('readystatechange', (e) => {
-                    sse.removeEventListener('abort', onAbort);
+                    if (e.readyState === SSE.CLOSED) handleEnd();
-                    sse.removeEventListener('readystatechange', onReadyStateChange);
+                });
-                };
+            }
+        });
 
-                sse.addEventListener('message', onMessage);
+        const reader = readable.getReader();
-                sse.addEventListener('error', onError);
-                sse.addEventListener('abort', onAbort);
-                sse.addEventListener('readystatechange', onReadyStateChange);
-            });
 
-            if (!event || event.data === '[DONE]') {
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done) {
                 break;
             }
-
+            try {
-            if (event.data) {
+                yield JSON.parse(value);
-                try {
+            } catch {
-                    yield JSON.parse(event.data);
+                break;
-                } catch (err) {
-                    console.error('Failed to parse SSE data:', event.data, err);
-                }
             }
         }
+
+        await reader.closed;
+        sse.close();
     }
 
     function isMessageEvent(e: unknown): e is { data: string } {

src/games/storywriter/utils/prompt.ts

@@ -0,0 +1,30 @@
+import LLM from "./llm";
+import type { AppState } from "../contexts/state";
+
+namespace Prompt {
+    export function compilePrompt(state: AppState, newMessage?: LLM.ChatMessage): LLM.ChatCompletionRequest | null {
+        const { currentStory, model } = state;
+
+        if (!currentStory || !model) {
+            return null;
+        }
+
+        const messages: LLM.ChatMessage[] = [
+            // TODO system prompt
+            // TODO part of story
+            ...currentStory.chatMessages,
+        ];
+
+        if (newMessage) {
+            messages.push(newMessage);
+        }
+
+        return {
+            model,
+            messages,
+            // TODO banned_tokens
+        };
+    }
+}
+
+export default Prompt;