Check for models modality

src/games/storywriter/components/settings/connection.tsx

@@ -30,7 +30,7 @@ export const ConnectionSettings = () => {
 
     const fetchModels = useMemo(() => async (conn: LLM.Connection | null) => {
         if (!conn) return [];
-        const r = await LLM.getModels(conn);
+        const r = await LLM.getTextModels(conn);
         return r.data;
     }, []);
 

src/games/storywriter/utils/llm.ts

@@ -118,13 +118,22 @@ namespace LLM {
         error: string;
     }
 
-    export interface ModelInfo {
+    type Modality = 'text' | 'image';
+
+    interface BaseModelInfo {
         id: string;
         object: 'model';
         created: number;
         owned_by: string;
-        context_length: number;
         supported_parameters: string[];
+        architecture?: {
+            input_modalities: Modality[];
+            output_modalities: Modality[];
+        };
+    }
+
+    interface ModelInfoText extends BaseModelInfo {
+        context_length: number;
         top_provider: {
             context_length: number;
             max_completion_tokens: number;
@@ -132,11 +141,22 @@ namespace LLM {
         };
     }
 
-    export interface ModelsResponse {
+    interface ModelInfoImage extends BaseModelInfo {
-        object: 'list';
-        data: ModelInfo[];
     }
 
+    export type ModelInfo = ModelInfoText | ModelInfoImage;
+
+    const isTextModel = (model: ModelInfo): model is ModelInfoText => ('context_length' in model);
+    const isImageModel = (model: ModelInfo): model is ModelInfoImage => Boolean(
+        !isTextModel(model) &&
+        model.architecture &&
+        (model.architecture.output_modalities).includes('image')
+    );
+
+    export interface ModelsResponse<T extends ModelInfo = ModelInfo> {
+        object: 'list';
+        data: T[];
+    }
 
     interface CountTokensRequestString {
         model: string;
@@ -246,8 +266,18 @@ namespace LLM {
         return e != null && typeof e === 'object' && 'data' in e && typeof e.data === 'string';
     }
 
-    export async function getModels(connection: Connection): Promise<ModelsResponse> {
+    export async function getTextModels(connection: Connection): Promise<ModelsResponse<ModelInfoText>> {
-        return request<ModelsResponse>(connection, '/v1/models');
+        const response = await request<ModelsResponse>(connection, '/v1/models');
+
+        response.data = response.data.filter(isTextModel);
+        return response as ModelsResponse<ModelInfoText>;
+    }
+
+    export async function getImageModels(connection: Connection): Promise<ModelsResponse<ModelInfoImage>> {
+        const response = await request<ModelsResponse>(connection, '/v1/models');
+
+        response.data = response.data.filter(isImageModel);
+        return response as ModelsResponse<ModelInfoImage>;
     }
 
     export async function countTokens(connection: Connection, body: CountTokensRequest) {
@@ -262,7 +292,7 @@ namespace LLM {
     }
 
     export async function generate(connection: Connection, config: ChatCompletionRequest) {
-        return request<ChatCompletionResponse| ChatCompletionError>(connection, '/v1/chat/completions', 'POST', {
+        return request<ChatCompletionResponse | ChatCompletionError>(connection, '/v1/chat/completions', 'POST', {
             ...config,
             stream: false,
         });