Update jinja template lib

bun.lock

@@ -1,16 +1,17 @@
 {
   "lockfileVersion": 1,
+  "configVersion": 0,
   "workspaces": {
     "": {
       "name": "binario",
       "dependencies": {
-        "@huggingface/gguf": "0.1.12",
+        "@huggingface/gguf": "0.3.4",
-        "@huggingface/hub": "0.19.0",
+        "@huggingface/hub": "2.9.0",
-        "@huggingface/jinja": "0.3.1",
+        "@huggingface/jinja": "0.5.5",
         "@huggingface/transformers": "3.0.2",
         "@inquirer/select": "2.3.10",
         "ace-builds": "1.36.3",
-        "classnames": "2.5.1",
+        "clsx": "2.1.1",
         "delay": "6.0.0",
         "preact": "10.22.0",
       },
@@ -28,15 +29,15 @@
     },
   },
   "packages": {
-    "@emnapi/runtime": ["@emnapi/runtime@1.3.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw=="],
+    "@emnapi/runtime": ["@emnapi/runtime@1.8.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg=="],
 
-    "@huggingface/gguf": ["@huggingface/gguf@0.1.12", "", {}, "sha512-m+u/ms28wE74v2VVCTncfI/KB2v897MRMOoRuYSU62P85fJ6/B2exMlHCNyAXkgDLeXBWDivXl4gPq+XbHmkaA=="],
+    "@huggingface/gguf": ["@huggingface/gguf@0.3.4", "", { "dependencies": { "@huggingface/tasks": "^0.19.85" }, "bin": { "gguf-view": "dist/cli.js" } }, "sha512-+qAgKfIXF75oonWfLzN4X5kc2gUvmbbiBXRZzZ99wQ4pe7RqpaNdByzRb9/BIJXFQB1MJiXdbEXFFWeeISiIZw=="],
 
-    "@huggingface/hub": ["@huggingface/hub@0.19.0", "", { "dependencies": { "@huggingface/tasks": "^0.12.28" } }, "sha512-5f0POHxsQzi1RrtGmk5I+PuSXQnWx4c7WU6+JofZcdrb5mT5frV01MpGS41HRPwoQm2ZWjBfzkRpAnqBNCw2Hg=="],
+    "@huggingface/hub": ["@huggingface/hub@2.9.0", "", { "dependencies": { "@huggingface/tasks": "^0.19.84" }, "optionalDependencies": { "cli-progress": "^3.12.0" }, "bin": { "hfjs": "dist/cli.js" } }, "sha512-g0UKVcPq0ULnakTB6ZGPNTHBVj9nHgsbukbI/gCp1a7uAp8/Jp8fxrKbNqJgeMiLLJcuQGd/4HXJLUYhjd5xxw=="],
 
-    "@huggingface/jinja": ["@huggingface/jinja@0.3.1", "", {}, "sha512-SbcBWUKDQ76lzlVYOloscUk0SJjuL1LcbZsfQv/Bxxc7dwJMYuS+DAQ+HhVw6ZkTFXArejaX5HQRuCuleYwYdA=="],
+    "@huggingface/jinja": ["@huggingface/jinja@0.5.5", "", {}, "sha512-xRlzazC+QZwr6z4ixEqYHo9fgwhTZ3xNSdljlKfUFGZSdlvt166DljRELFUfFytlYOYvo3vTisA/AFOuOAzFQQ=="],
 
-    "@huggingface/tasks": ["@huggingface/tasks@0.12.28", "", { "dependencies": { "@huggingface/gguf": "^0.1.12" } }, "sha512-kjc6PBhwo6+UmdelcdLku6Jj18bRDXlfRUweAaCSrXrX44enyNtm/L+Z8HZO1mqOAdRBVTz+MK2yWIwcb+8drg=="],
+    "@huggingface/tasks": ["@huggingface/tasks@0.19.85", "", {}, "sha512-4s5+GV257GRmT7Hrt190KlGsuq53IwFtCBj+J+bvoeIpn119589wY7WY/LS5VJMLwu7pUQnouLmMKeEOKO4BSQ=="],
 
     "@huggingface/transformers": ["@huggingface/transformers@3.0.2", "", { "dependencies": { "@huggingface/jinja": "^0.3.0", "onnxruntime-node": "1.19.2", "onnxruntime-web": "1.21.0-dev.20241024-d9ca84ef96", "sharp": "^0.33.5" } }, "sha512-lTyS81eQazMea5UCehDGFMfdcNRZyei7XQLH5X6j4AhA/18Ka0+5qPgMxUxuZLU4xkv60aY2KNz9Yzthv6WVJg=="],
 
@@ -156,14 +157,16 @@
 
     "chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="],
 
-    "classnames": ["classnames@2.5.1", "", {}, "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow=="],
-
     "clean-css": ["clean-css@4.2.4", "", { "dependencies": { "source-map": "~0.6.0" } }, "sha512-EJUDT7nDVFDvaQgAo2G/PJvxmp1o/c6iXLbswsBbUFXi1Nr+AjA2cKmfbKDMjMvzEe75g3P6JkaDDAKk96A85A=="],
 
+    "cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
+
     "cli-spinners": ["cli-spinners@2.9.2", "", {}, "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg=="],
 
     "cli-width": ["cli-width@4.1.0", "", {}, "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ=="],
 
+    "clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="],
+
     "color": ["color@4.2.3", "", { "dependencies": { "color-convert": "^2.0.1", "color-string": "^1.9.0" } }, "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A=="],
 
     "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="],
@@ -182,7 +185,7 @@
 
     "delay": ["delay@6.0.0", "", {}, "sha512-2NJozoOHQ4NuZuVIr5CWd0iiLVIRSDepakaovIN+9eIDHEhdCAEvSy2cuf1DCrPPQLvHmbqTHODlhHg8UCy4zw=="],
 
-    "detect-libc": ["detect-libc@2.0.3", "", {}, "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw=="],
+    "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
 
     "eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="],
 
@@ -202,7 +205,7 @@
 
     "html-minifier": ["html-minifier@4.0.0", "", { "dependencies": { "camel-case": "^3.0.0", "clean-css": "^4.2.1", "commander": "^2.19.0", "he": "^1.2.0", "param-case": "^2.1.1", "relateurl": "^0.2.7", "uglify-js": "^3.5.1" }, "bin": { "html-minifier": "./cli.js" } }, "sha512-aoGxanpFPLg7MkIl/DDFYtb0iWz7jMFGqFhvEDZga6/4QTjneiD8I/NXL1x5aaoCp7FSIT6h/OhykDdPsbtMig=="],
 
-    "is-arrayish": ["is-arrayish@0.3.2", "", {}, "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="],
+    "is-arrayish": ["is-arrayish@0.3.4", "", {}, "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA=="],
 
     "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="],
 
@@ -254,7 +257,7 @@
 
     "rxjs": ["rxjs@7.8.1", "", { "dependencies": { "tslib": "^2.1.0" } }, "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg=="],
 
-    "semver": ["semver@7.6.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A=="],
+    "semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="],
 
     "sharp": ["sharp@0.33.5", "", { "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", "semver": "^7.6.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.33.5", "@img/sharp-darwin-x64": "0.33.5", "@img/sharp-libvips-darwin-arm64": "1.0.4", "@img/sharp-libvips-darwin-x64": "1.0.4", "@img/sharp-libvips-linux-arm": "1.0.5", "@img/sharp-libvips-linux-arm64": "1.0.4", "@img/sharp-libvips-linux-s390x": "1.0.4", "@img/sharp-libvips-linux-x64": "1.0.4", "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", "@img/sharp-libvips-linuxmusl-x64": "1.0.4", "@img/sharp-linux-arm": "0.33.5", "@img/sharp-linux-arm64": "0.33.5", "@img/sharp-linux-s390x": "0.33.5", "@img/sharp-linux-x64": "0.33.5", "@img/sharp-linuxmusl-arm64": "0.33.5", "@img/sharp-linuxmusl-x64": "0.33.5", "@img/sharp-wasm32": "0.33.5", "@img/sharp-win32-ia32": "0.33.5", "@img/sharp-win32-x64": "0.33.5" } }, "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw=="],
 
@@ -264,7 +267,7 @@
 
     "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="],
 
-    "simple-swizzle": ["simple-swizzle@0.2.2", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg=="],
+    "simple-swizzle": ["simple-swizzle@0.2.4", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw=="],
 
     "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
 
@@ -300,6 +303,8 @@
 
     "yoctocolors-cjs": ["yoctocolors-cjs@2.1.2", "", {}, "sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA=="],
 
+    "@huggingface/transformers/@huggingface/jinja": ["@huggingface/jinja@0.3.1", "", {}, "sha512-SbcBWUKDQ76lzlVYOloscUk0SJjuL1LcbZsfQv/Bxxc7dwJMYuS+DAQ+HhVw6ZkTFXArejaX5HQRuCuleYwYdA=="],
+
     "@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="],
 
     "@isaacs/cliui/strip-ansi": ["strip-ansi@7.1.0", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ=="],

package.json

@@ -8,13 +8,13 @@
     "bake": "bun build/build.ts"
   },
   "dependencies": {
-    "@huggingface/gguf": "0.1.12",
+    "@huggingface/gguf": "0.3.4",
-    "@huggingface/hub": "0.19.0",
+    "@huggingface/hub": "2.9.0",
-    "@huggingface/jinja": "0.3.1",
+    "@huggingface/jinja": "0.5.5",
     "@huggingface/transformers": "3.0.2",
     "@inquirer/select": "2.3.10",
     "ace-builds": "1.36.3",
-    "classnames": "2.5.1",
+    "clsx": "2.1.1",
     "delay": "6.0.0",
     "preact": "10.22.0"
   },

src/common/display/brick.tsx

@@ -1,7 +1,7 @@
 import { render } from "preact";
 import type { ReactElement } from "preact/compat";
 import { clamp, range } from "@common/utils";
-import classNames from "classnames";
+import clsx from "clsx";
 
 import styles from './assets/brick.module.css';
 import "./assets/lcd.font.css";
@@ -202,7 +202,7 @@ export class BrickDisplay {
                     {range(FIELD_WIDTH * FIELD_HEIGHT).map(i => (
                         <div
                             key={i}
-                            class={classNames(styles.pixel, {
+                            class={clsx(styles.pixel, {
                                 [styles.active]: this.#field[i],
                             })}
                         />
@@ -214,7 +214,7 @@ export class BrickDisplay {
                         {range(MINI_FIELD_WIDTH * MINI_FIELD_HEIGHT).map(i => (
                             <div
                                 key={i}
-                                class={classNames(styles.pixel, {
+                                class={clsx(styles.pixel, {
                                     [styles.active]: this.#miniField[i],
                                 })}
                             />
@@ -234,8 +234,8 @@ export class BrickDisplay {
                         {this.helpText}
                     </div>
                     <div className={styles.footer}>
-                        <div className={classNames(styles.text, this.pause && styles.active)}>Pause</div>
+                        <div className={clsx(styles.text, this.pause && styles.active)}>Pause</div>
-                        <div className={classNames(styles.text, this.gameOver && styles.active)}>Game over</div>
+                        <div className={clsx(styles.text, this.gameOver && styles.active)}>Game over</div>
                     </div>
                 </div>
             </div>

src/games/ai-story/components/header/header.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useContext, useEffect, useMemo, useState } from "preact/hooks";
+import { useCallback, useContext, useMemo } from "preact/hooks";
 import { useBool } from "@common/hooks/useBool";
 import { Modal } from "@common/components/modal/Modal";
 

src/games/ai-story/components/message/message.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useContext, useEffect, useMemo, useRef, useState } from "preact/hooks";
+import { useCallback, useContext, useMemo, useRef, useState } from "preact/hooks";
 import { MessageTools, type IMessage } from "../../tools/messages";
 import { StateContext } from "../../contexts/state";
 import { DOMTools } from "../../tools/dom";

src/games/ai-story/index.tsx

@@ -1,10 +1,10 @@
 import { render } from "preact";
 import { StateContextProvider } from "./contexts/state";
+import { LLMContextProvider } from "./contexts/llm";
 import { App } from "./components/app";
 
 import './assets/style.css';
 import './assets/emoji.css';
-import { LLMContextProvider } from "./contexts/llm";
 
 export default function main() {
     render(

src/games/ai-story/tools/connection.ts

@@ -64,7 +64,7 @@ const DEFAULT_GENERATION_SETTINGS = {
     rep_pen_slope: 0.7,
     top_k: 100,
     top_p: 0.92,
-    banned_tokens: ['anticipat'],
+    banned_tokens: [] as string[],
     max_length: 300,
     trim_stop: true,
     stop_sequence: ['[INST]', '[/INST]', '</s>', '<|'],
@@ -393,4 +393,4 @@ export namespace Connection {
 
         return approximateTokens(prompt);
     }
-}
+}

src/games/ai-story/tools/huggingface.ts

@@ -106,6 +106,7 @@ export namespace Huggingface {
         }).sort((a, b) => b.downloads - a.downloads);
 
         let tokenizerConfig: TokenizerConfig | null = null;
+        let foundName = '';
 
         for (const model of models) {
             const { config, name } = model;
@@ -114,16 +115,18 @@ export namespace Huggingface {
 
             if (hasField(config, 'tokenizer_config') && isTokenizerConfig(config.tokenizer_config)) {
                 tokenizerConfig = config.tokenizer_config;
+                foundName = name;
                 break;
             }
 
             try {
                 console.log(`[huggingface] searching config in '${name}/tokenizer_config.json'`);
                 const fileResponse = await hub.downloadFile({ repo: name, path: 'tokenizer_config.json' });
-                if (fileResponse?.ok) {
+                if (fileResponse) {
                     const maybeConfig = await fileResponse.json();
                     if (isTokenizerConfig(maybeConfig)) {
                         tokenizerConfig = maybeConfig;
+                        foundName = `${name}/tokenizer_config.json`;
                         break;
                     }
                 }
@@ -138,8 +141,8 @@ export namespace Huggingface {
                         try {
                             console.log(`[huggingface] searching config in '${model.name}/${file.path}'`);
                             const fileInfo = await hub.fileDownloadInfo({ repo: model.name, path: file.path });
-                            if (fileInfo?.downloadLink) {
+                            if (fileInfo?.url) {
-                                const { metadata } = await gguf(fileInfo.downloadLink);
+                                const { metadata } = await gguf(fileInfo.url);
                                 if ('tokenizer.chat_template' in metadata) {
                                     const chat_template = metadata['tokenizer.chat_template'];
                                     const tokens = metadata['tokenizer.ggml.tokens'];
@@ -154,6 +157,7 @@ export namespace Huggingface {
 
                                     if (isTokenizerConfig(maybeConfig)) {
                                         tokenizerConfig = maybeConfig;
+                                        foundName = `${model.name}/${file.path}`;
                                         break;
                                     }
                                 } else if ('tokenizer.ggml.model' in metadata) {
@@ -172,7 +176,7 @@ export namespace Huggingface {
         }
 
         if (tokenizerConfig) {
-            console.log(`[huggingface] found config for '${modelName}'`);
+            console.log(`[huggingface] found config for '${modelName}' in '${foundName}'`);
             return {
                 chat_template: tokenizerConfig.chat_template,
                 eos_token: tokenizerConfig.eos_token,
@@ -265,7 +269,10 @@ export namespace Huggingface {
     export const findTokenizer = async (modelName: string): Promise<PreTrainedTokenizer | null> => {
         modelName = normalizeModel(modelName);
 
+        if (!modelName) return null;
+
         let tokenizer = tokenizerCache.get(modelName) ?? null;
+        let foundName = '';
 
         if (tokenizer) {
             return tokenizer;
@@ -287,15 +294,15 @@ export namespace Huggingface {
                 try {
                     console.log(`[huggingface] searching tokenizer in '${name}'`);
                     tokenizer = await AutoTokenizer.from_pretrained(name);
+                    foundName = name;
                     break;
                 } catch { }
             }
         }
 
-        tokenizerCache.set(modelName, tokenizer);
-
         if (tokenizer) {
-            console.log(`[huggingface] found tokenizer for '${modelName}'`);
+            tokenizerCache.set(modelName, tokenizer);
+            console.log(`[huggingface] found tokenizer for '${modelName}' in '${foundName}'`);
         } else {
             console.log(`[huggingface] not found tokenizer for '${modelName}'`);
         }

src/games/binario/ui.tsx

@@ -1,5 +1,5 @@
 import React, { render } from 'preact';
-import cn from 'classnames';
+import cn from 'clsx';
 import { TileType } from './world';
 
 import "@common/assets/vga.font.css"