Update jinja template lib
This commit is contained in:
parent
472037e3ca
commit
0711d3b89a
35
bun.lock
35
bun.lock
|
|
@ -1,16 +1,17 @@
|
|||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 0,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "binario",
|
||||
"dependencies": {
|
||||
"@huggingface/gguf": "0.1.12",
|
||||
"@huggingface/hub": "0.19.0",
|
||||
"@huggingface/jinja": "0.3.1",
|
||||
"@huggingface/gguf": "0.3.4",
|
||||
"@huggingface/hub": "2.9.0",
|
||||
"@huggingface/jinja": "0.5.5",
|
||||
"@huggingface/transformers": "3.0.2",
|
||||
"@inquirer/select": "2.3.10",
|
||||
"ace-builds": "1.36.3",
|
||||
"classnames": "2.5.1",
|
||||
"clsx": "2.1.1",
|
||||
"delay": "6.0.0",
|
||||
"preact": "10.22.0",
|
||||
},
|
||||
|
|
@ -28,15 +29,15 @@
|
|||
},
|
||||
},
|
||||
"packages": {
|
||||
"@emnapi/runtime": ["@emnapi/runtime@1.3.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw=="],
|
||||
"@emnapi/runtime": ["@emnapi/runtime@1.8.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg=="],
|
||||
|
||||
"@huggingface/gguf": ["@huggingface/gguf@0.1.12", "", {}, "sha512-m+u/ms28wE74v2VVCTncfI/KB2v897MRMOoRuYSU62P85fJ6/B2exMlHCNyAXkgDLeXBWDivXl4gPq+XbHmkaA=="],
|
||||
"@huggingface/gguf": ["@huggingface/gguf@0.3.4", "", { "dependencies": { "@huggingface/tasks": "^0.19.85" }, "bin": { "gguf-view": "dist/cli.js" } }, "sha512-+qAgKfIXF75oonWfLzN4X5kc2gUvmbbiBXRZzZ99wQ4pe7RqpaNdByzRb9/BIJXFQB1MJiXdbEXFFWeeISiIZw=="],
|
||||
|
||||
"@huggingface/hub": ["@huggingface/hub@0.19.0", "", { "dependencies": { "@huggingface/tasks": "^0.12.28" } }, "sha512-5f0POHxsQzi1RrtGmk5I+PuSXQnWx4c7WU6+JofZcdrb5mT5frV01MpGS41HRPwoQm2ZWjBfzkRpAnqBNCw2Hg=="],
|
||||
"@huggingface/hub": ["@huggingface/hub@2.9.0", "", { "dependencies": { "@huggingface/tasks": "^0.19.84" }, "optionalDependencies": { "cli-progress": "^3.12.0" }, "bin": { "hfjs": "dist/cli.js" } }, "sha512-g0UKVcPq0ULnakTB6ZGPNTHBVj9nHgsbukbI/gCp1a7uAp8/Jp8fxrKbNqJgeMiLLJcuQGd/4HXJLUYhjd5xxw=="],
|
||||
|
||||
"@huggingface/jinja": ["@huggingface/jinja@0.3.1", "", {}, "sha512-SbcBWUKDQ76lzlVYOloscUk0SJjuL1LcbZsfQv/Bxxc7dwJMYuS+DAQ+HhVw6ZkTFXArejaX5HQRuCuleYwYdA=="],
|
||||
"@huggingface/jinja": ["@huggingface/jinja@0.5.5", "", {}, "sha512-xRlzazC+QZwr6z4ixEqYHo9fgwhTZ3xNSdljlKfUFGZSdlvt166DljRELFUfFytlYOYvo3vTisA/AFOuOAzFQQ=="],
|
||||
|
||||
"@huggingface/tasks": ["@huggingface/tasks@0.12.28", "", { "dependencies": { "@huggingface/gguf": "^0.1.12" } }, "sha512-kjc6PBhwo6+UmdelcdLku6Jj18bRDXlfRUweAaCSrXrX44enyNtm/L+Z8HZO1mqOAdRBVTz+MK2yWIwcb+8drg=="],
|
||||
"@huggingface/tasks": ["@huggingface/tasks@0.19.85", "", {}, "sha512-4s5+GV257GRmT7Hrt190KlGsuq53IwFtCBj+J+bvoeIpn119589wY7WY/LS5VJMLwu7pUQnouLmMKeEOKO4BSQ=="],
|
||||
|
||||
"@huggingface/transformers": ["@huggingface/transformers@3.0.2", "", { "dependencies": { "@huggingface/jinja": "^0.3.0", "onnxruntime-node": "1.19.2", "onnxruntime-web": "1.21.0-dev.20241024-d9ca84ef96", "sharp": "^0.33.5" } }, "sha512-lTyS81eQazMea5UCehDGFMfdcNRZyei7XQLH5X6j4AhA/18Ka0+5qPgMxUxuZLU4xkv60aY2KNz9Yzthv6WVJg=="],
|
||||
|
||||
|
|
@ -156,14 +157,16 @@
|
|||
|
||||
"chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="],
|
||||
|
||||
"classnames": ["classnames@2.5.1", "", {}, "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow=="],
|
||||
|
||||
"clean-css": ["clean-css@4.2.4", "", { "dependencies": { "source-map": "~0.6.0" } }, "sha512-EJUDT7nDVFDvaQgAo2G/PJvxmp1o/c6iXLbswsBbUFXi1Nr+AjA2cKmfbKDMjMvzEe75g3P6JkaDDAKk96A85A=="],
|
||||
|
||||
"cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="],
|
||||
|
||||
"cli-spinners": ["cli-spinners@2.9.2", "", {}, "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg=="],
|
||||
|
||||
"cli-width": ["cli-width@4.1.0", "", {}, "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ=="],
|
||||
|
||||
"clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="],
|
||||
|
||||
"color": ["color@4.2.3", "", { "dependencies": { "color-convert": "^2.0.1", "color-string": "^1.9.0" } }, "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A=="],
|
||||
|
||||
"color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="],
|
||||
|
|
@ -182,7 +185,7 @@
|
|||
|
||||
"delay": ["delay@6.0.0", "", {}, "sha512-2NJozoOHQ4NuZuVIr5CWd0iiLVIRSDepakaovIN+9eIDHEhdCAEvSy2cuf1DCrPPQLvHmbqTHODlhHg8UCy4zw=="],
|
||||
|
||||
"detect-libc": ["detect-libc@2.0.3", "", {}, "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw=="],
|
||||
"detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
|
||||
|
||||
"eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="],
|
||||
|
||||
|
|
@ -202,7 +205,7 @@
|
|||
|
||||
"html-minifier": ["html-minifier@4.0.0", "", { "dependencies": { "camel-case": "^3.0.0", "clean-css": "^4.2.1", "commander": "^2.19.0", "he": "^1.2.0", "param-case": "^2.1.1", "relateurl": "^0.2.7", "uglify-js": "^3.5.1" }, "bin": { "html-minifier": "./cli.js" } }, "sha512-aoGxanpFPLg7MkIl/DDFYtb0iWz7jMFGqFhvEDZga6/4QTjneiD8I/NXL1x5aaoCp7FSIT6h/OhykDdPsbtMig=="],
|
||||
|
||||
"is-arrayish": ["is-arrayish@0.3.2", "", {}, "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="],
|
||||
"is-arrayish": ["is-arrayish@0.3.4", "", {}, "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA=="],
|
||||
|
||||
"is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="],
|
||||
|
||||
|
|
@ -254,7 +257,7 @@
|
|||
|
||||
"rxjs": ["rxjs@7.8.1", "", { "dependencies": { "tslib": "^2.1.0" } }, "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg=="],
|
||||
|
||||
"semver": ["semver@7.6.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A=="],
|
||||
"semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="],
|
||||
|
||||
"sharp": ["sharp@0.33.5", "", { "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", "semver": "^7.6.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.33.5", "@img/sharp-darwin-x64": "0.33.5", "@img/sharp-libvips-darwin-arm64": "1.0.4", "@img/sharp-libvips-darwin-x64": "1.0.4", "@img/sharp-libvips-linux-arm": "1.0.5", "@img/sharp-libvips-linux-arm64": "1.0.4", "@img/sharp-libvips-linux-s390x": "1.0.4", "@img/sharp-libvips-linux-x64": "1.0.4", "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", "@img/sharp-libvips-linuxmusl-x64": "1.0.4", "@img/sharp-linux-arm": "0.33.5", "@img/sharp-linux-arm64": "0.33.5", "@img/sharp-linux-s390x": "0.33.5", "@img/sharp-linux-x64": "0.33.5", "@img/sharp-linuxmusl-arm64": "0.33.5", "@img/sharp-linuxmusl-x64": "0.33.5", "@img/sharp-wasm32": "0.33.5", "@img/sharp-win32-ia32": "0.33.5", "@img/sharp-win32-x64": "0.33.5" } }, "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw=="],
|
||||
|
||||
|
|
@ -264,7 +267,7 @@
|
|||
|
||||
"signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="],
|
||||
|
||||
"simple-swizzle": ["simple-swizzle@0.2.2", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg=="],
|
||||
"simple-swizzle": ["simple-swizzle@0.2.4", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw=="],
|
||||
|
||||
"source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
|
||||
|
||||
|
|
@ -300,6 +303,8 @@
|
|||
|
||||
"yoctocolors-cjs": ["yoctocolors-cjs@2.1.2", "", {}, "sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA=="],
|
||||
|
||||
"@huggingface/transformers/@huggingface/jinja": ["@huggingface/jinja@0.3.1", "", {}, "sha512-SbcBWUKDQ76lzlVYOloscUk0SJjuL1LcbZsfQv/Bxxc7dwJMYuS+DAQ+HhVw6ZkTFXArejaX5HQRuCuleYwYdA=="],
|
||||
|
||||
"@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="],
|
||||
|
||||
"@isaacs/cliui/strip-ansi": ["strip-ansi@7.1.0", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ=="],
|
||||
|
|
|
|||
|
|
@ -8,13 +8,13 @@
|
|||
"bake": "bun build/build.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@huggingface/gguf": "0.1.12",
|
||||
"@huggingface/hub": "0.19.0",
|
||||
"@huggingface/jinja": "0.3.1",
|
||||
"@huggingface/gguf": "0.3.4",
|
||||
"@huggingface/hub": "2.9.0",
|
||||
"@huggingface/jinja": "0.5.5",
|
||||
"@huggingface/transformers": "3.0.2",
|
||||
"@inquirer/select": "2.3.10",
|
||||
"ace-builds": "1.36.3",
|
||||
"classnames": "2.5.1",
|
||||
"clsx": "2.1.1",
|
||||
"delay": "6.0.0",
|
||||
"preact": "10.22.0"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { render } from "preact";
|
||||
import type { ReactElement } from "preact/compat";
|
||||
import { clamp, range } from "@common/utils";
|
||||
import classNames from "classnames";
|
||||
import clsx from "clsx";
|
||||
|
||||
import styles from './assets/brick.module.css';
|
||||
import "./assets/lcd.font.css";
|
||||
|
|
@ -202,7 +202,7 @@ export class BrickDisplay {
|
|||
{range(FIELD_WIDTH * FIELD_HEIGHT).map(i => (
|
||||
<div
|
||||
key={i}
|
||||
class={classNames(styles.pixel, {
|
||||
class={clsx(styles.pixel, {
|
||||
[styles.active]: this.#field[i],
|
||||
})}
|
||||
/>
|
||||
|
|
@ -214,7 +214,7 @@ export class BrickDisplay {
|
|||
{range(MINI_FIELD_WIDTH * MINI_FIELD_HEIGHT).map(i => (
|
||||
<div
|
||||
key={i}
|
||||
class={classNames(styles.pixel, {
|
||||
class={clsx(styles.pixel, {
|
||||
[styles.active]: this.#miniField[i],
|
||||
})}
|
||||
/>
|
||||
|
|
@ -234,8 +234,8 @@ export class BrickDisplay {
|
|||
{this.helpText}
|
||||
</div>
|
||||
<div className={styles.footer}>
|
||||
<div className={classNames(styles.text, this.pause && styles.active)}>Pause</div>
|
||||
<div className={classNames(styles.text, this.gameOver && styles.active)}>Game over</div>
|
||||
<div className={clsx(styles.text, this.pause && styles.active)}>Pause</div>
|
||||
<div className={clsx(styles.text, this.gameOver && styles.active)}>Game over</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { useCallback, useContext, useEffect, useMemo, useState } from "preact/hooks";
|
||||
import { useCallback, useContext, useMemo } from "preact/hooks";
|
||||
import { useBool } from "@common/hooks/useBool";
|
||||
import { Modal } from "@common/components/modal/Modal";
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { useCallback, useContext, useEffect, useMemo, useRef, useState } from "preact/hooks";
|
||||
import { useCallback, useContext, useMemo, useRef, useState } from "preact/hooks";
|
||||
import { MessageTools, type IMessage } from "../../tools/messages";
|
||||
import { StateContext } from "../../contexts/state";
|
||||
import { DOMTools } from "../../tools/dom";
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import { render } from "preact";
|
||||
import { StateContextProvider } from "./contexts/state";
|
||||
import { LLMContextProvider } from "./contexts/llm";
|
||||
import { App } from "./components/app";
|
||||
|
||||
import './assets/style.css';
|
||||
import './assets/emoji.css';
|
||||
import { LLMContextProvider } from "./contexts/llm";
|
||||
|
||||
export default function main() {
|
||||
render(
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ const DEFAULT_GENERATION_SETTINGS = {
|
|||
rep_pen_slope: 0.7,
|
||||
top_k: 100,
|
||||
top_p: 0.92,
|
||||
banned_tokens: ['anticipat'],
|
||||
banned_tokens: [] as string[],
|
||||
max_length: 300,
|
||||
trim_stop: true,
|
||||
stop_sequence: ['[INST]', '[/INST]', '</s>', '<|'],
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ export namespace Huggingface {
|
|||
}).sort((a, b) => b.downloads - a.downloads);
|
||||
|
||||
let tokenizerConfig: TokenizerConfig | null = null;
|
||||
let foundName = '';
|
||||
|
||||
for (const model of models) {
|
||||
const { config, name } = model;
|
||||
|
|
@ -114,16 +115,18 @@ export namespace Huggingface {
|
|||
|
||||
if (hasField(config, 'tokenizer_config') && isTokenizerConfig(config.tokenizer_config)) {
|
||||
tokenizerConfig = config.tokenizer_config;
|
||||
foundName = name;
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
console.log(`[huggingface] searching config in '${name}/tokenizer_config.json'`);
|
||||
const fileResponse = await hub.downloadFile({ repo: name, path: 'tokenizer_config.json' });
|
||||
if (fileResponse?.ok) {
|
||||
if (fileResponse) {
|
||||
const maybeConfig = await fileResponse.json();
|
||||
if (isTokenizerConfig(maybeConfig)) {
|
||||
tokenizerConfig = maybeConfig;
|
||||
foundName = `${name}/tokenizer_config.json`;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -138,8 +141,8 @@ export namespace Huggingface {
|
|||
try {
|
||||
console.log(`[huggingface] searching config in '${model.name}/${file.path}'`);
|
||||
const fileInfo = await hub.fileDownloadInfo({ repo: model.name, path: file.path });
|
||||
if (fileInfo?.downloadLink) {
|
||||
const { metadata } = await gguf(fileInfo.downloadLink);
|
||||
if (fileInfo?.url) {
|
||||
const { metadata } = await gguf(fileInfo.url);
|
||||
if ('tokenizer.chat_template' in metadata) {
|
||||
const chat_template = metadata['tokenizer.chat_template'];
|
||||
const tokens = metadata['tokenizer.ggml.tokens'];
|
||||
|
|
@ -154,6 +157,7 @@ export namespace Huggingface {
|
|||
|
||||
if (isTokenizerConfig(maybeConfig)) {
|
||||
tokenizerConfig = maybeConfig;
|
||||
foundName = `${model.name}/${file.path}`;
|
||||
break;
|
||||
}
|
||||
} else if ('tokenizer.ggml.model' in metadata) {
|
||||
|
|
@ -172,7 +176,7 @@ export namespace Huggingface {
|
|||
}
|
||||
|
||||
if (tokenizerConfig) {
|
||||
console.log(`[huggingface] found config for '${modelName}'`);
|
||||
console.log(`[huggingface] found config for '${modelName}' in '${foundName}'`);
|
||||
return {
|
||||
chat_template: tokenizerConfig.chat_template,
|
||||
eos_token: tokenizerConfig.eos_token,
|
||||
|
|
@ -265,7 +269,10 @@ export namespace Huggingface {
|
|||
export const findTokenizer = async (modelName: string): Promise<PreTrainedTokenizer | null> => {
|
||||
modelName = normalizeModel(modelName);
|
||||
|
||||
if (!modelName) return null;
|
||||
|
||||
let tokenizer = tokenizerCache.get(modelName) ?? null;
|
||||
let foundName = '';
|
||||
|
||||
if (tokenizer) {
|
||||
return tokenizer;
|
||||
|
|
@ -287,15 +294,15 @@ export namespace Huggingface {
|
|||
try {
|
||||
console.log(`[huggingface] searching tokenizer in '${name}'`);
|
||||
tokenizer = await AutoTokenizer.from_pretrained(name);
|
||||
foundName = name;
|
||||
break;
|
||||
} catch { }
|
||||
}
|
||||
}
|
||||
|
||||
tokenizerCache.set(modelName, tokenizer);
|
||||
|
||||
if (tokenizer) {
|
||||
console.log(`[huggingface] found tokenizer for '${modelName}'`);
|
||||
tokenizerCache.set(modelName, tokenizer);
|
||||
console.log(`[huggingface] found tokenizer for '${modelName}' in '${foundName}'`);
|
||||
} else {
|
||||
console.log(`[huggingface] not found tokenizer for '${modelName}'`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import React, { render } from 'preact';
|
||||
import cn from 'classnames';
|
||||
import cn from 'clsx';
|
||||
import { TileType } from './world';
|
||||
|
||||
import "@common/assets/vga.font.css"
|
||||
|
|
|
|||
Loading…
Reference in New Issue