pabloid
/
tsgames
1
0
Fork 0
Code

Optimize templates

This commit is contained in:
Pabloader 2026-02-17 14:23:00 +00:00
parent ce67d07269
commit adc55d3001
1 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/games/ai-story/tools/huggingface.ts
View File

@ -106,7 +106,7 @@ export namespace Huggingface {
console.log(`[huggingface] searching config in '${name}/tokenizer_config.json'`); console.log(`[huggingface] searching config in '${name}/tokenizer_config.json'`);
const fileResponse = await hub.downloadFile({ const fileResponse = await hub.downloadFile({
repo: name, repo: name,
path: 'tokenizer_config.json', path: 'tokenizer_config.json',
}); });
if (fileResponse) { if (fileResponse) {
const maybeConfig = JSON.parse(await fileResponse.text()); const maybeConfig = JSON.parse(await fileResponse.text());
@ -236,13 +236,17 @@ export namespace Huggingface {
if (config?.chat_template?.trim()) { if (config?.chat_template?.trim()) {
template = config.chat_template.trim() template = config.chat_template.trim()
.replace(/raise_exception\(('[^')]+'|"[^")]+")\)/g, `''`)
.replaceAll('eos_token', `'${config.eos_token ?? ''}'`) .replaceAll('eos_token', `'${config.eos_token ?? ''}'`)
.replaceAll('bos_token', `''`); .replaceAll('bos_token', `''`)
.replace(/\{\{ ?(''|"") ?\}\}/g, '')
if (config.bos_token) { .replace(/\n'/g, `\\n'`)
template = template .replace(/\n"/g, `\\n"`)
.replace(/\{\{ ?(''|"") ?\}\}/g, ''); .replace(/'\s*\+\s*'/g, '')
} .replace(/"\s*\+\s*"/g, '')
.replace(/\{%\s*else\s*%\}\{%\s*endif\s*%\}/gi, '{% endif %}')
.replace(/\{%\s*elif[^}]+%\}\{%\s*endif\s*%\}/gi, '{% endif %}')
.replace(/\{%\s*if[^}]+%\}\{%\s*endif\s*%\}/gi, '');
} }
} }
@ -308,7 +312,6 @@ export namespace Huggingface {
try { try {
let template = compiledTemplates.get(templateString); let template = compiledTemplates.get(templateString);
if (!template) { if (!template) {
templateString = templateString.replace(/raise_exception\(('[^')]+'|"[^")]+")\)/g, `''`)
template = new Template(templateString); template = new Template(templateString);
compiledTemplates.set(templateString, template); compiledTemplates.set(templateString, template);
} }