pabloid
/
tsgames
1
0
Fork 0
Code

Optimize templates

This commit is contained in:
Pabloader 2026-02-17 14:23:00 +00:00
parent ce67d07269
commit adc55d3001
1 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/games/ai-story/tools/huggingface.ts
View File

@ -106,7 +106,7 @@ export namespace Huggingface {
console.log(`[huggingface] searching config in '${name}/tokenizer_config.json'`);
const fileResponse = await hub.downloadFile({
repo: name,
path: 'tokenizer_config.json',
path: 'tokenizer_config.json',
});
if (fileResponse) {
const maybeConfig = JSON.parse(await fileResponse.text());
@ -236,13 +236,17 @@ export namespace Huggingface {
if (config?.chat_template?.trim()) {
template = config.chat_template.trim()
.replace(/raise_exception\(('[^')]+'|"[^")]+")\)/g, `''`)
.replaceAll('eos_token', `'${config.eos_token ?? ''}'`)
.replaceAll('bos_token', `''`);
if (config.bos_token) {
template = template
.replace(/\{\{ ?(''|"") ?\}\}/g, '');
}
.replaceAll('bos_token', `''`)
.replace(/\{\{ ?(''|"") ?\}\}/g, '')
.replace(/\n'/g, `\\n'`)
.replace(/\n"/g, `\\n"`)
.replace(/'\s*\+\s*'/g, '')
.replace(/"\s*\+\s*"/g, '')
.replace(/\{%\s*else\s*%\}\{%\s*endif\s*%\}/gi, '{% endif %}')
.replace(/\{%\s*elif[^}]+%\}\{%\s*endif\s*%\}/gi, '{% endif %}')
.replace(/\{%\s*if[^}]+%\}\{%\s*endif\s*%\}/gi, '');
}
}
@ -308,7 +312,6 @@ export namespace Huggingface {
try {
let template = compiledTemplates.get(templateString);
if (!template) {
templateString = templateString.replace(/raise_exception\(('[^')]+'|"[^")]+")\)/g, `''`)
template = new Template(templateString);
compiledTemplates.set(templateString, template);
}