From 1bfbdb134e4b983f7cbbde252d004483e31206a2 Mon Sep 17 00:00:00 2001 From: o7si <32285332+o7si@users.noreply.github.com> Date: Thu, 11 Jun 2026 15:37:23 +0800 Subject: [PATCH] vocab : adopt leading TemplateProcessing special token as BOS (#24428) --- gguf-py/gguf/vocab.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index f8d3b3e740..d93b94f2d7 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -256,6 +256,11 @@ class SpecialVocab: if special_first := tmpl_single[0].get('SpecialToken', {}).get('id'): if not tokenizer_config: special_bos = special_first + elif special_first not in (special_bos, special_cls): + if not special_bos: + tokenizer_config['bos_token'] = special_bos = special_first + if not special_cls: + tokenizer_config['cls_token'] = special_cls = special_first self.add_special_token['bos'] = True if special_first in (special_bos, special_cls) else False if special_first not in (special_bos, special_cls): logger.warning(f'Unknown leading special token {special_first!r} in TemplateProcessing')