mglambda
diff --git a/‎convert_hf_to_gguf.py
Copy file name to clipboardExpand all lines: convert_hf_to_gguf.py
+4-2Lines changed: 4 additions & 2 deletions b/‎convert_hf_to_gguf.py
Copy file name to clipboardExpand all lines: convert_hf_to_gguf.py
+4-2Lines changed: 4 additions & 2 deletions
@@ -529,6 +529,8 @@ def get_vocab_base(self) -> tuple[list[str], list[int], str]:
         reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()}
         added_vocab = tokenizer.get_added_vocab()
 
+        added_tokens_decoder = tokenizer.added_tokens_decoder
+
         for i in range(vocab_size):
             if i not in reverse_vocab:
                 tokens.append(f"[PAD{i}]")
@@ -538,13 +540,13 @@ def get_vocab_base(self) -> tuple[list[str], list[int], str]:
                 if token in added_vocab:
                     # The tokenizer in llama.cpp assumes the CONTROL and USER_DEFINED tokens are pre-normalized.
                     # To avoid unexpected issues - we make sure to normalize non-normalized tokens
-                    if not tokenizer.added_tokens_decoder[i].normalized:
+                    if not added_tokens_decoder[i].normalized:
                         previous_token = token
                         token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))
                         if previous_token != token:
                             logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer")
 
-                    if tokenizer.added_tokens_decoder[i].special or self.does_token_look_special(token):
+                    if added_tokens_decoder[i].special or self.does_token_look_special(token):
                         toktypes.append(gguf.TokenType.CONTROL)
                     else:
                         # NOTE: this was added for Gemma.