Xakep56
diff --git a/‎llama_cpp/llama.py
Copy file name to clipboardExpand all lines: llama_cpp/llama.py
+1-1Lines changed: 1 addition & 1 deletion b/‎llama_cpp/llama.py
Copy file name to clipboardExpand all lines: llama_cpp/llama.py
+1-1Lines changed: 1 addition & 1 deletion
diff --git a/‎tests/test_llama.py
Copy file name to clipboardExpand all lines: tests/test_llama.py
+3-3Lines changed: 3 additions & 3 deletions b/‎tests/test_llama.py
Copy file name to clipboardExpand all lines: tests/test_llama.py
+3-3Lines changed: 3 additions & 3 deletions
diff --git a/‎vendor/llama.cpp
Copy file name to clipboard b/‎vendor/llama.cpp
Copy file name to clipboard
@@ -455,7 +455,7 @@ def detokenize(self, tokens: List[int]) -> bytes:
             output += bytes(buffer[:n])
         # NOTE: Llama1 models automatically added a space at the start of the prompt
         # this line removes a leading space if the first token is a beginning of sentence token
-        return output[1:] if len(tokens) > 0 and tokens[0] == self.token_bos() else output
+        return output
 
     def set_cache(self, cache: Optional[BaseLlamaCache]):
         """Set the cache.
 
@@ -14,16 +14,16 @@ def test_llama_cpp_tokenization():
 
     tokens = llama.tokenize(text)
     assert tokens[0] == llama.token_bos()
-    assert tokens == [1, 15043, 2787]
+    assert tokens == [1, 10994, 2787]
     detokenized = llama.detokenize(tokens)
     assert detokenized == text
 
     tokens = llama.tokenize(text, add_bos=False)
     assert tokens[0] != llama.token_bos()
-    assert tokens == [15043, 2787]
+    assert tokens == [10994, 2787]
 
     detokenized = llama.detokenize(tokens)
-    assert detokenized != text
+    assert detokenized == text
 
 
 @pytest.mark.skip(reason="bug in tokenization where leading space is always inserted even if not after eos")