From 0e70984fb69d621c191913bf870b7d9201bcc3d5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 2 Mar 2024 22:20:04 -0500 Subject: [PATCH 001/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 36 ++++++++++++++++++++++++++++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 15932561c6..88ba41ca80 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -148,6 +148,12 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa ctypes.c_bool, ctypes.c_void_p, ctypes.c_bool, ctypes.c_void_p ) +# // Abort callback +# // If not NULL, called before ggml computation +# // If it returns true, the computation is aborted +# typedef bool (*ggml_abort_callback)(void * data); +ggml_abort_callback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_void_p) + # llama.h bindings _lib.llama_max_devices.argtypes = [] @@ -560,10 +566,16 @@ class llama_model_params(ctypes.Structure): # enum ggml_type type_v; // data type for V cache # // Keep the booleans together to avoid misalignment during copy-by-value. -# bool logits_all; // the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) +# bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) # bool embedding; // embedding mode only # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU # bool do_pooling; // whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) + +# // Abort callback +# // if it returns true, execution of llama_decode() will be aborted +# // currently works only with CPU execution +# ggml_abort_callback abort_callback; +# void * abort_callback_data; # }; class llama_context_params(ctypes.Structure): """Parameters for llama_context @@ -591,6 +603,8 @@ class llama_context_params(ctypes.Structure): embedding (bool): embedding mode only offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU do_pooling (bool): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) + abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted + abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback """ _fields_ = [ @@ -616,6 +630,8 @@ class llama_context_params(ctypes.Structure): ("embedding", ctypes.c_bool), ("offload_kqv", ctypes.c_bool), ("do_pooling", ctypes.c_bool), + ("abort_callback", ggml_abort_callback), + ("abort_callback_data", ctypes.c_void_p), ] @@ -1703,8 +1719,24 @@ def llama_set_n_threads( """ ... +# // Set abort callback +# LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); +@ctypes_function( + "llama_set_abort_callback", + [llama_context_p_ctypes, ggml_abort_callback, ctypes.c_void_p], + None, +) +def llama_set_abort_callback( + ctx: llama_context_p, + abort_callback: Callable[[ctypes.c_void_p], None], + abort_callback_data: ctypes.c_void_p, + /, +): + """Set abort callback""" + ... + -# // Token logits obtained from the last call to llama_eval() +# // Token logits obtained from the last call to llama_decode() # // The logits for the last token are stored in the last row # // Logits for which llama_batch.logits[i] == 0 are undefined # // Rows: n_tokens provided with llama_batch diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c2224f003b..9731134296 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c2224f003bf9cf558b1a3c57033563e11a4de9a5 +Subproject commit 9731134296af3a6839cd682e51d9c2109a871de5 From 663659f7301963e0a3e98662e14668a6632c6295 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 3 Mar 2024 04:20:41 +0100 Subject: [PATCH 002/561] docs: fix small typo in README: 'model know how' -> 'model knows how' (#1244) Co-authored-by: Andrei --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1d296e905f..3323f3899b 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,7 @@ By default [`from_pretrained`](https://llama-cpp-python.readthedocs.io/en/latest The high-level API also provides a simple interface for chat completion. -Chat completion requires that the model know how to format the messages into a single prompt. +Chat completion requires that the model knows how to format the messages into a single prompt. The `Llama` class does this using pre-registered chat formats (ie. `chatml`, `llama-2`, `gemma`, etc) or by providing a custom chat handler object. The model will will format the messages into a single prompt using the following order of precedence: From 13177aae0f674100f7a7d23c54fc9f14012bf6a2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 2 Mar 2024 22:46:40 -0500 Subject: [PATCH 003/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 375c6efa75..e16a6dfcce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.55] + +- feat: Update llama.cpp to ggerganov/9731134296af3a6839cd682e51d9c2109a871de5 +- docs: fix small typo in README: 'model know how' -> 'model knows how' by @boegel in #1244 + ## [0.2.54] - feat: Update llama.cpp to ggerganov/llama.cpp@cb49e0f8c906e5da49e9f6d64a57742a9a241c6a diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index a9a82220d6..519ab51b75 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.54" \ No newline at end of file +__version__ = "0.2.55" \ No newline at end of file From 87a6e5797eb7b0cd63ad27c528fb950c80c84ad8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 3 Mar 2024 11:27:04 -0500 Subject: [PATCH 004/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 14 +++++++++----- vendor/llama.cpp | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 88ba41ca80..08adfe2054 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -320,10 +320,12 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN # enum llama_pooling_type { +# LLAMA_POOLING_TYPE_UNSPECIFIED = -1, # LLAMA_POOLING_TYPE_NONE = 0, # LLAMA_POOLING_TYPE_MEAN = 1, # LLAMA_POOLING_TYPE_CLS = 2, # }; +LLAMA_POOLING_TYPE_UNSPECIFIED = -1 LLAMA_POOLING_TYPE_NONE = 0 LLAMA_POOLING_TYPE_MEAN = 1 LLAMA_POOLING_TYPE_CLS = 2 @@ -547,7 +549,10 @@ class llama_model_params(ctypes.Structure): # uint32_t n_batch; // prompt processing maximum batch size # uint32_t n_threads; // number of threads to use for generation # uint32_t n_threads_batch; // number of threads to use for batch processing -# int32_t rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` + +# enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` +# enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id +# // (ignored if no pooling layer) # // ref: https://github.com/ggerganov/llama.cpp/pull/2054 # float rope_freq_base; // RoPE base frequency, 0 = from model @@ -569,7 +574,6 @@ class llama_model_params(ctypes.Structure): # bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) # bool embedding; // embedding mode only # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU -# bool do_pooling; // whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) # // Abort callback # // if it returns true, execution of llama_decode() will be aborted @@ -587,6 +591,7 @@ class llama_context_params(ctypes.Structure): n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type` + pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) rope_freq_base (float): RoPE base frequency, 0 = from model rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model @@ -602,7 +607,6 @@ class llama_context_params(ctypes.Structure): logits_all (bool): the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) embedding (bool): embedding mode only offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU - do_pooling (bool): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback """ @@ -613,7 +617,8 @@ class llama_context_params(ctypes.Structure): ("n_batch", ctypes.c_uint32), ("n_threads", ctypes.c_uint32), ("n_threads_batch", ctypes.c_uint32), - ("rope_scaling_type", ctypes.c_int32), + ("rope_scaling_type", ctypes.c_int), + ("pooling_type", ctypes.c_int), ("rope_freq_base", ctypes.c_float), ("rope_freq_scale", ctypes.c_float), ("yarn_ext_factor", ctypes.c_float), @@ -629,7 +634,6 @@ class llama_context_params(ctypes.Structure): ("logits_all", ctypes.c_bool), ("embedding", ctypes.c_bool), ("offload_kqv", ctypes.c_bool), - ("do_pooling", ctypes.c_bool), ("abort_callback", ggml_abort_callback), ("abort_callback_data", ctypes.c_void_p), ] diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 9731134296..67be2ce101 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 9731134296af3a6839cd682e51d9c2109a871de5 +Subproject commit 67be2ce1015d070b3b2cd488bcb041eefb61de72 From 93dc56ace8e3de97f6f39a7071ff63aaf29d376f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 6 Mar 2024 01:32:00 -0500 Subject: [PATCH 005/561] Update llama.cpp --- llama_cpp/llama.py | 6 +++--- llama_cpp/llama_cpp.py | 34 ++++++++++++++++++++++++++-------- vendor/llama.cpp | 2 +- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 108a4cfa9d..7187b4a178 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -293,7 +293,7 @@ def __init__( self.context_params.logits_all = ( logits_all if draft_model is None else True ) # Must be set to True for speculative decoding - self.context_params.embedding = embedding + self.context_params.embeddings = embedding # TODO: Rename to embeddings self.context_params.offload_kqv = offload_kqv # Sampling Params @@ -787,7 +787,7 @@ def embed( n_embd = self.n_embd() n_batch = self.n_batch - if self.context_params.embedding == False: + if self.context_params.embeddings == False: raise RuntimeError( "Llama model must be created with embedding=True to call this method" ) @@ -1725,7 +1725,7 @@ def __getstate__(self): yarn_beta_slow=self.context_params.yarn_beta_slow, yarn_orig_ctx=self.context_params.yarn_orig_ctx, logits_all=self.context_params.logits_all, - embedding=self.context_params.embedding, + embedding=self.context_params.embeddings, # Sampling Params last_n_tokens_size=self.last_n_tokens_size, # LoRA Params diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 08adfe2054..92b96766c4 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -399,7 +399,7 @@ class llama_token_data_array(ctypes.Structure): # // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) # // - pos : the positions of the respective token in the sequence # // - seq_id : the sequence to which the respective token belongs -# // - logits : if zero, the logits for the respective token will not be output +# // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output # // # typedef struct llama_batch { # int32_t n_tokens; @@ -409,7 +409,7 @@ class llama_token_data_array(ctypes.Structure): # llama_pos * pos; # int32_t * n_seq_id; # llama_seq_id ** seq_id; -# int8_t * logits; +# int8_t * logits; // TODO: rename this to "output" # // NOTE: helpers for smooth API transition - can be deprecated in the future @@ -572,7 +572,7 @@ class llama_model_params(ctypes.Structure): # // Keep the booleans together to avoid misalignment during copy-by-value. # bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) -# bool embedding; // embedding mode only +# bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU # // Abort callback @@ -605,7 +605,7 @@ class llama_context_params(ctypes.Structure): type_k (int): data type for K cache type_v (int): data type for V cache logits_all (bool): the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) - embedding (bool): embedding mode only + embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback @@ -632,7 +632,7 @@ class llama_context_params(ctypes.Structure): ("type_k", ctypes.c_int), ("type_v", ctypes.c_int), ("logits_all", ctypes.c_bool), - ("embedding", ctypes.c_bool), + ("embeddings", ctypes.c_bool), ("offload_kqv", ctypes.c_bool), ("abort_callback", ggml_abort_callback), ("abort_callback_data", ctypes.c_void_p), @@ -1774,8 +1774,8 @@ def llama_get_logits_ith( ... -# Get the embeddings for the input -# shape: [n_embd] (1-dimensional) +# // Get all output token embeddings +# // shape: [n_tokens*n_embd] (1-dimensional) # LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); @ctypes_function( "llama_get_embeddings", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float) @@ -1786,8 +1786,9 @@ def llama_get_embeddings(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float] ... -# // Get the embeddings for the ith sequence +# // Get the embeddings for the ith token # // llama_get_embeddings(ctx) + i*n_embd +# // shape: [n_embd] (1-dimensional) # LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); @ctypes_function( "llama_get_embeddings_ith", @@ -1802,6 +1803,23 @@ def llama_get_embeddings_ith( ... +# // Get the embeddings for a sequence id +# // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE +# // shape: [n_embd] (1-dimensional) +# LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); +@ctypes_function( + "llama_get_embeddings_seq", + [llama_context_p_ctypes, llama_seq_id], + ctypes.POINTER(ctypes.c_float), +) +def llama_get_embeddings_seq( + ctx: llama_context_p, seq_id: Union[llama_seq_id, int], / +) -> CtypesArray[ctypes.c_float]: + """Get the embeddings for a sequence id + Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE + shape: [n_embd] (1-dimensional)""" + ... + # // # // Vocab # // diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 67be2ce101..8ced9f7e32 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 67be2ce1015d070b3b2cd488bcb041eefb61de72 +Subproject commit 8ced9f7e3225adb8501e9821ed1bbd92e3a5c7ae From 40c6b54f6880e1cbb8f6393d9097328ffd422e13 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 8 Mar 2024 20:58:50 -0500 Subject: [PATCH 006/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 16 ++++++++++++---- vendor/llama.cpp | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 92b96766c4..0176e49a81 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -429,10 +429,12 @@ class llama_batch(ctypes.Structure): The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens Attributes: + n_tokens (int): number of tokens token (ctypes.Array[llama_token]): the token ids of the input (used when embd is NULL) embd (ctypes.Array[ctypes.ctypes.c_float]): token embeddings (i.e. float vector of size n_embd) (used when token is NULL) pos (ctypes.Array[ctypes.Array[llama_pos]]): the positions of the respective token in the sequence seq_id (ctypes.Array[ctypes.Array[llama_seq_id]]): the sequence to which the respective token belongs + logits (ctypes.Array[ctypes.ctypes.c_int8]): if zero, the logits for the respective token will not be output """ _fields_ = [ @@ -547,6 +549,7 @@ class llama_model_params(ctypes.Structure): # uint32_t seed; // RNG seed, -1 for random # uint32_t n_ctx; // text context, 0 = from model # uint32_t n_batch; // prompt processing maximum batch size +# uint32_t n_parallel; // number of parallel sequences (i.e. distinct states for recurrent models) # uint32_t n_threads; // number of threads to use for generation # uint32_t n_threads_batch; // number of threads to use for batch processing @@ -588,6 +591,7 @@ class llama_context_params(ctypes.Structure): seed (int): RNG seed, -1 for random n_ctx (int): text context, 0 = from model n_batch (int): prompt processing maximum batch size + n_parallel (int): number of parallel sequences (i.e. distinct states for recurrent models) n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type` @@ -615,6 +619,7 @@ class llama_context_params(ctypes.Structure): ("seed", ctypes.c_uint32), ("n_ctx", ctypes.c_uint32), ("n_batch", ctypes.c_uint32), + ("n_parallel", ctypes.c_uint32), ("n_threads", ctypes.c_uint32), ("n_threads_batch", ctypes.c_uint32), ("rope_scaling_type", ctypes.c_int), @@ -1322,7 +1327,7 @@ def llama_kv_cache_clear(ctx: llama_context_p, /): # // seq_id < 0 : match any sequence # // p0 < 0 : [0, p1] # // p1 < 0 : [p0, inf) -# LLAMA_API void llama_kv_cache_seq_rm( +# LLAMA_API bool llama_kv_cache_seq_rm( # struct llama_context * ctx, # llama_seq_id seq_id, # llama_pos p0, @@ -1335,7 +1340,7 @@ def llama_kv_cache_clear(ctx: llama_context_p, /): llama_pos, llama_pos, ], - None, + ctypes.c_bool, ) def llama_kv_cache_seq_rm( ctx: llama_context_p, @@ -1343,7 +1348,7 @@ def llama_kv_cache_seq_rm( p0: Union[llama_pos, int], p1: Union[llama_pos, int], /, -): +) -> bool: """Removes all tokens that belong to the specified sequence and have positions in [p0, p1) seq_id < 0 : match any sequence p0 < 0 : [0, p1] @@ -1754,7 +1759,10 @@ def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]: The logits for the last token are stored in the last row Logits for which llama_batch.logits[i] == 0 are undefined Rows: n_tokens provided with llama_batch - Cols: n_vocab""" + Cols: n_vocab + + Returns: + Pointer to the logits buffer of shape (n_tokens, n_vocab)""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8ced9f7e32..c2101a2e90 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8ced9f7e3225adb8501e9821ed1bbd92e3a5c7ae +Subproject commit c2101a2e909ac7c08976d414e64e96c90ee5fa9e From 2811014bae356401856a9c0796f42e719f2e8c3c Mon Sep 17 00:00:00 2001 From: Douglas Hanley Date: Fri, 8 Mar 2024 19:59:35 -0600 Subject: [PATCH 007/561] feat: Switch embed to llama_get_embeddings_seq (#1263) * switch to llama_get_embeddings_seq * Remove duplicate definition of llama_get_embeddings_seq Co-authored-by: Andrei --------- Co-authored-by: Andrei --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 7187b4a178..aabbb7e715 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -814,7 +814,7 @@ def decode_batch(n_seq: int): # store embeddings for i in range(n_seq): - embedding: List[float] = llama_cpp.llama_get_embeddings_ith( + embedding: List[float] = llama_cpp.llama_get_embeddings_seq( self._ctx.ctx, i )[:n_embd] if normalize: From 1f3156d4f2d8c5439dbb2ad72b8c1de84703eb09 Mon Sep 17 00:00:00 2001 From: Kevin Cao Date: Fri, 8 Mar 2024 21:00:10 -0500 Subject: [PATCH 008/561] fix: Check for existence of clip model path (#1264) --- llama_cpp/llama_chat_format.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 69ed6019a7..4eb2b0291c 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1848,6 +1848,9 @@ def __init__(self, clip_model_path: str, verbose: bool = False): self.verbose = verbose self._clip_free = self._llava_cpp._libllava.clip_free # type: ignore + if not os.path.exists(clip_model_path): + raise ValueError(f"Clip model path does not exist: {clip_model_path}") + with suppress_stdout_stderr(disable=self.verbose): self.clip_ctx = self._llava_cpp.clip_model_load( self.clip_model_path.encode(), 0 From c139f8b5d50f6f416a24c0ba65983a3fb84bf2f3 Mon Sep 17 00:00:00 2001 From: Felipe Lorenz Date: Fri, 8 Mar 2024 21:09:00 -0500 Subject: [PATCH 009/561] feat: Add endpoints for tokenize, detokenize and count tokens (#1136) * Add endpoint to count tokens * Add tokenize and detokenize endpoints * Change response key to tokens for tokenize endpoint * Fix dependency bug * Cleanup * Remove example added by mistake * Move tokenize, detokenize, and count to Extras namespace. Tag existing endpoints --------- Co-authored-by: Andrei Betlen --- llama_cpp/server/app.py | 71 +++++++++++++++++++++++++++++++++++++-- llama_cpp/server/types.py | 36 ++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index ec92809860..aa6afc1122 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -41,6 +41,11 @@ CreateEmbeddingRequest, CreateChatCompletionRequest, ModelList, + TokenizeInputRequest, + TokenizeInputResponse, + TokenizeInputCountResponse, + DetokenizeInputRequest, + DetokenizeInputResponse, ) from llama_cpp.server.errors import RouteErrorHandler @@ -196,6 +201,9 @@ async def authenticate( ) +openai_v1_tag = "OpenAI V1" + + @router.post( "/v1/completions", summary="Completion", @@ -227,11 +235,13 @@ async def authenticate( }, } }, + tags=[openai_v1_tag], ) @router.post( "/v1/engines/copilot-codex/completions", include_in_schema=False, dependencies=[Depends(authenticate)], + tags=[openai_v1_tag], ) async def create_completion( request: Request, @@ -297,7 +307,10 @@ def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: @router.post( - "/v1/embeddings", summary="Embedding", dependencies=[Depends(authenticate)] + "/v1/embeddings", + summary="Embedding", + dependencies=[Depends(authenticate)], + tags=[openai_v1_tag], ) async def create_embedding( request: CreateEmbeddingRequest, @@ -339,6 +352,7 @@ async def create_embedding( }, } }, + tags=[openai_v1_tag], ) async def create_chat_completion( request: Request, @@ -391,7 +405,12 @@ def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: return iterator_or_completion -@router.get("/v1/models", summary="Models", dependencies=[Depends(authenticate)]) +@router.get( + "/v1/models", + summary="Models", + dependencies=[Depends(authenticate)], + tags=[openai_v1_tag], +) async def get_models( llama_proxy: LlamaProxy = Depends(get_llama_proxy), ) -> ModelList: @@ -407,3 +426,51 @@ async def get_models( for model_alias in llama_proxy ], } + + +extras_tag = "Extras" + + +@router.post( + "/extras/tokenize", + summary="Tokenize", + dependencies=[Depends(authenticate)], + tags=[extras_tag], +) +async def tokenize( + body: TokenizeInputRequest, + llama_proxy: LlamaProxy = Depends(get_llama_proxy), +) -> TokenizeInputResponse: + tokens = llama_proxy(body.model).tokenize(body.input.encode("utf-8"), special=True) + + return {"tokens": tokens} + + +@router.post( + "/extras/tokenize/count", + summary="Tokenize Count", + dependencies=[Depends(authenticate)], + tags=[extras_tag], +) +async def count_query_tokens( + body: TokenizeInputRequest, + llama_proxy: LlamaProxy = Depends(get_llama_proxy), +) -> TokenizeInputCountResponse: + tokens = llama_proxy(body.model).tokenize(body.input.encode("utf-8"), special=True) + + return {"count": len(tokens)} + + +@router.post( + "/extras/detokenize", + summary="Detokenize", + dependencies=[Depends(authenticate)], + tags=[extras_tag], +) +async def detokenize( + body: DetokenizeInputRequest, + llama_proxy: LlamaProxy = Depends(get_llama_proxy), +) -> DetokenizeInputResponse: + text = llama_proxy(body.model).detokenize(body.tokens).decode("utf-8") + + return {"text": text} diff --git a/llama_cpp/server/types.py b/llama_cpp/server/types.py index 9a4b81e09f..c8b2ebc6cf 100644 --- a/llama_cpp/server/types.py +++ b/llama_cpp/server/types.py @@ -264,3 +264,39 @@ class ModelData(TypedDict): class ModelList(TypedDict): object: Literal["list"] data: List[ModelData] + + +class TokenizeInputRequest(BaseModel): + model: Optional[str] = model_field + input: Optional[str] = Field(description="The input to tokenize.") + + model_config = { + "json_schema_extra": {"examples": [{"input": "How many tokens in this query?"}]} + } + + +class TokenizeInputResponse(BaseModel): + tokens: List[int] = Field(description="A list of tokens.") + + model_config = {"json_schema_extra": {"example": {"tokens": [123, 321, 222]}}} + + +class TokenizeInputCountResponse(BaseModel): + count: int = Field(description="The number of tokens in the input.") + + model_config = {"json_schema_extra": {"example": {"count": 5}}} + + +class DetokenizeInputRequest(BaseModel): + model: Optional[str] = model_field + tokens: List[int] = Field(description="A list of toekns to detokenize.") + + model_config = {"json_schema_extra": {"example": [{"tokens": [123, 321, 222]}]}} + + +class DetokenizeInputResponse(BaseModel): + text: str = Field(description="The detokenized text.") + + model_config = { + "json_schema_extra": {"example": {"text": "How many tokens in this query?"}} + } From d02a9cf16ff88ad011e2eb1ce29f4d9400f13cd1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 8 Mar 2024 21:10:53 -0500 Subject: [PATCH 010/561] Fixed json strings grammar by blacklisting character control set. Closes #1259 --- llama_cpp/llama_grammar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 6a37857b99..9cc48a93b3 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -1337,7 +1337,7 @@ def print_grammar(file: TextIO, state: parse_state) -> None: string ::= "\"" ( - [^"\\] | + [^"\\\x7F\x00-\x1F] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws @@ -1366,7 +1366,7 @@ def print_grammar(file: TextIO, state: parse_state) -> None: string ::= "\"" ( - [^"\\] | + [^"\\\x7F\x00-\x1F] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes )* "\"" ws From a7281994d87927e42d8e636295c786057e98d8fe Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 8 Mar 2024 21:14:44 -0500 Subject: [PATCH 011/561] chore: Bump version --- CHANGELOG.md | 8 ++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e16a6dfcce..90dd1e6909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.56] + +- feat: Update llama.cpp to ggerganov/llama.cpp@c2101a2e909ac7c08976d414e64e96c90ee5fa9e +- feat(server): Add endpoints for tokenize, detokenize and count tokens by @felipelo in #1136 +- feat: Switch embed to llama_get_embeddings_seq by @iamlemec in #1263 +- fix: Fixed json strings grammar by blacklisting character control set by @ExtReMLapin in d02a9cf16ff88ad011e2eb1ce29f4d9400f13cd1 +- fix: Check for existence of clip model path by @kejcao in #1264 + ## [0.2.55] - feat: Update llama.cpp to ggerganov/9731134296af3a6839cd682e51d9c2109a871de5 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 519ab51b75..fcbc7150ce 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.55" \ No newline at end of file +__version__ = "0.2.56" \ No newline at end of file From 08e910f7a7e3657cf210ab8633a6994e1bde7164 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 10 Mar 2024 23:45:05 -0400 Subject: [PATCH 012/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 11 +++++++++++ vendor/llama.cpp | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 0176e49a81..8f0ba546c3 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1728,6 +1728,17 @@ def llama_set_n_threads( """ ... + +# // Set whether to use causal attention or not +# // If set to true, the model will only attend to the past tokens +# LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); +@ctypes_function("llama_set_causal_attn", [llama_context_p_ctypes, ctypes.c_bool], None) +def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /): + """Set whether to use causal attention or not + If set to true, the model will only attend to the past tokens""" + ... + + # // Set abort callback # LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); @ctypes_function( diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c2101a2e90..3814a07392 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c2101a2e909ac7c08976d414e64e96c90ee5fa9e +Subproject commit 3814a07392d2bdc22911652bc7c2f9bdb0ce042e From dd0ee56217c60a20a192dc7f1523dba9a006bbc9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 13 Mar 2024 15:57:35 -0400 Subject: [PATCH 013/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 160 ++++++++++++++++++++++------------------- vendor/llama.cpp | 2 +- 2 files changed, 87 insertions(+), 75 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 8f0ba546c3..884a06384d 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -548,8 +548,9 @@ class llama_model_params(ctypes.Structure): # struct llama_context_params { # uint32_t seed; // RNG seed, -1 for random # uint32_t n_ctx; // text context, 0 = from model -# uint32_t n_batch; // prompt processing maximum batch size -# uint32_t n_parallel; // number of parallel sequences (i.e. distinct states for recurrent models) +# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode +# uint32_t n_ubatch; // physical maximum batch size +# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) # uint32_t n_threads; // number of threads to use for generation # uint32_t n_threads_batch; // number of threads to use for batch processing @@ -578,6 +579,7 @@ class llama_model_params(ctypes.Structure): # bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU + # // Abort callback # // if it returns true, execution of llama_decode() will be aborted # // currently works only with CPU execution @@ -590,8 +592,9 @@ class llama_context_params(ctypes.Structure): Attributes: seed (int): RNG seed, -1 for random n_ctx (int): text context, 0 = from model - n_batch (int): prompt processing maximum batch size - n_parallel (int): number of parallel sequences (i.e. distinct states for recurrent models) + n_batch (int): logical maximum batch size that can be submitted to llama_decode + n_ubatch (int): physical maximum batch size + n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type` @@ -619,7 +622,8 @@ class llama_context_params(ctypes.Structure): ("seed", ctypes.c_uint32), ("n_ctx", ctypes.c_uint32), ("n_batch", ctypes.c_uint32), - ("n_parallel", ctypes.c_uint32), + ("n_ubatch", ctypes.c_uint32), + ("n_seq_max", ctypes.c_uint32), ("n_threads", ctypes.c_uint32), ("n_threads_batch", ctypes.c_uint32), ("rope_scaling_type", ctypes.c_int), @@ -667,7 +671,7 @@ class llama_context_params(ctypes.Structure): # bool allow_requantize; // allow quantizing non-f32/f16 tensors # bool quantize_output_tensor; // quantize output.weight # bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored -# bool pure; // disable k-quant mixtures and quantize all tensors to the same type +# bool pure; // quantize all tensors to the default type # void * imatrix; // pointer to importance matrix data # } llama_model_quantize_params; class llama_model_quantize_params(ctypes.Structure): @@ -679,7 +683,7 @@ class llama_model_quantize_params(ctypes.Structure): allow_requantize (bool): allow quantizing non-f32/f16 tensors quantize_output_tensor (bool): quantize output.weight only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored - pure (bool): disable k-quant mixtures and quantize all tensors to the same type + pure (bool): quantize all tensors to the default type imatrix (ctypes.ctypes.c_void_p): pointer to importance matrix data """ @@ -860,8 +864,7 @@ def llama_backend_init(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): - ... +def llama_numa_init(numa: int, /): ... # // Call once at the end of the program - currently only used for MPI @@ -886,8 +889,7 @@ def llama_backend_free(): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: - ... +) -> Optional[llama_model_p]: ... # LLAMA_API void llama_free_model(struct llama_model * model); @@ -896,8 +898,7 @@ def llama_load_model_from_file( [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): - ... +def llama_free_model(model: llama_model_p, /): ... # LLAMA_API struct llama_context * llama_new_context_with_model( @@ -910,8 +911,7 @@ def llama_free_model(model: llama_model_p, /): ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: - ... +) -> Optional[llama_context_p]: ... # // Frees all allocated memory @@ -932,80 +932,77 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: - ... +def llama_time_us() -> int: ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: - ... +def llama_max_devices() -> int: ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: - ... +def llama_supports_mmap() -> bool: ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: - ... +def llama_supports_mlock() -> bool: ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: - ... +def llama_supports_gpu_offload() -> bool: ... # LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: - ... +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: - ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: - ... +def llama_n_batch(ctx: llama_context_p, /) -> int: ... + + +# LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); +@ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) +def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... + + +# LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); +@ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) +def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... # LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: - ... +def llama_vocab_type(model: llama_model_p, /) -> int: ... # LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: - ... +def llama_rope_type(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: - ... +def llama_n_vocab(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: - ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_embd (const struct llama_model * model); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: - ... +def llama_n_embd(model: llama_model_p, /) -> int: ... # // Get the model's RoPE frequency scaling factor @@ -1192,8 +1189,7 @@ def llama_model_apply_lora_from_file( path_base_model: Union[ctypes.c_char_p, bytes, None], n_threads: Union[ctypes.c_int32, int], /, -) -> int: - ... +) -> int: ... # // @@ -1219,7 +1215,7 @@ class llama_kv_cache_view_cell(ctypes.Structure): # // Maximum number of sequences that can exist in a cell. It's not an error # // if there are more sequences in a cell than this value, however they will # // not be visible in the view cells_sequences. -# int32_t n_max_seq; +# int32_t n_seq_max; # // Number of tokens in the cache. For example, if there are two populated # // cells, the first with 1 sequence id in it and the second with 2 sequence @@ -1240,7 +1236,7 @@ class llama_kv_cache_view_cell(ctypes.Structure): # struct llama_kv_cache_view_cell * cells; -# // The sequences for each cell. There will be n_max_seq items per cell. +# // The sequences for each cell. There will be n_seq_max items per cell. # llama_seq_id * cells_sequences; # }; class llama_kv_cache_view(ctypes.Structure): @@ -1260,14 +1256,14 @@ class llama_kv_cache_view(ctypes.Structure): # // Create an empty KV cache view. (use only for debugging purposes) -# LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_max_seq); +# LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max); @ctypes_function( "llama_kv_cache_view_init", [llama_context_p_ctypes, ctypes.c_int32], llama_kv_cache_view, ) def llama_kv_cache_view_init( - ctx: llama_context_p, n_max_seq: Union[ctypes.c_int32, int], / + ctx: llama_context_p, n_seq_max: Union[ctypes.c_int32, int], / ) -> llama_kv_cache_view: """Create an empty KV cache view. (use only for debugging purposes)""" ... @@ -1582,8 +1578,7 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: - ... +) -> int: ... # LLAMA_API bool llama_save_session_file( @@ -1607,8 +1602,7 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: - ... +) -> int: ... # // @@ -1756,6 +1750,18 @@ def llama_set_abort_callback( ... +# // Wait until all computations are finished +# // This is automatically done when using one of the functions below to obtain the computation results +# // and is not necessary to call it explicitly in most cases +# LLAMA_API void llama_synchronize(struct llama_context * ctx); +@ctypes_function("llama_synchronize", [llama_context_p_ctypes], None) +def llama_synchronize(ctx: llama_context_p, /): + """Wait until all computations are finished + This is automatically done when using one of the functions below to obtain the computation results + and is not necessary to call it explicitly in most cases""" + ... + + # // Token logits obtained from the last call to llama_decode() # // The logits for the last token are stored in the last row # // Logits for which llama_batch.logits[i] == 0 are undefined @@ -1771,7 +1777,7 @@ def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]: Logits for which llama_batch.logits[i] == 0 are undefined Rows: n_tokens provided with llama_batch Cols: n_vocab - + Returns: Pointer to the logits buffer of shape (n_tokens, n_vocab)""" ... @@ -1839,6 +1845,7 @@ def llama_get_embeddings_seq( shape: [n_embd] (1-dimensional)""" ... + # // # // Vocab # // @@ -1850,8 +1857,7 @@ def llama_get_embeddings_seq( ) def llama_token_get_text( model: llama_model_p, token: Union[llama_token, int], / -) -> bytes: - ... +) -> bytes: ... # LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); @@ -1860,8 +1866,7 @@ def llama_token_get_text( ) def llama_token_get_score( model: llama_model_p, token: Union[llama_token, int], / -) -> float: - ... +) -> float: ... # LLAMA_API enum llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token); @@ -1870,8 +1875,7 @@ def llama_token_get_score( ) def llama_token_get_type( model: llama_model_p, token: Union[llama_token, int], / -) -> int: - ... +) -> int: ... # // Special tokens @@ -1924,20 +1928,17 @@ def llama_token_prefix(model: llama_model_p) -> int: # LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: - ... +def llama_token_middle(model: llama_model_p, /) -> int: ... # LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: - ... +def llama_token_suffix(model: llama_model_p, /) -> int: ... # LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle @ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: - ... +def llama_token_eot(model: llama_model_p, /) -> int: ... # // @@ -1947,7 +1948,7 @@ def llama_token_eot(model: llama_model_p, /) -> int: # /// @details Convert the provided text into tokens. # /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. -# /// @return Returns the number of tokens on success, no more than n_max_tokens +# /// @return Returns the number of tokens on success, no more than n_tokens_max # /// @return Returns a negative number on failure - the number of tokens that would have been returned # /// @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. # /// Does not insert a leading space. @@ -1956,7 +1957,7 @@ def llama_token_eot(model: llama_model_p, /) -> int: # const char * text, # int32_t text_len, # llama_token * tokens, -# int32_t n_max_tokens, +# int32_t n_tokens_max, # bool add_bos, # bool special); @ctypes_function( @@ -1977,12 +1978,26 @@ def llama_tokenize( text: bytes, text_len: Union[ctypes.c_int, int], tokens: CtypesArray[llama_token], - n_max_tokens: Union[ctypes.c_int, int], + n_tokens_max: Union[ctypes.c_int, int], add_bos: Union[ctypes.c_bool, bool], special: Union[ctypes.c_bool, bool], /, ) -> int: - """Convert the provided text into tokens.""" + """Convert the provided text into tokens. + + Args: + model: The model to use for tokenization. + text: The text to tokenize. + text_len: The length of the text. + tokens: The tokens pointer must be large enough to hold the resulting tokens. + n_max_tokens: The maximum number of tokens to return. + add_bos: Whether to add a beginning-of-sentence token. + special: Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. + Does not insert a leading space. + + Returns: + Returns the number of tokens on success, no more than n_tokens_max + Returns a negative number on failure - the number of tokens that would have been returned""" ... @@ -2054,8 +2069,7 @@ def llama_chat_apply_template( chat: CtypesArray[llama_chat_message], n_msg: int, /, -) -> int: - ... +) -> int: ... # // @@ -2656,8 +2670,7 @@ def llama_beam_search( n_past: Union[ctypes.c_int, int], n_predict: Union[ctypes.c_int, int], /, -): - ... +): ... # Performance information @@ -2734,5 +2747,4 @@ def llama_log_set( [ctypes.c_void_p, llama_context_p_ctypes], None, ) -def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): - ... +def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 3814a07392..19885d205e 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 3814a07392d2bdc22911652bc7c2f9bdb0ce042e +Subproject commit 19885d205e768579ab090d1e99281cae58c21b54 From d318cc8b83981833fbab2314d97af21104e52d99 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 14 Mar 2024 09:17:41 -0400 Subject: [PATCH 014/561] fix: Set default pooling_type to mean, check for null pointer. --- llama_cpp/llama.py | 10 ++++++++-- llama_cpp/llama_cpp.py | 1 - 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index aabbb7e715..18db1a0b6d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -79,6 +79,7 @@ def __init__( n_threads: Optional[int] = None, n_threads_batch: Optional[int] = None, rope_scaling_type: Optional[int] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, + pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_MEAN, rope_freq_base: float = 0.0, rope_freq_scale: float = 0.0, yarn_ext_factor: float = -1.0, @@ -151,6 +152,7 @@ def __init__( n_threads: Number of threads to use for generation n_threads_batch: Number of threads to use for batch processing rope_scaling_type: RoPE scaling type, from `enum llama_rope_scaling_type`. ref: https://github.com/ggerganov/llama.cpp/pull/2054 + pooling_type: Pooling type, from `enum llama_pooling_type`. rope_freq_base: RoPE base frequency, 0 = from model rope_freq_scale: RoPE frequency scaling factor, 0 = from model yarn_ext_factor: YaRN extrapolation mix factor, negative = from model @@ -271,6 +273,7 @@ def __init__( if rope_scaling_type is not None else llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED ) + self.context_params.pooling_type = pooling_type self.context_params.rope_freq_base = ( rope_freq_base if rope_freq_base != 0.0 else 0 ) @@ -814,9 +817,12 @@ def decode_batch(n_seq: int): # store embeddings for i in range(n_seq): - embedding: List[float] = llama_cpp.llama_get_embeddings_seq( + ptr = llama_cpp.llama_get_embeddings_seq( self._ctx.ctx, i - )[:n_embd] + ) + if not ptr: + raise RuntimeError("Failed to get embeddings from sequence pooling type is not set") + embedding: List[float] = ptr[:n_embd] if normalize: norm = float(np.linalg.norm(embedding)) embedding = [v / norm for v in embedding] diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 884a06384d..dc45fd9333 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -579,7 +579,6 @@ class llama_model_params(ctypes.Structure): # bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU - # // Abort callback # // if it returns true, execution of llama_decode() will be aborted # // currently works only with CPU execution From 4084aabe867b8ec2aba1b22659e59c9318b0d1f3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 14 Mar 2024 10:04:57 -0400 Subject: [PATCH 015/561] fix: set default pooling type to unspecified --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 18db1a0b6d..fed84d5792 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -79,7 +79,7 @@ def __init__( n_threads: Optional[int] = None, n_threads_batch: Optional[int] = None, rope_scaling_type: Optional[int] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, - pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_MEAN, + pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED, rope_freq_base: float = 0.0, rope_freq_scale: float = 0.0, yarn_ext_factor: float = -1.0, From 1a9b8af2dd39ed1c9050df0df2714e61d2740ee2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 14 Mar 2024 11:46:48 -0400 Subject: [PATCH 016/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 19885d205e..044ec4b2a5 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 19885d205e768579ab090d1e99281cae58c21b54 +Subproject commit 044ec4b2a567f649459ccd20af2f387c784faa51 From 20e6815252d0efd9f015f7adbf108faaf36e3f3c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Mar 2024 12:58:34 -0400 Subject: [PATCH 017/561] fix: json mode --- llama_cpp/llama_chat_format.py | 73 ++++++++++++++++------------------ 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 4eb2b0291c..81ca5520b6 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -339,16 +339,7 @@ def chat_completion_handler( stop = stop + rstop if response_format is not None and response_format["type"] == "json_object": - try: - # create grammar from json schema - if "schema" in response_format: - grammar = llama_grammar.LlamaGrammar.from_json_schema( - json.dumps(response_format["schema"]), verbose=llama.verbose - ) - except Exception as e: - grammar = llama_grammar.LlamaGrammar.from_string( - llama_grammar.JSON_GBNF, verbose=llama.verbose - ) + grammar = _grammar_for_response_format(response_format, verbose=llama.verbose) completion_or_chunks = llama.create_completion( prompt=prompt, @@ -606,6 +597,35 @@ def _format_chatglm3( ret += role return ret +def _grammar_for_json(verbose:bool=False): + return llama_grammar.LlamaGrammar.from_string(llama_grammar.JSON_GBNF, verbose=verbose) + +def _grammar_for_json_schema( + schema: str, + verbose: bool = False, + fallback_to_json: bool = True +): + try: + return llama_grammar.LlamaGrammar.from_json_schema(schema, verbose=verbose) + except Exception as e: + if fallback_to_json: + return _grammar_for_json(verbose=verbose) + else: + raise e + +def _grammar_for_response_format( + response_format: llama_types.ChatCompletionRequestResponseFormat, + verbose: bool = False +): + if response_format["type"] != "json_object": + return None + + if "schema" in response_format: + return _grammar_for_json_schema( + json.dumps(response_format["schema"]), verbose=verbose + ) + else: + return _grammar_for_json(verbose=verbose) ### Chat Formats ### @@ -1994,16 +2014,7 @@ def __call__( prompt = llama.input_ids[: llama.n_tokens].tolist() if response_format is not None and response_format["type"] == "json_object": - try: - # create grammar from json schema - if "schema" in response_format: - grammar = llama_grammar.LlamaGrammar.from_json_schema( - json.dumps(response_format["schema"]) - ) - except Exception as e: - grammar = llama_grammar.LlamaGrammar.from_string( - llama_grammar.JSON_GBNF - ) + grammar = _grammar_for_response_format(response_format) return _convert_completion_to_chat( llama.create_completion( @@ -2159,26 +2170,10 @@ def chatml_function_calling( tool_calls=None, add_generation_prompt=True, ) + if response_format is not None and response_format["type"] == "json_object": - try: - grammar = ( - llama_grammar.LlamaGrammar.from_json_schema( - json.dumps(response_format["schema"]) - ) - if "schema" in response_format - else None - ) - except Exception as e: - if llama.verbose: - print( - "Failed to parse response format as JSON schema, falling back to default grammar" - ) - print(e) - grammar = ( - llama_grammar.LlamaGrammar.from_string(llama_grammar.JSON_GBNF) - if grammar is None - else grammar - ) + grammar = _grammar_for_response_format(response_format) + return _convert_completion_to_chat( llama.create_completion( prompt=prompt, From 6eb25231e4dafeb792ffc9597c27330344c970b1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Mar 2024 12:58:45 -0400 Subject: [PATCH 018/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 14 ++++++++------ vendor/llama.cpp | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index dc45fd9333..b9593cf7a5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -198,13 +198,15 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # enum llama_vocab_type { -# LLAMA_VOCAB_TYPE_SPM = 0, // SentencePiece -# LLAMA_VOCAB_TYPE_BPE = 1, // Byte Pair Encoding -# LLAMA_VOCAB_TYPE_WPM = 2, // WordPiece +# LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab +# LLAMA_VOCAB_TYPE_SPM = 1, // SentencePiece +# LLAMA_VOCAB_TYPE_BPE = 2, // Byte Pair Encoding +# LLAMA_VOCAB_TYPE_WPM = 3, // WordPiece # }; -LLAMA_VOCAB_TYPE_SPM = 0 -LLAMA_VOCAB_TYPE_BPE = 1 -LLAMA_VOCAB_TYPE_WPM = 2 +LLAMA_VOCAB_TYPE_NONE = 0 +LLAMA_VOCAB_TYPE_SPM = 1 +LLAMA_VOCAB_TYPE_BPE = 2 +LLAMA_VOCAB_TYPE_WPM = 3 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 044ec4b2a5..4e9a7f7f7f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 044ec4b2a567f649459ccd20af2f387c784faa51 +Subproject commit 4e9a7f7f7fb6acbddd1462909c8d696e38edbfcc From 8d298b47507c9d82945f0add20d1ddb3e8ea0aa4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 18 Mar 2024 10:26:36 -0400 Subject: [PATCH 019/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 85 ++++++++++++++++++++++++++++++++++++++---- vendor/llama.cpp | 2 +- 2 files changed, 78 insertions(+), 9 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index b9593cf7a5..6b5c1bc157 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -581,6 +581,7 @@ class llama_model_params(ctypes.Structure): # bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU + # // Abort callback # // if it returns true, execution of llama_decode() will be aborted # // currently works only with CPU execution @@ -1006,6 +1007,11 @@ def llama_n_ctx_train(model: llama_model_p, /) -> int: ... def llama_n_embd(model: llama_model_p, /) -> int: ... +# LLAMA_API int32_t llama_n_layer (const struct llama_model * model); +@ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) +def llama_n_layer(model: llama_model_p, /) -> int: ... + + # // Get the model's RoPE frequency scaling factor # LLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model); @ctypes_function("llama_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float) @@ -1166,12 +1172,18 @@ def llama_model_quantize( ... +# // Apply a LoRA adapter to a loaded model +# // path_base_model is the path to a higher quality model to use as a base for +# // the layers modified by the adapter. Can be NULL to use the current loaded model. +# // The model needs to be reloaded before applying a new adapter, otherwise the adapter +# // will be applied on top of the previous one +# // Returns 0 on success # LLAMA_API int32_t llama_model_apply_lora_from_file( # const struct llama_model * model, -# const char * path_lora, -# float scale, -# const char * path_base_model, -# int32_t n_threads); +# const char * path_lora, +# float scale, +# const char * path_base_model, +# int32_t n_threads); @ctypes_function( "llama_model_apply_lora_from_file", [ @@ -1190,7 +1202,57 @@ def llama_model_apply_lora_from_file( path_base_model: Union[ctypes.c_char_p, bytes, None], n_threads: Union[ctypes.c_int32, int], /, -) -> int: ... +) -> int: + """Apply a LoRA adapter to a loaded model + path_base_model is the path to a higher quality model to use as a base for + the layers modified by the adapter. Can be NULL to use the current loaded model. + The model needs to be reloaded before applying a new adapter, otherwise the adapter + will be applied on top of the previous one + Returns 0 on success""" + ... + + +# // Apply a loaded control vector to a llama_context, or if data is NULL, clear +# // the currently loaded vector. +# // n_embd should be the size of a single layer's control, and data should point +# // to an n_embd x n_layers buffer starting from layer 1. +# // il_start and il_end are the layer range the vector should apply to (both inclusive) +# // See llama_control_vector_load in common to load a control vector. +# LLAMA_API int32_t llama_control_vector_apply( +# struct llama_context * lctx, +# const float * data, +# size_t len, +# int32_t n_embd, +# int32_t il_start, +# int32_t il_end); +@ctypes_function( + "llama_control_vector_apply", + [ + llama_context_p_ctypes, + ctypes.POINTER(ctypes.c_float), + ctypes.c_size_t, + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_int32, + ], + ctypes.c_int32, +) +def llama_control_vector_apply( + lctx: llama_context_p, + data: CtypesPointerOrRef[ctypes.c_float], + len: int, + n_embd: int, + il_start: int, + il_end: int, + /, +) -> int: + """Apply a loaded control vector to a llama_context, or if data is NULL, clear + the currently loaded vector. + n_embd should be the size of a single layer's control, and data should point + to an n_embd x n_layers buffer starting from layer 1. + il_start and il_end are the layer range the vector should apply to (both inclusive) + See llama_control_vector_load in common to load a control vector.""" + ... # // @@ -1205,6 +1267,12 @@ def llama_model_apply_lora_from_file( # llama_pos pos; # }; class llama_kv_cache_view_cell(ctypes.Structure): + """Information associated with an individual cell in the KV cache view. + + Attributes: + pos (llama_pos): The position for this cell. Takes KV cache shifts into account. + May be negative if the cell is not populated.""" + _fields_ = [("pos", llama_pos)] @@ -1985,7 +2053,7 @@ def llama_tokenize( /, ) -> int: """Convert the provided text into tokens. - + Args: model: The model to use for tokenization. text: The text to tokenize. @@ -1995,10 +2063,11 @@ def llama_tokenize( add_bos: Whether to add a beginning-of-sentence token. special: Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. Does not insert a leading space. - + Returns: Returns the number of tokens on success, no more than n_tokens_max - Returns a negative number on failure - the number of tokens that would have been returned""" + Returns a negative number on failure - the number of tokens that would have been returned + """ ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4e9a7f7f7f..ac9ee6a4ad 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4e9a7f7f7fb6acbddd1462909c8d696e38edbfcc +Subproject commit ac9ee6a4ad740bc1ee484ede43e9f92b5af244c1 From 8a60c7bc8cae7aa9770eeac0f482d39350763a6f Mon Sep 17 00:00:00 2001 From: Jeffrey Fong Date: Mon, 18 Mar 2024 22:40:57 +0800 Subject: [PATCH 020/561] fix: Fix and optimize functionary chat handler (#1282) * fix functionary chat logic * further fixes --------- Co-authored-by: Andrei --- llama_cpp/llama_chat_format.py | 131 ++++++++++++++++----------------- 1 file changed, 65 insertions(+), 66 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 81ca5520b6..c89cce8792 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1596,13 +1596,15 @@ def prepare_messages_for_inference( function_call = ( tool_choice if isinstance(tool_choice, str) else tool_choice["function"] ) + else: + function_call = "auto" prompt = prepare_messages_for_inference( messages, tokenizer, version, functions, tools ) # If no tools/functions are provided - if function_call is None and (functions is None or len(functions) == 0): + if function_call == "none" or functions is None or len(functions) == 0: if version == "v1": stop = END_ASSISTANT_TOKEN else: @@ -1630,6 +1632,7 @@ def prepare_messages_for_inference( logits_processor=logits_processor, grammar=grammar, ) + completion_or_completion_chunks["choices"][0]["text"] = completion_or_completion_chunks["choices"][0]["text"].lstrip() return _convert_completion_to_chat(completion_or_completion_chunks, stream=stream) # type: ignore assert stream is False # TODO: support stream mode @@ -1692,13 +1695,12 @@ def create_completion(stop): return completion + content = "" function_calls, function_bodies = [], [] if version == "v1": # If no or "auto" tool_choice/function_call - if function_call is None or ( - isinstance(function_call, str) and function_call == "auto" - ): + if isinstance(function_call, str) and function_call == "auto": stops = ["\n", END_ASSISTANT_TOKEN] # If tool_choice/function_call is "none" elif isinstance(function_call, str) and function_call == "none": @@ -1747,70 +1749,67 @@ def create_completion(stop): else: function_bodies.append(completion_text.strip()) else: - # Loop until all parallel function calls are generated - while True: - # If no or "auto" tool_choice/function_call - if function_call is None or ( - isinstance(function_call, str) and function_call == "auto" - ): - grammar = None - stops = CONTENT_TOKEN - # If tool_choice/function_call is "none" - elif isinstance(function_call, str) and function_call == "none": - prompt = ( - prepare_messages_for_inference(messages, tokenizer, version, [], []) - + "all\n<|content|>" - ) - stops = STOP_TOKEN - # If tool_choice/function_call is provided - elif isinstance(function_call, dict): - prompt += f"{function_call['name']}\n{CONTENT_TOKEN}" - stops = STOP_TOKEN - function_call = function_call["name"] - function_calls.append(function_call) - grammar = get_grammar(function_call) - else: - prompt = prompt - stops = STOP_TOKEN - + # If tool_choice/function_call is "none" + if isinstance(function_call, str) and function_call == "none": + prompt = ( + prepare_messages_for_inference(messages, tokenizer, version, [], []) + + "all\n<|content|>" + ) + stops = [STOP_TOKEN, FROM_TOKEN] + completion = create_completion(stop=stops) + completion["choices"][0]["text"] = completion["choices"][0]["text"].strip() + return _convert_completion_to_chat(completion, stream=stream) # type: ignore + # If tool_choice/function_call is provided + elif isinstance(function_call, dict): + prompt += f"{function_call['name']}\n{CONTENT_TOKEN}" + function_call = function_call["name"] + function_calls.append(function_call) + grammar = get_grammar(function_call) + stops = [STOP_TOKEN, FROM_TOKEN] completion = create_completion(stop=stops) completion_text = completion["choices"][0]["text"] - - # If the generation does not involve a function call - if prompt.endswith("all\n<|content|>") and not completion_text.startswith( - "all" - ): - return _convert_completion_to_chat(completion, stream=stream) # type: ignore - # Generate model response if the model decides not to call any function - elif prompt.endswith(RECIPIENT_TOKEN) and completion_text.startswith("all"): - prompt += completion_text + CONTENT_TOKEN - completion = create_completion(stop=STOP_TOKEN) - return _convert_completion_to_chat(completion, stream=stream) # type: ignore - # Generate parameters if model decides to call a function - elif prompt.endswith(RECIPIENT_TOKEN): - function_calls.append(completion_text[:-1]) - grammar = get_grammar(function_calls[-1]) - completion = create_completion(stop=[STOP_TOKEN, "\n"]) - function_bodies.append(completion["choices"][0]["text"].strip()) - prompt += f"{function_calls[-1]}\n{CONTENT_TOKEN}{function_bodies[-1]}" + function_bodies.append(completion_text.strip()) + # If "auto" or no tool_choice/function_call + elif isinstance(function_call, str) and function_call == "auto": + while True: + # Generate function name first grammar = None - - # Try to generate the beginning of next turn - # If empty completion, break from loop - next_turn_completion_text = create_completion( - stop=[STOP_TOKEN, RECIPIENT_TOKEN] - )["choices"][0]["text"] - if len(next_turn_completion_text) > 0: - prompt += f"\n{FROM_TOKEN}assistant\n{RECIPIENT_TOKEN}" + stops = CONTENT_TOKEN + completion = create_completion(stop=stops) + completion_text = completion["choices"][0]["text"] + function_name = completion_text.strip() + if function_name == "all": + prompt += "all\n<|content|>" else: - break - # Break from loop if tool_choice/function_call is provided as a dict - else: - function_bodies.append(completion_text.strip()) - break + function_call = completion_text.strip() + prompt += f"{function_call}\n<|content|>" + function_calls.append(function_call) + grammar = get_grammar(function_call) + # Generate content + stops = [RECIPIENT_TOKEN, STOP_TOKEN] + completion = create_completion(stop=stops) + completion_text = completion["choices"][0]["text"] + if function_name == "all": + content += completion_text.removesuffix("\n<|from|>assistant\n").removesuffix("\n<|from|> assistant\n") + content = content.lstrip() + # Check whether the model wants to generate another turn + if "<|from|> assistant" in completion_text or "<|from|>assistant" in completion_text: + cleaned_completion_text = completion_text.removesuffix("\n<|from|>assistant\n").removesuffix("\n<|from|> assistant\n").strip() + prompt += f"{cleaned_completion_text}\n<|from|>assistant\n<|recipient|>" + else: + break + else: + function_bodies.append(completion_text.strip()) + # Check whether the model wants to generate another turn + prompt += completion_text.strip() + grammar = None + completion = create_completion(stop=stops) + if "<|from|> assistant" in completion["choices"][0]["text"] or "<|from|>assistant" in completion["choices"][0]["text"]: + prompt += "\n<|from|>assistant\n<|recipient|>" + else: + break assert "usage" in completion - assert len(function_calls) > 0 assert len(function_calls) == len(function_bodies) tool_calls = [] @@ -1843,14 +1842,14 @@ def create_completion(stop): "index": 0, "message": { "role": "assistant", - "content": None, + "content": None if content == "" else content, "function_call": { "name": tool_calls[0]["function"]["name"], "arguments": tool_calls[0]["function"]["arguments"], - }, - "tool_calls": tool_calls, + } if len(tool_calls) > 0 else None, + "tool_calls": tool_calls if len(tool_calls) > 0 else None, }, - "finish_reason": "tool_calls", + "finish_reason": "tool_calls" if len(tool_calls) > 0 else "stop", } ], usage=completion["usage"], From bf64752535ac73032a25d6ba9ae0f064246964f2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 18 Mar 2024 11:37:30 -0400 Subject: [PATCH 021/561] chore: Bump version --- CHANGELOG.md | 7 +++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90dd1e6909..a85eaa4362 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.57] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ac9ee6a4ad740bc1ee484ede43e9f92b5af244c1 +- fix: set default embedding pooling type to unspecified by @abetlen in 4084aabe867b8ec2aba1b22659e59c9318b0d1f3 +- fix: Fix and optimize functionary chat handler by @jeffrey-fong in #1282 +- fix: json mode for basic chat formats by @abetlen in 20e6815252d0efd9f015f7adbf108faaf36e3f3c + ## [0.2.56] - feat: Update llama.cpp to ggerganov/llama.cpp@c2101a2e909ac7c08976d414e64e96c90ee5fa9e diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index fcbc7150ce..1e802fa7e3 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.56" \ No newline at end of file +__version__ = "0.2.57" \ No newline at end of file From 18d7ce918f45bc2bcf80102239142c49ebe29925 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 19 Mar 2024 04:40:24 -0400 Subject: [PATCH 022/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ac9ee6a4ad..b80cf3b2d1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ac9ee6a4ad740bc1ee484ede43e9f92b5af244c1 +Subproject commit b80cf3b2d1dee0ad325f7a794fecc66befce7336 From 60d8498f212ca1eb4303d95610022a055718bbb8 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 19 Mar 2024 04:55:57 -0400 Subject: [PATCH 023/561] feat: Add tools/functions variables to Jinja2ChatFormatter, add function response formatting for all simple chat formats (#1273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add tools/functions variables to Jinja2ChatFormatter Also fixed missing tools/tool_choices parameters in chat_formatter_to_chat_completion_handler(). * Set grammar when doing explicit function calling * Add function / tool response for all chat formats --------- Co-authored-by: Sigbjørn Skjæret --- llama_cpp/llama_chat_format.py | 408 +++++++++++++++++++-------------- 1 file changed, 233 insertions(+), 175 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index c89cce8792..5bda163fdb 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -188,6 +188,10 @@ def __call__( self, *, messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, **kwargs: Any, ) -> ChatFormatterResponse: def raise_exception(message: str): @@ -199,6 +203,10 @@ def raise_exception(message: str): bos_token=self.bos_token, raise_exception=raise_exception, add_generation_prompt=self.add_generation_prompt, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, ) return ChatFormatterResponse(prompt=prompt, stop=[self.eos_token]) @@ -288,6 +296,183 @@ def _convert_completion_to_chat( return _convert_text_completion_to_chat(completion) +def _convert_completion_to_chat_function( + tool_name: str, + completion_or_chunks: Union[ + llama_types.CreateCompletionResponse, + Iterator[llama_types.CreateCompletionStreamResponse], + ], + stream: bool, +): + if not stream: + completion: llama_types.CreateCompletionResponse = completion_or_chunks # type: ignore + assert "usage" in completion + tool_id = "call_" + "_0_" + tool_name + "_" + completion["id"] + # TODO: Fix for legacy function calls + chat_completion: llama_types.CreateChatCompletionResponse = { + "id": "chat" + completion["id"], + "object": "chat.completion", + "created": completion["created"], + "model": completion["model"], + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "function_call": { + "name": tool_name, + "arguments": completion["choices"][0]["text"], + }, + "tool_calls": [ + { + "id": tool_id, + "type": "function", + "function": { + "name": tool_name, + "arguments": completion["choices"][0]["text"], + }, + } + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": completion["usage"], + } + return chat_completion + else: + chunks: Iterator[llama_types.CreateCompletionStreamResponse] = completion_or_chunks # type: ignore + + def _stream_response_to_function_stream( + chunks: Iterator[llama_types.CreateCompletionStreamResponse], + ) -> Iterator[llama_types.CreateChatCompletionStreamResponse]: + # blank first message + first = True + id_ = None + created = None + model = None + tool_id = None + for chunk in chunks: + if first: + id_ = "chat" + chunk["id"] + created = chunk["created"] + model = chunk["model"] + tool_id = "call_" + "_0_" + tool_name + "_" + chunk["id"] + yield { + "id": id_, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": None, + "logprobs": None, + "delta": { + "role": "assistant", + "content": None, + "function_call": None, + "tool_calls": None, + }, + } + ], + } + yield { + "id": "chat" + chunk["id"], + "object": "chat.completion.chunk", + "created": chunk["created"], + "model": chunk["model"], + "choices": [ + { + "index": 0, + "finish_reason": None, + "logprobs": None, + "delta": { + "role": None, + "content": None, + "function_call": { + "name": tool_name, + "arguments": chunk["choices"][0]["text"], + }, + "tool_calls": [ + { + "index": 0, + "id": tool_id, + "type": "function", + "function": { + "name": tool_name, + "arguments": "", + }, + } + ], + }, + } + ], + } + first = False + continue + assert tool_id is not None + yield { + "id": "chat" + chunk["id"], + "object": "chat.completion.chunk", + "created": chunk["created"], + "model": chunk["model"], + "choices": [ + { + "index": 0, + "finish_reason": None, + "logprobs": None, + "delta": { + "role": None, + "content": None, + "function_call": { + "name": tool_name, + "arguments": chunk["choices"][0]["text"], + }, + "tool_calls": [ + { + "index": 0, + "id": tool_id, + "type": "function", + "function": { + "name": tool_name, + "arguments": chunk["choices"][0][ + "text" + ], + }, + } + ], + }, + } + ], + } + + if id_ is not None and created is not None and model is not None: + yield { + "id": id_, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "logprobs": None, + "delta": { + "role": None, + "content": None, + "function_call": None, + "tool_calls": None, + }, + } + ], + } + + return _stream_response_to_function_stream(chunks) + + + def chat_formatter_to_chat_completion_handler( chat_formatter: ChatFormatter, ) -> LlamaChatCompletionHandler: @@ -331,6 +516,8 @@ def chat_completion_handler( messages=messages, functions=functions, function_call=function_call, + tools=tools, + tool_choice=tool_choice, ) prompt = result.prompt if result.stop is not None: @@ -341,6 +528,47 @@ def chat_completion_handler( if response_format is not None and response_format["type"] == "json_object": grammar = _grammar_for_response_format(response_format, verbose=llama.verbose) + # Convert legacy functions to tools + if functions is not None: + tools = [ + { + "type": "function", + "function": function, + } + for function in functions + ] + + # Convert legacy function_call to tool_choice + if function_call is not None: + if isinstance(function_call, str) and ( + function_call == "none" or function_call == "auto" + ): + tool_choice = function_call + if isinstance(function_call, dict) and "name" in function_call: + tool_choice = { + "type": "function", + "function": { + "name": function_call["name"], + }, + } + + tool = None + if tool_choice is not None and isinstance(tool_choice, dict) and tools is not None: + name = tool_choice["function"]["name"] + tool = next((t for t in tools if t["function"]["name"] == name), None) + if tool is None: + raise ValueError(f"Tool choice '{name}' not found in tools.") + schema = tool["function"]["parameters"] + try: + # create grammar from json schema + grammar = llama_grammar.LlamaGrammar.from_json_schema( + json.dumps(schema), verbose=llama.verbose + ) + except Exception as e: + grammar = llama_grammar.LlamaGrammar.from_string( + llama_grammar.JSON_GBNF, verbose=llama.verbose + ) + completion_or_chunks = llama.create_completion( prompt=prompt, temperature=temperature, @@ -364,6 +592,11 @@ def chat_completion_handler( grammar=grammar, logit_bias=logit_bias, ) + if tool is not None: + tool_name = tool["function"]["name"] + return _convert_completion_to_chat_function( + tool_name, completion_or_chunks, stream + ) return _convert_completion_to_chat(completion_or_chunks, stream=stream) return chat_completion_handler @@ -2198,181 +2431,6 @@ def chatml_function_calling( stream=stream, ) - def _convert_completion_to_chat_function( - tool_name: str, - completion_or_chunks: Union[ - llama_types.CreateCompletionResponse, - Iterator[llama_types.CreateCompletionStreamResponse], - ], - stream: bool, - ): - if not stream: - completion: llama_types.CreateCompletionResponse = completion_or_chunks # type: ignore - assert "usage" in completion - tool_id = "call_" + "_0_" + tool_name + "_" + completion["id"] - # TODO: Fix for legacy function calls - chat_completion: llama_types.CreateChatCompletionResponse = { - "id": "chat" + completion["id"], - "object": "chat.completion", - "created": completion["created"], - "model": completion["model"], - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": None, - "function_call": { - "name": tool_name, - "arguments": completion["choices"][0]["text"], - }, - "tool_calls": [ - { - "id": tool_id, - "type": "function", - "function": { - "name": tool_name, - "arguments": completion["choices"][0]["text"], - }, - } - ], - }, - "finish_reason": "tool_calls", - } - ], - "usage": completion["usage"], - } - return chat_completion - else: - chunks: Iterator[llama_types.CreateCompletionStreamResponse] = completion_or_chunks # type: ignore - - def _stream_response_to_function_stream( - chunks: Iterator[llama_types.CreateCompletionStreamResponse], - ) -> Iterator[llama_types.CreateChatCompletionStreamResponse]: - # blank first message - first = True - id_ = None - created = None - model = None - tool_id = None - for chunk in chunks: - if first: - id_ = "chat" + chunk["id"] - created = chunk["created"] - model = chunk["model"] - tool_id = "call_" + "_0_" + tool_name + "_" + chunk["id"] - yield { - "id": id_, - "object": "chat.completion.chunk", - "created": created, - "model": model, - "choices": [ - { - "index": 0, - "finish_reason": None, - "logprobs": None, - "delta": { - "role": "assistant", - "content": None, - "function_call": None, - "tool_calls": None, - }, - } - ], - } - yield { - "id": "chat" + chunk["id"], - "object": "chat.completion.chunk", - "created": chunk["created"], - "model": chunk["model"], - "choices": [ - { - "index": 0, - "finish_reason": None, - "logprobs": None, - "delta": { - "role": None, - "content": None, - "function_call": { - "name": tool_name, - "arguments": chunk["choices"][0]["text"], - }, - "tool_calls": [ - { - "index": 0, - "id": tool_id, - "type": "function", - "function": { - "name": tool_name, - "arguments": "", - }, - } - ], - }, - } - ], - } - first = False - continue - assert tool_id is not None - yield { - "id": "chat" + chunk["id"], - "object": "chat.completion.chunk", - "created": chunk["created"], - "model": chunk["model"], - "choices": [ - { - "index": 0, - "finish_reason": None, - "logprobs": None, - "delta": { - "role": None, - "content": None, - "function_call": { - "name": tool_name, - "arguments": chunk["choices"][0]["text"], - }, - "tool_calls": [ - { - "index": 0, - "id": tool_id, - "type": "function", - "function": { - "name": tool_name, - "arguments": chunk["choices"][0][ - "text" - ], - }, - } - ], - }, - } - ], - } - - if id_ is not None and created is not None and model is not None: - yield { - "id": id_, - "object": "chat.completion.chunk", - "created": created, - "model": model, - "choices": [ - { - "index": 0, - "finish_reason": "tool_calls", - "logprobs": None, - "delta": { - "role": None, - "content": None, - "function_call": None, - "tool_calls": None, - }, - } - ], - } - - return _stream_response_to_function_stream(chunks) - # Case 2: Tool choice by user if isinstance(tool_choice, dict): tool_name = tool_choice["function"]["name"] From f7decc956207f181710a43e2795f478ad2a5fc5e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 19 Mar 2024 10:52:53 -0400 Subject: [PATCH 024/561] docs: Add chat examples to openapi ui --- llama_cpp/server/app.py | 68 +++++++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index aa6afc1122..2e1081e423 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -12,14 +12,7 @@ import anyio from anyio.streams.memory import MemoryObjectSendStream from starlette.concurrency import run_in_threadpool, iterate_in_threadpool -from fastapi import ( - Depends, - FastAPI, - APIRouter, - Request, - HTTPException, - status, -) +from fastapi import Depends, FastAPI, APIRouter, Request, HTTPException, status, Body from fastapi.middleware import Middleware from fastapi.middleware.cors import CORSMiddleware from fastapi.security import HTTPBearer @@ -356,7 +349,64 @@ async def create_embedding( ) async def create_chat_completion( request: Request, - body: CreateChatCompletionRequest, + body: CreateChatCompletionRequest = Body( + openapi_examples={ + "normal": { + "summary": "Chat Completion", + "value": { + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + }, + }, + "json_mode": { + "summary": "JSON Mode", + "value": { + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Who won the world series in 2020"}, + ], + "response_format": { "type": "json_object" } + }, + }, + "tool_calling": { + "summary": "Tool Calling", + "value": { + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Extract Jason is 30 years old."}, + ], + "tools": [ + { + "type": "function", + "function": { + "name": "User", + "description": "User record", + "parameters": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "number"}, + }, + "required": ["name", "age"], + }, + } + } + ], + "tool_choice": { + "type": "function", + "function": { + "name": "User", + } + } + }, + }, + } + ), llama_proxy: LlamaProxy = Depends(get_llama_proxy), ) -> llama_cpp.ChatCompletion: exclude = { From 740f3f38125d1cc1bbe80f25944a292d0e966868 Mon Sep 17 00:00:00 2001 From: bretello Date: Wed, 20 Mar 2024 17:46:09 +0100 Subject: [PATCH 025/561] fix: set LLAMA_METAL_EMBED_LIBRARY=on on MacOS arm64 (#1289) --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b4df8ef451..7415149076 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,11 @@ if (LLAMA_BUILD) set(LLAMA_FMA "Off" CACHE BOOL "llama: enable FMA" FORCE) set(LLAMA_F16C "Off" CACHE BOOL "llama: enable F16C" FORCE) endif() + + if (APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") + set(LLAMA_METAL_EMBED_LIBRARY "On" CACHE BOOL "llama: embed metal library" FORCE) + endif() + add_subdirectory(vendor/llama.cpp) install( TARGETS llama From 3db03b73027036cf336fda2448894c36d3899cab Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 20 Mar 2024 13:27:43 -0400 Subject: [PATCH 026/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b80cf3b2d1..f9c7ba3447 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b80cf3b2d1dee0ad325f7a794fecc66befce7336 +Subproject commit f9c7ba34476ffc4f13ae2cdb1aec493a16eb8d47 From c89be28ef945017e8b64ea5f194ae7073fd872e8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 20 Mar 2024 20:50:47 -0400 Subject: [PATCH 027/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f9c7ba3447..42e21c6882 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f9c7ba34476ffc4f13ae2cdb1aec493a16eb8d47 +Subproject commit 42e21c68826f2e56b9592dccd9f3c43895b6890d From e325a831f015fdc807b436bc3c48e52cce658a18 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 22 Mar 2024 23:43:29 -0400 Subject: [PATCH 028/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 64 ++++++++++++++++++++++++++++++++++++------ vendor/llama.cpp | 2 +- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 6b5c1bc157..1b8f6caa99 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -668,13 +668,15 @@ class llama_context_params(ctypes.Structure): # // model quantization parameters # typedef struct llama_model_quantize_params { -# int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() -# enum llama_ftype ftype; // quantize to this llama_ftype -# bool allow_requantize; // allow quantizing non-f32/f16 tensors -# bool quantize_output_tensor; // quantize output.weight -# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored -# bool pure; // quantize all tensors to the default type -# void * imatrix; // pointer to importance matrix data +# int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() +# enum llama_ftype ftype; // quantize to this llama_ftype +# enum ggml_type output_tensor_type; // output tensor type +# enum ggml_type token_embedding_type; // itoken embeddings tensor type +# bool allow_requantize; // allow quantizing non-f32/f16 tensors +# bool quantize_output_tensor; // quantize output.weight +# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored +# bool pure; // quantize all tensors to the default type +# void * imatrix; // pointer to importance matrix data # } llama_model_quantize_params; class llama_model_quantize_params(ctypes.Structure): """Parameters for llama_model_quantize @@ -682,16 +684,20 @@ class llama_model_quantize_params(ctypes.Structure): Attributes: nthread (int): number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() ftype (int): quantize to this llama_ftype + output_tensor_type (int): output tensor type + token_embedding_type (int): itoken embeddings tensor type allow_requantize (bool): allow quantizing non-f32/f16 tensors quantize_output_tensor (bool): quantize output.weight only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored pure (bool): quantize all tensors to the default type - imatrix (ctypes.ctypes.c_void_p): pointer to importance matrix data + imatrix (ctypes.c_void_p): pointer to importance matrix data """ _fields_ = [ ("nthread", ctypes.c_int32), ("ftype", ctypes.c_int), + ("output_tensor_type", ctypes.c_int), + ("token_embedding_type", ctypes.c_int), ("allow_requantize", ctypes.c_bool), ("quantize_output_tensor", ctypes.c_bool), ("only_copy", ctypes.c_bool), @@ -2743,6 +2749,48 @@ def llama_beam_search( ): ... +# /// @details Build a split GGUF final path for this chunk. +# /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" +# // Returns the split_path length. +# LLAMA_API int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count); +@ctypes_function( + "llama_split_path", + [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int, ctypes.c_int], + ctypes.c_int, +) +def llama_split_path( + split_path: bytes, + maxlen: Union[ctypes.c_size_t, int], + path_prefix: bytes, + split_no: Union[ctypes.c_int, int], + split_count: Union[ctypes.c_int, int], + /, +) -> int: + """Build a split GGUF final path for this chunk.""" + ... + + +# /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. +# /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" +# // Returns the split_prefix length. +# LLAMA_API int llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int split_no, int split_count); +@ctypes_function( + "llama_split_prefix", + [ctypes.c_char_p, ctypes.c_size_t, ctypes.c_char_p, ctypes.c_int, ctypes.c_int], + ctypes.c_int, +) +def llama_split_prefix( + split_prefix: bytes, + maxlen: Union[ctypes.c_size_t, int], + split_path: bytes, + split_no: Union[ctypes.c_int, int], + split_count: Union[ctypes.c_int, int], + /, +) -> int: + """Extract the path prefix from the split_path if and only if the split_no and split_count match.""" + ... + + # Performance information diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 42e21c6882..50ccaf5eac 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 42e21c68826f2e56b9592dccd9f3c43895b6890d +Subproject commit 50ccaf5eacb50a2ca378a4ef0dc7aeb45fead652 From c1325dcdfba7cb331b4c09d110001658b94ebb9f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 22 Mar 2024 23:44:04 -0400 Subject: [PATCH 029/561] fix: tool_call missing first token. --- llama_cpp/llama_chat_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 5bda163fdb..ccf4fd0b73 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -402,7 +402,7 @@ def _stream_response_to_function_stream( "type": "function", "function": { "name": tool_name, - "arguments": "", + "arguments": chunk["choices"][0]["text"], }, } ], From d11ccc30361a947177e46d31c632988661f5af02 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 23 Mar 2024 17:14:15 -0400 Subject: [PATCH 030/561] fix(server): minor type fixes --- llama_cpp/server/app.py | 6 +++--- llama_cpp/server/types.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 2e1081e423..d5e2f7ee83 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -493,7 +493,7 @@ async def tokenize( ) -> TokenizeInputResponse: tokens = llama_proxy(body.model).tokenize(body.input.encode("utf-8"), special=True) - return {"tokens": tokens} + return TokenizeInputResponse(tokens=tokens) @router.post( @@ -508,7 +508,7 @@ async def count_query_tokens( ) -> TokenizeInputCountResponse: tokens = llama_proxy(body.model).tokenize(body.input.encode("utf-8"), special=True) - return {"count": len(tokens)} + return TokenizeInputCountResponse(count=len(tokens)) @router.post( @@ -523,4 +523,4 @@ async def detokenize( ) -> DetokenizeInputResponse: text = llama_proxy(body.model).detokenize(body.tokens).decode("utf-8") - return {"text": text} + return DetokenizeInputResponse(text=text) diff --git a/llama_cpp/server/types.py b/llama_cpp/server/types.py index c8b2ebc6cf..378f8d74c8 100644 --- a/llama_cpp/server/types.py +++ b/llama_cpp/server/types.py @@ -268,7 +268,7 @@ class ModelList(TypedDict): class TokenizeInputRequest(BaseModel): model: Optional[str] = model_field - input: Optional[str] = Field(description="The input to tokenize.") + input: str = Field(description="The input to tokenize.") model_config = { "json_schema_extra": {"examples": [{"input": "How many tokens in this query?"}]} From 364678bde55ca75db2788563e2d75e6303720e6d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 24 Mar 2024 12:27:49 -0400 Subject: [PATCH 031/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 50ccaf5eac..a0e584defd 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 50ccaf5eacb50a2ca378a4ef0dc7aeb45fead652 +Subproject commit a0e584defd8c16e7a51ab895f595df0448d710d0 From a93b9149f8ebab3219680e8f8cd46e53177aad74 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 25 Mar 2024 11:10:14 -0400 Subject: [PATCH 032/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a0e584defd..2f34b865b6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a0e584defd8c16e7a51ab895f595df0448d710d0 +Subproject commit 2f34b865b62b1d2b5eb8a27885e4de220deeacbd From b64fa4e2c05385d7cb36154dce42a29ca2b669ee Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 25 Mar 2024 23:09:07 -0400 Subject: [PATCH 033/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 2f34b865b6..e190f1fca6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 2f34b865b62b1d2b5eb8a27885e4de220deeacbd +Subproject commit e190f1fca6f60d80944f9e8709d343a025c4d245 From 901fe02461fbad0aa817060b8a4062924b89f418 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 26 Mar 2024 22:58:53 -0400 Subject: [PATCH 034/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 30 ++++++++++++++++++++---------- vendor/llama.cpp | 2 +- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1b8f6caa99..dc39e4c3b1 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -175,8 +175,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 4 -LLAMA_SESSION_VERSION = 4 +# define LLAMA_SESSION_VERSION 5 +LLAMA_SESSION_VERSION = 5 # struct llama_model; @@ -274,6 +274,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors # LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors # LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -677,6 +678,7 @@ class llama_context_params(ctypes.Structure): # bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored # bool pure; // quantize all tensors to the default type # void * imatrix; // pointer to importance matrix data +# void * kv_overrides; // pointer to vector containing overrides # } llama_model_quantize_params; class llama_model_quantize_params(ctypes.Structure): """Parameters for llama_model_quantize @@ -691,6 +693,7 @@ class llama_model_quantize_params(ctypes.Structure): only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored pure (bool): quantize all tensors to the default type imatrix (ctypes.c_void_p): pointer to importance matrix data + kv_overrides (ctypes.c_void_p): pointer to vector containing overrides """ _fields_ = [ @@ -703,6 +706,7 @@ class llama_model_quantize_params(ctypes.Structure): ("only_copy", ctypes.c_bool), ("pure", ctypes.c_bool), ("imatrix", ctypes.c_void_p), + ("kv_overrides", ctypes.c_void_p), ] @@ -1838,9 +1842,9 @@ def llama_synchronize(ctx: llama_context_p, /): # // Token logits obtained from the last call to llama_decode() -# // The logits for the last token are stored in the last row -# // Logits for which llama_batch.logits[i] == 0 are undefined -# // Rows: n_tokens provided with llama_batch +# // The logits for which llama_batch.logits[i] != 0 are stored contiguously +# // in the order they have appeared in the batch. +# // Rows: number of tokens for which llama_batch.logits[i] != 0 # // Cols: n_vocab # LLAMA_API float * llama_get_logits(struct llama_context * ctx); @ctypes_function( @@ -1859,7 +1863,8 @@ def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]: # // Logits for the ith token. Equivalent to: -# // llama_get_logits(ctx) + i*n_vocab +# // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab +# // returns NULL for invalid ids. # LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i); @ctypes_function( "llama_get_logits_ith", @@ -1874,8 +1879,12 @@ def llama_get_logits_ith( ... -# // Get all output token embeddings -# // shape: [n_tokens*n_embd] (1-dimensional) +# // Get all output token embeddings. +# // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model, +# // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously +# // in the order they have appeared in the batch. +# // shape: [n_outputs*n_embd] +# // Otherwise, returns NULL. # LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); @ctypes_function( "llama_get_embeddings", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float) @@ -1886,9 +1895,10 @@ def llama_get_embeddings(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float] ... -# // Get the embeddings for the ith token -# // llama_get_embeddings(ctx) + i*n_embd +# // Get the embeddings for the ith token. Equivalent to: +# // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd # // shape: [n_embd] (1-dimensional) +# // returns NULL for invalid ids. # LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); @ctypes_function( "llama_get_embeddings_ith", diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e190f1fca6..32c8486e1f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e190f1fca6f60d80944f9e8709d343a025c4d245 +Subproject commit 32c8486e1f0297393cb22ac0a0d26a6b17ad4d54 From 125b2358c9ea92cbcb416e2bd2c8f93132e641ac Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 28 Mar 2024 12:06:46 -0400 Subject: [PATCH 035/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 10 +++++++--- vendor/llama.cpp | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index dc39e4c3b1..1db47becc8 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -199,14 +199,18 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # enum llama_vocab_type { # LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab -# LLAMA_VOCAB_TYPE_SPM = 1, // SentencePiece -# LLAMA_VOCAB_TYPE_BPE = 2, // Byte Pair Encoding -# LLAMA_VOCAB_TYPE_WPM = 3, // WordPiece +# LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback +# LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE +# LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece # }; LLAMA_VOCAB_TYPE_NONE = 0 +"""For models without vocab""" LLAMA_VOCAB_TYPE_SPM = 1 +"""LLaMA tokenizer based on byte-level BPE with byte fallback""" LLAMA_VOCAB_TYPE_BPE = 2 +"""GPT-2 tokenizer based on byte-level BPE""" LLAMA_VOCAB_TYPE_WPM = 3 +"""BERT tokenizer based on WordPiece""" # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 32c8486e1f..5106ef482c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 32c8486e1f0297393cb22ac0a0d26a6b17ad4d54 +Subproject commit 5106ef482c65ac60ac14da9a68c7b37bca4c6993 From dcbe57fcf8c4af829ef0085720b8730b0516f99a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 29 Mar 2024 12:45:27 -0400 Subject: [PATCH 036/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 5106ef482c..069574775c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 5106ef482c65ac60ac14da9a68c7b37bca4c6993 +Subproject commit 069574775cea67d8a977904e4d534aff47a671f4 From 1e60dba082464b8828dca0a6d05a2fe46fcc4f7c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 29 Mar 2024 13:34:23 -0400 Subject: [PATCH 037/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 069574775c..ba0c7c70ab 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 069574775cea67d8a977904e4d534aff47a671f4 +Subproject commit ba0c7c70ab5b15f1f2be7fb0dfbe0366dda30d6c From aa9f1ae011fbc22893750209af500fee3167f21c Mon Sep 17 00:00:00 2001 From: windspirit95 <32880949+windspirit95@users.noreply.github.com> Date: Mon, 1 Apr 2024 02:30:13 +0900 Subject: [PATCH 038/561] feat: Add logprobs support to chat completions (#1311) * Add logprobs return in ChatCompletionResponse * Fix duplicate field * Set default to false * Simplify check * Add server example --------- Co-authored-by: Andrei Betlen --- llama_cpp/llama.py | 1 + llama_cpp/llama_chat_format.py | 5 +++++ llama_cpp/llama_types.py | 1 + llama_cpp/server/app.py | 12 ++++++++++++ llama_cpp/server/types.py | 10 +++++++++- 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index fed84d5792..66caaa958a 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1653,6 +1653,7 @@ def create_chat_completion( top_k=top_k, min_p=min_p, typical_p=typical_p, + logprobs=top_logprobs if logprobs else None, stream=stream, stop=stop, seed=seed, diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index ccf4fd0b73..06cf9ced7d 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -231,6 +231,7 @@ def _convert_text_completion_to_chat( "role": "assistant", "content": completion["choices"][0]["text"], }, + "logprobs": completion["choices"][0]["logprobs"], "finish_reason": completion["choices"][0]["finish_reason"], } ], @@ -254,6 +255,7 @@ def _convert_text_completion_chunks_to_chat( "delta": { "role": "assistant", }, + "logprobs": None, "finish_reason": None, } ], @@ -273,6 +275,7 @@ def _convert_text_completion_chunks_to_chat( if chunk["choices"][0]["finish_reason"] is None else {} ), + "logprobs": chunk["choices"][0]["logprobs"], "finish_reason": chunk["choices"][0]["finish_reason"], } ], @@ -487,6 +490,7 @@ def chat_completion_handler( temperature: float = 0.2, top_p: float = 0.95, top_k: int = 40, + logprobs: int = 0, min_p: float = 0.05, typical_p: float = 1.0, stream: bool = False, @@ -576,6 +580,7 @@ def chat_completion_handler( top_k=top_k, min_p=min_p, typical_p=typical_p, + logprobs=logprobs, stream=stream, stop=stop, seed=seed, diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 1b1befebef..87e000f140 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -84,6 +84,7 @@ class ChatCompletionFunction(TypedDict): class ChatCompletionResponseChoice(TypedDict): index: int message: "ChatCompletionResponseMessage" + logprobs: Optional[CompletionLogprobs] finish_reason: Optional[str] diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index d5e2f7ee83..815ed3c5ea 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -405,6 +405,18 @@ async def create_chat_completion( } }, }, + "logprobs": { + "summary": "Logprobs", + "value": { + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + "logprobs": True, + "top_logprobs": 10 + }, + }, } ), llama_proxy: LlamaProxy = Depends(get_llama_proxy), diff --git a/llama_cpp/server/types.py b/llama_cpp/server/types.py index 378f8d74c8..ce9c87a694 100644 --- a/llama_cpp/server/types.py +++ b/llama_cpp/server/types.py @@ -130,7 +130,6 @@ class CreateCompletionRequest(BaseModel): presence_penalty: Optional[float] = presence_penalty_field frequency_penalty: Optional[float] = frequency_penalty_field logit_bias: Optional[Dict[str, float]] = Field(None) - logprobs: Optional[int] = Field(None) seed: Optional[int] = Field(None) # ignored or currently unsupported @@ -209,6 +208,15 @@ class CreateChatCompletionRequest(BaseModel): default=None, description="The maximum number of tokens to generate. Defaults to inf", ) + logprobs: Optional[bool] = Field( + default=False, + description="Whether to output the logprobs or not. Default is True" + ) + top_logprobs: Optional[int] = Field( + default=None, + ge=0, + description="The number of logprobs to generate. If None, no logprobs are generated. logprobs need to set to True.", + ) temperature: float = temperature_field top_p: float = top_p_field min_p: float = min_p_field From f165048a69b2ad5b70d4fd2ec4adebfb411b5b47 Mon Sep 17 00:00:00 2001 From: Limour <93720049+Limour-dev@users.noreply.github.com> Date: Mon, 1 Apr 2024 22:19:28 +0800 Subject: [PATCH 039/561] feat: add support for KV cache quantization options (#1307) * add KV cache quantization options https://github.com/abetlen/llama-cpp-python/discussions/1220 https://github.com/abetlen/llama-cpp-python/issues/1305 * Add ggml_type * Use ggml_type instead of string for quantization * Add server support --------- Co-authored-by: Andrei Betlen --- llama_cpp/llama.py | 59 ++++++++++----------------------- llama_cpp/llama_cpp.py | 64 ++++++++++++++++++++++++++++++++++++ llama_cpp/server/model.py | 3 ++ llama_cpp/server/settings.py | 9 +++++ 4 files changed, 94 insertions(+), 41 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 66caaa958a..dcc7be7580 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -105,6 +105,9 @@ def __init__( draft_model: Optional[LlamaDraftModel] = None, # Tokenizer Override tokenizer: Optional[BaseLlamaTokenizer] = None, + # KV cache quantization + type_k: Optional[int] = None, + type_v: Optional[int] = None, # Misc verbose: bool = True, # Extra Params @@ -172,6 +175,8 @@ def __init__( draft_model: Optional draft model to use for speculative decoding. tokenizer: Optional tokenizer to override the default tokenizer from llama.cpp. verbose: Print verbose output to stderr. + type_k: KV cache data type for K (default: f16) + type_v: KV cache data type for V (default: f16) Raises: ValueError: If the model path does not exist. @@ -298,7 +303,11 @@ def __init__( ) # Must be set to True for speculative decoding self.context_params.embeddings = embedding # TODO: Rename to embeddings self.context_params.offload_kqv = offload_kqv - + # KV cache quantization + if type_k is not None: + self.context_params.type_k = type_k + if type_v is not None: + self.context_params.type_v = type_v # Sampling Params self.last_n_tokens_size = last_n_tokens_size @@ -1724,6 +1733,7 @@ def __getstate__(self): n_threads=self.context_params.n_threads, n_threads_batch=self.context_params.n_threads_batch, rope_scaling_type=self.context_params.rope_scaling_type, + pooling_type=self.context_params.pooling_type, rope_freq_base=self.context_params.rope_freq_base, rope_freq_scale=self.context_params.rope_freq_scale, yarn_ext_factor=self.context_params.yarn_ext_factor, @@ -1733,6 +1743,7 @@ def __getstate__(self): yarn_orig_ctx=self.context_params.yarn_orig_ctx, logits_all=self.context_params.logits_all, embedding=self.context_params.embeddings, + offload_kqv=self.context_params.offload_kqv, # Sampling Params last_n_tokens_size=self.last_n_tokens_size, # LoRA Params @@ -1744,51 +1755,17 @@ def __getstate__(self): # Chat Format Params chat_format=self.chat_format, chat_handler=self.chat_handler, + # Speculative Decidng + draft_model=self.draft_model, + # KV cache quantization + type_k=self.context_params.type_k, + type_v=self.context_params.type_v, # Misc verbose=self.verbose, ) def __setstate__(self, state): - self.__init__( - model_path=state["model_path"], - # Model Params - n_gpu_layers=state["n_gpu_layers"], - split_mode=state["split_mode"], - main_gpu=state["main_gpu"], - tensor_split=state["tensor_split"], - vocab_only=state["vocab_only"], - use_mmap=state["use_mmap"], - use_mlock=state["use_mlock"], - kv_overrides=state["kv_overrides"], - # Context Params - seed=state["seed"], - n_ctx=state["n_ctx"], - n_batch=state["n_batch"], - n_threads=state["n_threads"], - n_threads_batch=state["n_threads_batch"], - rope_freq_base=state["rope_freq_base"], - rope_freq_scale=state["rope_freq_scale"], - rope_scaling_type=state["rope_scaling_type"], - yarn_ext_factor=state["yarn_ext_factor"], - yarn_attn_factor=state["yarn_attn_factor"], - yarn_beta_fast=state["yarn_beta_fast"], - yarn_beta_slow=state["yarn_beta_slow"], - yarn_orig_ctx=state["yarn_orig_ctx"], - logits_all=state["logits_all"], - embedding=state["embedding"], - # Sampling Params - last_n_tokens_size=state["last_n_tokens_size"], - # LoRA Params - lora_base=state["lora_base"], - lora_path=state["lora_path"], - # Backend Params - numa=state["numa"], - # Chat Format Params - chat_format=state["chat_format"], - chat_handler=state["chat_handler"], - # Misc - verbose=state["verbose"], - ) + self.__init__(**state) def save_state(self) -> LlamaState: assert self._ctx.ctx is not None diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1db47becc8..accc02cb85 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -141,6 +141,70 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa byref = ctypes.byref # type: ignore +# from ggml.h +# // NOTE: always add types at the end of the enum to keep backward compatibility +# enum ggml_type { +# GGML_TYPE_F32 = 0, +# GGML_TYPE_F16 = 1, +# GGML_TYPE_Q4_0 = 2, +# GGML_TYPE_Q4_1 = 3, +# // GGML_TYPE_Q4_2 = 4, support has been removed +# // GGML_TYPE_Q4_3 = 5, support has been removed +# GGML_TYPE_Q5_0 = 6, +# GGML_TYPE_Q5_1 = 7, +# GGML_TYPE_Q8_0 = 8, +# GGML_TYPE_Q8_1 = 9, +# GGML_TYPE_Q2_K = 10, +# GGML_TYPE_Q3_K = 11, +# GGML_TYPE_Q4_K = 12, +# GGML_TYPE_Q5_K = 13, +# GGML_TYPE_Q6_K = 14, +# GGML_TYPE_Q8_K = 15, +# GGML_TYPE_IQ2_XXS = 16, +# GGML_TYPE_IQ2_XS = 17, +# GGML_TYPE_IQ3_XXS = 18, +# GGML_TYPE_IQ1_S = 19, +# GGML_TYPE_IQ4_NL = 20, +# GGML_TYPE_IQ3_S = 21, +# GGML_TYPE_IQ2_S = 22, +# GGML_TYPE_IQ4_XS = 23, +# GGML_TYPE_I8 = 24, +# GGML_TYPE_I16 = 25, +# GGML_TYPE_I32 = 26, +# GGML_TYPE_I64 = 27, +# GGML_TYPE_F64 = 28, +# GGML_TYPE_IQ1_M = 29, +# GGML_TYPE_COUNT, +# }; +GGML_TYPE_F32 = 0 +GGML_TYPE_F16 = 1 +GGML_TYPE_Q4_0 = 2 +GGML_TYPE_Q4_1 = 3 +GGML_TYPE_Q5_0 = 6 +GGML_TYPE_Q5_1 = 7 +GGML_TYPE_Q8_0 = 8 +GGML_TYPE_Q8_1 = 9 +GGML_TYPE_Q2_K = 10 +GGML_TYPE_Q3_K = 11 +GGML_TYPE_Q4_K = 12 +GGML_TYPE_Q5_K = 13 +GGML_TYPE_Q6_K = 14 +GGML_TYPE_Q8_K = 15 +GGML_TYPE_IQ2_XXS = 16 +GGML_TYPE_IQ2_XS = 17 +GGML_TYPE_IQ3_XXS = 18 +GGML_TYPE_IQ1_S = 19 +GGML_TYPE_IQ4_NL = 20 +GGML_TYPE_IQ3_S = 21 +GGML_TYPE_IQ2_S = 22 +GGML_TYPE_IQ4_XS = 23 +GGML_TYPE_I8 = 24 +GGML_TYPE_I16 = 25 +GGML_TYPE_I32 = 26 +GGML_TYPE_I64 = 27 +GGML_TYPE_F64 = 28 +GGML_TYPE_IQ1_M = 29 +GGML_TYPE_COUNT = 30 # from ggml-backend.h # typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index dace8d547a..c24fca6523 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -175,6 +175,9 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler=chat_handler, # Speculative Decoding draft_model=draft_model, + # KV Cache Quantization + type_k=settings.type_k, + type_v=settings.type_v, # Tokenizer tokenizer=tokenizer, # Misc diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index daa913face..9ebdd0d8c5 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -159,6 +159,15 @@ class ModelSettings(BaseSettings): default=10, description="Number of tokens to predict using the draft model.", ) + # KV Cache Quantization + type_k: Optional[int] = Field( + default=None, + description="Type of the key cache quantization.", + ) + type_v: Optional[int] = Field( + default=None, + description="Type of the value cache quantization.", + ) # Misc verbose: bool = Field( default=True, description="Whether to print debug information." From a0f373e31009d678312ea8d9a3c70bac96232490 Mon Sep 17 00:00:00 2001 From: lawfordp2017 Date: Mon, 1 Apr 2024 08:21:00 -0600 Subject: [PATCH 040/561] fix: Changed local API doc references to hosted (#1317) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3323f3899b..1e2b640182 100644 --- a/README.md +++ b/README.md @@ -321,7 +321,7 @@ For OpenAI API v1 compatibility, you use the [`create_chat_completion_openai_v1` ### JSON and JSON Schema Mode -To constrain chat responses to only valid JSON or a specific JSON Schema use the `response_format` argument in [`create_chat_completion`](http://localhost:8000/api-reference/#llama_cpp.Llama.create_chat_completion). +To constrain chat responses to only valid JSON or a specific JSON Schema use the `response_format` argument in [`create_chat_completion`](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_chat_completion). #### JSON Mode @@ -529,7 +529,7 @@ llama = Llama( ### Embeddings -To generate text embeddings use [`create_embedding`](http://localhost:8000/api-reference/#llama_cpp.Llama.create_embedding). +To generate text embeddings use [`create_embedding`](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_embedding). ```python import llama_cpp From 45bf5ae5829aab6d38f5920b620f653fb9261f16 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 Apr 2024 10:28:22 -0400 Subject: [PATCH 041/561] chore: Bump version --- CHANGELOG.md | 11 ++++++++++- llama_cpp/__init__.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a85eaa4362..8786c01c8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.58] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ba0c7c70ab5b15f1f2be7fb0dfbe0366dda30d6c +- feat: add support for KV cache quantization options by @Limour-dev in #1307 +- feat: Add logprobs support to chat completions by @windspirit95 in #1311 +- fix: set LLAMA_METAL_EMBED_LIBRARY=on on MacOS arm64 by @bretello in #1289 +- feat: Add tools/functions variables to Jinja2ChatFormatter, add function response formatting for all simple chat formats by @CISC in #1273 +- fix: Changed local API doc references to hosted by by @lawfordp2017 in #1317 + ## [0.2.57] - feat: Update llama.cpp to ggerganov/llama.cpp@ac9ee6a4ad740bc1ee484ede43e9f92b5af244c1 @@ -24,7 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.2.55] -- feat: Update llama.cpp to ggerganov/9731134296af3a6839cd682e51d9c2109a871de5 +- feat: Update llama.cpp to ggerganov/llama.cpp@9731134296af3a6839cd682e51d9c2109a871de5 - docs: fix small typo in README: 'model know how' -> 'model knows how' by @boegel in #1244 ## [0.2.54] diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 1e802fa7e3..e246595cc1 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.57" \ No newline at end of file +__version__ = "0.2.58" \ No newline at end of file From 62aad610e155fdb74064316ee7e83a1251bc8fc0 Mon Sep 17 00:00:00 2001 From: Yuri Mikhailov Date: Tue, 2 Apr 2024 04:25:43 +0900 Subject: [PATCH 042/561] fix: last tokens passing to sample_repetition_penalties function (#1295) Co-authored-by: ymikhaylov Co-authored-by: Andrei --- llama_cpp/_internals.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 22d0bef8a1..79f6543a89 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -730,12 +730,14 @@ def sample( if len(self.prev) > 0: nl_token = ctx_main.model.token_nl() nl_logit = logits_array[nl_token] - if self.params.penalty_last_n > 0: + last_tokens = self.prev[-self.params.penalty_last_n:] + last_tokens_size = min(len(last_tokens), self.params.penalty_last_n) + if last_tokens_size > 0: + last_tokens_p = (llama_cpp.llama_token * len(last_tokens))(*last_tokens) ctx_main.sample_repetition_penalties( token_data_array, - # TODO: Only create this once - (llama_cpp.llama_token * len(self.prev))(*self.prev), - self.params.penalty_last_n, + last_tokens_p, + last_tokens_size, self.params.penalty_repeat, self.params.penalty_freq, self.params.penalty_present, From e465157804b2aa734ac3b8f7957d3a92acf13fa9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 Apr 2024 00:55:19 -0400 Subject: [PATCH 043/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ba0c7c70ab..52604860f9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ba0c7c70ab5b15f1f2be7fb0dfbe0366dda30d6c +Subproject commit 52604860f93063ef98863921da697576af1c7665 From 8649d7671bd1a7c0d9cc6a5ad91c6ca286512ab3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 Apr 2024 15:30:31 -0400 Subject: [PATCH 044/561] fix: segfault when logits_all=False. Closes #1319 --- llama_cpp/llama.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index dcc7be7580..e07d57ae4b 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -535,14 +535,16 @@ def eval(self, tokens: Sequence[int]): # Save tokens self.input_ids[n_past : n_past + n_tokens] = batch # Save logits - rows = n_tokens - cols = self._n_vocab - offset = ( - 0 if self.context_params.logits_all else n_tokens - 1 - ) # NOTE: Only save the last token logits if logits_all is False - self.scores[n_past + offset : n_past + n_tokens, :].reshape(-1)[ - : - ] = self._ctx.get_logits()[offset * cols : rows * cols] + if self.context_params.logits_all: + rows = n_tokens + cols = self._n_vocab + logits = self._ctx.get_logits()[: rows * cols] + self.scores[n_past : n_past + n_tokens, :].reshape(-1)[: :] = logits + else: + rows = 1 + cols = self._n_vocab + logits = self._ctx.get_logits()[: rows * cols] + self.scores[n_past + n_tokens - 1, :].reshape(-1)[: :] = logits # Update n_tokens self.n_tokens += n_tokens From 5a930ee9a1a54d12d48cdb7ce8cc63f6bc41d1bb Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 3 Apr 2024 15:32:13 -0400 Subject: [PATCH 045/561] feat: Binary wheels for CPU, CUDA (12.1 - 12.3), Metal (#1247) * Generate binary wheel index on release * Add total release downloads badge * Update download label * Use official cibuildwheel action * Add workflows to build CUDA and Metal wheels * Update generate index workflow * Update workflow name --- .github/workflows/build-and-release.yaml | 10 +- .github/workflows/build-wheels-cuda.yaml | 131 ++++++++++++++++++ .github/workflows/build-wheels-metal.yaml | 87 ++++++++++++ .../generate-index-from-release.yaml | 48 +++++++ README.md | 1 + scripts/release-to-pep-503.sh | 58 ++++++++ 6 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/build-wheels-cuda.yaml create mode 100644 .github/workflows/build-wheels-metal.yaml create mode 100644 .github/workflows/generate-index-from-release.yaml create mode 100755 scripts/release-to-pep-503.sh diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 63c81f1e3f..76b5f7fcf0 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-20.04, windows-2019, macos-11] steps: - uses: actions/checkout@v3 @@ -23,19 +23,19 @@ jobs: with: python-version: "3.8" - - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.12.1 - - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install -e .[all] - name: Build wheels - run: python -m cibuildwheel --output-dir wheelhouse + uses: pypa/cibuildwheel@v2.16.5 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" + with: + package-dir: . + output-dir: wheelhouse - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml new file mode 100644 index 0000000000..a222dceb41 --- /dev/null +++ b/.github/workflows/build-wheels-cuda.yaml @@ -0,0 +1,131 @@ +name: Build Wheels (CUDA) + +on: workflow_dispatch + +permissions: + contents: write + +jobs: + define_matrix: + name: Define Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + defaults: + run: + shell: pwsh + + steps: + - name: Define Job Output + id: set-matrix + run: | + $matrix = @{ + 'os' = @('ubuntu-20.04', 'windows-latest') + 'pyver' = @("3.10", "3.11", "3.12") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2") + 'releasetag' = @("basic") + } + + $matrixOut = ConvertTo-Json $matrix -Compress + Write-Output ('matrix=' + $matrixOut) >> $env:GITHUB_OUTPUT + + build_wheels: + name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag == 'wheels' && 'AVX2' || matrix.releasetag }} + needs: define_matrix + runs-on: ${{ matrix.os }} + strategy: + matrix: ${{ fromJSON(needs.define_matrix.outputs.matrix) }} + defaults: + run: + shell: pwsh + env: + CUDAVER: ${{ matrix.cuda }} + AVXVER: ${{ matrix.releasetag }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.pyver }} + + - name: Setup Mamba + uses: conda-incubator/setup-miniconda@v2.2.0 + with: + activate-environment: "build" + python-version: ${{ matrix.pyver }} + miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true + add-pip-as-python-dependency: true + auto-activate-base: false + + - name: VS Integration Cache + id: vs-integration-cache + if: runner.os == 'Windows' + uses: actions/cache@v3.3.2 + with: + path: ./MSBuildExtensions + key: cuda-${{ matrix.cuda }}-vs-integration + + - name: Get Visual Studio Integration + if: runner.os == 'Windows' && steps.vs-integration-cache.outputs.cache-hit != 'true' + run: | + if ($env:CUDAVER -eq '12.1.1') {$x = '12.1.0'} else {$x = $env:CUDAVER} + $links = (Invoke-RestMethod 'https://github.com/Jimver/cuda-toolkit/raw/dc0ca7bb29c5a92f7a963d3d5c93f8d59765136a/src/links/windows-links.ts').Trim().split().where({$_ -ne ''}) + for ($i=$q=0;$i -lt $links.count -and $q -lt 2;$i++) {if ($links[$i] -eq "'$x',") {$q++}} + Invoke-RestMethod $links[$i].Trim("'") -OutFile 'cudainstaller.zip' + & 'C:\Program Files\7-Zip\7z.exe' e cudainstaller.zip -oMSBuildExtensions -r *\MSBuildExtensions\* > $null + Remove-Item 'cudainstaller.zip' + + - name: Install Visual Studio Integration + if: runner.os == 'Windows' + run: | + $y = (gi '.\MSBuildExtensions').fullname + '\*' + (gi 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Microsoft\VC\*\BuildCustomizations').fullname.foreach({cp $y $_}) + $cupath = 'CUDA_PATH_V' + $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','_') + echo "$cupath=$env:CONDA_PREFIX" >> $env:GITHUB_ENV + + - name: Install Dependencies + env: + MAMBA_DOWNLOAD_FAILFAST: "0" + MAMBA_NO_LOW_SPEED_LIMIT: "1" + run: | + $cudaVersion = $env:CUDAVER + mamba install -y 'cuda' -c nvidia/label/cuda-$cudaVersion + python -m pip install build wheel + + - name: Build Wheel + run: | + $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','') + $env:CUDA_PATH = $env:CONDA_PREFIX + $env:CUDA_HOME = $env:CONDA_PREFIX + $env:CUDA_TOOLKIT_ROOT_DIR = $env:CONDA_PREFIX + if ($IsLinux) { + $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH + } + $env:VERBOSE = '1' + $env:CMAKE_ARGS = '-DLLAMA_CUBLAS=on -DCMAKE_CUDA_ARCHITECTURES=all' + $env:CMAKE_ARGS = "-DLLAMA_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS" + if ($env:AVXVER -eq 'AVX') { + $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off' + } + if ($env:AVXVER -eq 'AVX512') { + $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX512=on' + } + if ($env:AVXVER -eq 'basic') { + $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off' + } + python -m build --wheel + # write the build tag to the output + Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV + + - uses: softprops/action-gh-release@v1 + with: + files: dist/* + # Set tag_name to -cu + tag_name: ${{ github.ref_name }}-cu${{ env.CUDA_VERSION }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml new file mode 100644 index 0000000000..2cca477860 --- /dev/null +++ b/.github/workflows/build-wheels-metal.yaml @@ -0,0 +1,87 @@ +name: Build Wheels (Metal) + +on: workflow_dispatch + +permissions: + contents: write + +jobs: + define_matrix: + name: Define Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + defaults: + run: + shell: pwsh + + steps: + - name: Define Job Output + id: set-matrix + run: | + $matrix = @{ + 'os' = @('macos-11', 'macos-12', 'macos-13') + 'pyver' = @('3.10', '3.11', '3.12') + } + + $matrixOut = ConvertTo-Json $matrix -Compress + Write-Output ('matrix=' + $matrixOut) >> $env:GITHUB_OUTPUT + + build_wheels: + name: ${{ matrix.os }} Python ${{ matrix.pyver }} + needs: define_matrix + runs-on: ${{ matrix.os }} + strategy: + matrix: ${{ fromJSON(needs.define_matrix.outputs.matrix) }} + env: + OSVER: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.pyver }} + + - name: Install Dependencies + run: | + python -m pip install build wheel cmake + + - name: Build Wheel + run: | + XCODE15PATH="/Applications/Xcode_15.0.app/Contents/Developer" + XCODE15BINPATH="${XCODE15PATH}/Toolchains/XcodeDefault.xctoolchain/usr/bin" + export CMAKE_ARGS="-DLLAMA_NATIVE=off -DLLAMA_METAL=on" + [[ "$OSVER" == "macos-13" ]] && export CC="${XCODE15BINPATH}/cc" && export CXX="${XCODE15BINPATH}/c++" && export MACOSX_DEPLOYMENT_TARGET="13.0" + [[ "$OSVER" == "macos-12" ]] && export MACOSX_DEPLOYMENT_TARGET="12.0" + [[ "$OSVER" == "macos-11" ]] && export MACOSX_DEPLOYMENT_TARGET="11.0" + + export CMAKE_OSX_ARCHITECTURES="arm64" && export ARCHFLAGS="-arch arm64" + VERBOSE=1 python -m build --wheel + + if [[ "$OSVER" == "macos-13" ]]; then + export SDKROOT="${XCODE15PATH}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.0.sdk" + export MACOSX_DEPLOYMENT_TARGET="14.0" + VERBOSE=1 python -m build --wheel + fi + + for file in ./dist/*.whl; do cp "$file" "${file/arm64.whl/aarch64.whl}"; done + + export CMAKE_OSX_ARCHITECTURES="x86_64" && export CMAKE_ARGS="-DLLAMA_NATIVE=off -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off -DLLAMA_METAL=on" && export ARCHFLAGS="-arch x86_64" + VERBOSE=1 python -m build --wheel + + if [[ "$OSVER" == "macos-13" ]]; then + export SDKROOT="${XCODE15PATH}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.0.sdk" + export MACOSX_DEPLOYMENT_TARGET="14.0" + VERBOSE=1 python -m build --wheel + fi + + - uses: softprops/action-gh-release@v1 + with: + files: dist/* + # set release name to -metal + tag_name: ${{ github.ref_name }}-metal + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml new file mode 100644 index 0000000000..9042d6cfea --- /dev/null +++ b/.github/workflows/generate-index-from-release.yaml @@ -0,0 +1,48 @@ +name: Wheels Index + +on: + # Trigger on any new release + release: + types: [published] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + # Single deploy job since we're just deploying + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Pages + uses: actions/configure-pages@v4 + - name: Build + run: | + ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$' + ./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$' + ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' + ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + # Upload entire repository + path: 'index' + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/README.md b/README.md index 1e2b640182..f251f45f96 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/) [![PyPI - License](https://img.shields.io/pypi/l/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/) +[![Github All Releases](https://img.shields.io/github/downloads/abetlen/llama-cpp-python/total.svg?label=Github%20Downloads)]() Simple Python bindings for **@ggerganov's** [`llama.cpp`](https://github.com/ggerganov/llama.cpp) library. This package provides: diff --git a/scripts/release-to-pep-503.sh b/scripts/release-to-pep-503.sh new file mode 100755 index 0000000000..00ae56721e --- /dev/null +++ b/scripts/release-to-pep-503.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Get output directory or default to index/whl/cpu +output_dir=${1:-"index/whl/cpu"} + +# Create output directory +mkdir -p $output_dir + +# Change to output directory +pushd $output_dir + +# Create an index html file +echo "" > index.html +echo "" >> index.html +echo " " >> index.html +echo " " >> index.html +echo " llama-cpp-python" >> index.html +echo "
" >> index.html +echo " " >> index.html +echo "" >> index.html +echo "" >> index.html + +# Create llama-cpp-python directory +mkdir -p llama-cpp-python + +# Change to llama-cpp-python directory +pushd llama-cpp-python + +# Create an index html file +echo "" > index.html +echo "" >> index.html +echo " " >> index.html +echo "

Links for llama-cpp-python

" >> index.html + +# Get all releases +releases=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases | jq -r .[].tag_name) + +# Get pattern from second arg or default to valid python package version pattern +pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"} + +# Filter releases by pattern +releases=$(echo $releases | tr ' ' '\n' | grep -E $pattern) + +# For each release, get all assets +for release in $releases; do + assets=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release | jq -r .assets) + echo "

$release

" >> index.html + for asset in $(echo $assets | jq -r .[].browser_download_url); do + if [[ $asset == *".whl" ]]; then + echo " $asset" >> index.html + echo "
" >> index.html + fi + done +done + +echo " " >> index.html +echo "" >> index.html +echo "" >> index.html From 5a5193636b4fe6c1b186bf3c8ee18e1b96401bed Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 Apr 2024 15:35:28 -0400 Subject: [PATCH 046/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 52604860f9..60cdf40cc3 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 52604860f93063ef98863921da697576af1c7665 +Subproject commit 60cdf40cc32f0ad4cb11e0ca8fd38f3b93d8d640 From 34081ddc5bbe2fe667ab201abcdbda64c5caa49c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 Apr 2024 15:38:27 -0400 Subject: [PATCH 047/561] chore: Bump version --- CHANGELOG.md | 7 +++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8786c01c8b..fc4e29b38d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.59] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ba0c7c70ab5b15f1f2be7fb0dfbe0366dda30d6c +- feat: Binary wheels for CPU, CUDA (12.1 - 12.3), Metal by @abetlen, @jllllll, and @oobabooga in #1247 +- fix: segfault when logits_all=False by @abetlen in 8649d7671bd1a7c0d9cc6a5ad91c6ca286512ab3 +- fix: last tokens passing to sample_repetition_penalties function by @ymikhailov in #1295 + ## [0.2.58] - feat: Update llama.cpp to ggerganov/llama.cpp@ba0c7c70ab5b15f1f2be7fb0dfbe0366dda30d6c diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index e246595cc1..2f5219cc83 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.58" \ No newline at end of file +__version__ = "0.2.59" \ No newline at end of file From 612e78d32297002b52b264f29ae32bcd85ad54c6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 3 Apr 2024 16:15:29 -0400 Subject: [PATCH 048/561] fix(ci): use correct script name --- scripts/{release-to-pep-503.sh => releases-to-pep-503.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{release-to-pep-503.sh => releases-to-pep-503.sh} (100%) diff --git a/scripts/release-to-pep-503.sh b/scripts/releases-to-pep-503.sh similarity index 100% rename from scripts/release-to-pep-503.sh rename to scripts/releases-to-pep-503.sh From c50309e52ab90ffc71ebfc083f7397d8d79851eb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 4 Apr 2024 02:49:19 -0400 Subject: [PATCH 049/561] docs: LLAMA_CUBLAS -> LLAMA_CUDA --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f251f45f96..5e16614f28 100644 --- a/README.md +++ b/README.md @@ -102,10 +102,10 @@ CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-
cuBLAS (CUDA) -To install with cuBLAS, set the `LLAMA_CUBLAS=on` environment variable before installing: +To install with cuBLAS, set the `LLAMA_CUDA=on` environment variable before installing: ```bash -CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python +CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python ```
@@ -569,7 +569,7 @@ python3 -m llama_cpp.server --model models/7B/llama-model.gguf Similar to Hardware Acceleration section above, you can also install with GPU (cuBLAS) support like this: ```bash -CMAKE_ARGS="-DLLAMA_CUBLAS=on" FORCE_CMAKE=1 pip install 'llama-cpp-python[server]' +CMAKE_ARGS="-DLLAMA_CUDA=on" FORCE_CMAKE=1 pip install 'llama-cpp-python[server]' python3 -m llama_cpp.server --model models/7B/llama-model.gguf --n_gpu_layers 35 ``` From 1db3b58fdc7c216749401a2901fc64495032f1ff Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 4 Apr 2024 02:57:06 -0400 Subject: [PATCH 050/561] docs: Add docs explaining how to install pre-built wheels. --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/README.md b/README.md index 5e16614f28..a0ef83ccf2 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,15 @@ This will also build `llama.cpp` from source and install it alongside this pytho If this fails, add `--verbose` to the `pip install` see the full cmake build log. +**Pre-built Wheel (New)** + +It is also possible to install a pre-built wheel with basic CPU support. + +```bash +pip install llama-cpp-python \ + --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu +``` + ### Installation Configuration `llama.cpp` supports a number of hardware acceleration backends to speed up inference as well as backend specific options. See the [llama.cpp README](https://github.com/ggerganov/llama.cpp#build) for a full list. @@ -108,6 +117,30 @@ To install with cuBLAS, set the `LLAMA_CUDA=on` environment variable before inst CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python ``` +**Pre-built Wheel (New)** + +It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements: + +- CUDA Version is 12.1, 12.2 or 12.3 +- Python Version is 3.10, 3.11 or 3.12 + +```bash +pip install llama-cpp-python \ + --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/ +``` + +Where `` is one of the following: +- `cu121`: CUDA 12.1 +- `cu122`: CUDA 12.2 +- `cu123`: CUDA 12.3 + +For example, to install the CUDA 12.1 wheel: + +```bash +pip install llama-cpp-python \ + --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121 +``` +
@@ -119,6 +152,18 @@ To install with Metal (MPS), set the `LLAMA_METAL=on` environment variable befor CMAKE_ARGS="-DLLAMA_METAL=on" pip install llama-cpp-python ``` +**Pre-built Wheel (New)** + +It is also possible to install a pre-built wheel with Metal support. As long as your system meets some requirements: + +- MacOS Version is 11.0 or later +- Python Version is 3.10, 3.11 or 3.12 + +```bash +pip install llama-cpp-python \ + --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/metal +``` +
From 909ef66951fb9f3f5c724841b0c98a598a2e0802 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 4 Apr 2024 03:08:47 -0400 Subject: [PATCH 051/561] docs: Rename cuBLAS section to CUDA --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a0ef83ccf2..c4e194bfab 100644 --- a/README.md +++ b/README.md @@ -109,9 +109,9 @@ CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-
-cuBLAS (CUDA) +CUDA -To install with cuBLAS, set the `LLAMA_CUDA=on` environment variable before installing: +To install with CUDA support, set the `LLAMA_CUDA=on` environment variable before installing: ```bash CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python From 7265a5dc0eb230a646ad868bad7ed5160d815328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Fri, 5 Apr 2024 15:14:03 +0200 Subject: [PATCH 052/561] fix(docs): incorrect tool_choice example (#1330) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c4e194bfab..cd41a3804e 100644 --- a/README.md +++ b/README.md @@ -458,12 +458,12 @@ The high-level API supports OpenAI compatible function and tool calling. This is } } }], - tool_choice=[{ + tool_choice={ "type": "function", "function": { "name": "UserDetail" } - }] + } ) ``` From 9111b6e03aba199e00cc7f3c7b6df2a19420665c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 5 Apr 2024 09:21:02 -0400 Subject: [PATCH 053/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 60cdf40cc3..a307375c02 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 60cdf40cc32f0ad4cb11e0ca8fd38f3b93d8d640 +Subproject commit a307375c02cac45cff53cf2520330b43fecc7718 From 49bc66bfa255f0c6d421cf142ecdbfb967546a51 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 5 Apr 2024 10:50:49 -0400 Subject: [PATCH 054/561] fix: missing logprobs in response, incorrect response type for functionary, minor type issues. Closes #1328 #1314 --- llama_cpp/llama_chat_format.py | 48 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 06cf9ced7d..705202e8c4 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -6,7 +6,7 @@ import dataclasses import random import string -from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Union, Protocol +from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Union, Protocol, cast import jinja2 @@ -338,6 +338,7 @@ def _convert_completion_to_chat_function( } ], }, + "logprobs": None, "finish_reason": "tool_calls", } ], @@ -1191,7 +1192,6 @@ def format_mistral_instruct( elif ( message["role"] == "assistant" and message["content"] is not None - and isinstance(message["content"], str) ): prompt += " [/INST]" + message["content"] + eos prompt += " [/INST]" @@ -1263,7 +1263,7 @@ def format_gemma( **kwargs: Any, ) -> ChatFormatterResponse: system_message = _get_system_message(messages) - if system_message is not None and system_message != "": + if system_message != "": logger.debug( "`role='system'` messages are not allowed on Google's Gemma models." ) @@ -1628,6 +1628,7 @@ def message_to_str(msg: llama_types.ChatCompletionRequestMessage): } ], }, + "logprobs": None, "finish_reason": "tool_calls", } ], @@ -1909,14 +1910,14 @@ def get_grammar(function_call): return grammar def create_completion(stop): - completion: llama_types.Completion = llama.create_completion( + completion = cast(llama_types.Completion, llama.create_completion( prompt=prompt, temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p, typical_p=typical_p, - stream=stream, + stream=False, stop=stop, max_tokens=max_tokens, presence_penalty=presence_penalty, @@ -1929,7 +1930,7 @@ def create_completion(stop): model=model, logits_processor=logits_processor, grammar=grammar, - ) + )) return completion @@ -2050,7 +2051,7 @@ def create_completion(stop): assert "usage" in completion assert len(function_calls) == len(function_bodies) - tool_calls = [] + tool_calls: List[llama_types.ChatCompletionMessageToolCall] = [] for function_call, function_body in zip(function_calls, function_bodies): tool_calls.append( { @@ -2070,6 +2071,12 @@ def create_completion(stop): ) # TODO: support stream mode + function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = { + "function_call": { + "name": tool_calls[0]["function"]["name"], + "arguments": tool_calls[0]["function"]["arguments"], + } + } if len(tool_calls) == 1 else {} return llama_types.CreateChatCompletionResponse( id="chat" + completion["id"], object="chat.completion", @@ -2078,14 +2085,12 @@ def create_completion(stop): choices=[ { "index": 0, + "logprobs": None, "message": { "role": "assistant", "content": None if content == "" else content, - "function_call": { - "name": tool_calls[0]["function"]["name"], - "arguments": tool_calls[0]["function"]["arguments"], - } if len(tool_calls) > 0 else None, - "tool_calls": tool_calls if len(tool_calls) > 0 else None, + "tool_calls": tool_calls, + **function_call_dict, }, "finish_reason": "tool_calls" if len(tool_calls) > 0 else "stop", } @@ -2565,8 +2570,8 @@ def chatml_function_calling( tool_name = text[len("functions.") :] tool = next((tool for tool in tools if tool["function"]["name"] == tool_name), None) if not stream: - completions = [] - completions_tool_name = [] + completions: List[llama_types.CreateCompletionResponse] = [] + completions_tool_name: List[str] = [] while tool is not None: prompt += f"functions.{tool_name}:\n" try: @@ -2603,6 +2608,7 @@ def chatml_function_calling( logits_processor=logits_processor, grammar=grammar, ) + completion_or_chunks = cast(llama_types.CreateCompletionResponse, completion_or_chunks) completions.append(completion_or_chunks) completions_tool_name.append(tool_name) prompt += completion_or_chunks["choices"][0]["text"] @@ -2631,6 +2637,7 @@ def chatml_function_calling( follow_up_gbnf_tool_grammar, verbose=llama.verbose ), ) + response = cast(llama_types.CreateCompletionResponse, response) tool_name = response["choices"][0]["text"][len("functions.") :] tool = next( @@ -2638,7 +2645,7 @@ def chatml_function_calling( ) # Merge completions - function_call = { + function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = { "function_call": { "name": tool_name, "arguments": completions[0]["choices"][0]["text"], @@ -2653,6 +2660,7 @@ def chatml_function_calling( { "finish_reason": "tool_calls", "index": 0, + "logprobs": None, "message": { "role": "assistant", "content": None, @@ -2673,20 +2681,22 @@ def chatml_function_calling( zip(completions_tool_name, completions) ) ], - **function_call + **function_call_dict }, } ], "usage": { "completion_tokens": sum( - completion["usage"]["completion_tokens"] + completion["usage"]["completion_tokens"] if "usage" in completion else 0 for completion in completions ), "prompt_tokens": sum( - completion["usage"]["prompt_tokens"] for completion in completions + completion["usage"]["prompt_tokens"] if "usage" in completion else 0 + for completion in completions ), "total_tokens": sum( - completion["usage"]["total_tokens"] for completion in completions + completion["usage"]["total_tokens"] if "usage" in completion else 0 + for completion in completions ), }, } From 1ae3abbcc3af7f4a25a3ffc40b246f18039565e8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 5 Apr 2024 10:50:49 -0400 Subject: [PATCH 055/561] fix: missing logprobs in response, incorrect response type for functionary, minor type issues. Closes #1328 Closes #1314 --- llama_cpp/llama_chat_format.py | 48 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 06cf9ced7d..705202e8c4 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -6,7 +6,7 @@ import dataclasses import random import string -from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Union, Protocol +from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Union, Protocol, cast import jinja2 @@ -338,6 +338,7 @@ def _convert_completion_to_chat_function( } ], }, + "logprobs": None, "finish_reason": "tool_calls", } ], @@ -1191,7 +1192,6 @@ def format_mistral_instruct( elif ( message["role"] == "assistant" and message["content"] is not None - and isinstance(message["content"], str) ): prompt += " [/INST]" + message["content"] + eos prompt += " [/INST]" @@ -1263,7 +1263,7 @@ def format_gemma( **kwargs: Any, ) -> ChatFormatterResponse: system_message = _get_system_message(messages) - if system_message is not None and system_message != "": + if system_message != "": logger.debug( "`role='system'` messages are not allowed on Google's Gemma models." ) @@ -1628,6 +1628,7 @@ def message_to_str(msg: llama_types.ChatCompletionRequestMessage): } ], }, + "logprobs": None, "finish_reason": "tool_calls", } ], @@ -1909,14 +1910,14 @@ def get_grammar(function_call): return grammar def create_completion(stop): - completion: llama_types.Completion = llama.create_completion( + completion = cast(llama_types.Completion, llama.create_completion( prompt=prompt, temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p, typical_p=typical_p, - stream=stream, + stream=False, stop=stop, max_tokens=max_tokens, presence_penalty=presence_penalty, @@ -1929,7 +1930,7 @@ def create_completion(stop): model=model, logits_processor=logits_processor, grammar=grammar, - ) + )) return completion @@ -2050,7 +2051,7 @@ def create_completion(stop): assert "usage" in completion assert len(function_calls) == len(function_bodies) - tool_calls = [] + tool_calls: List[llama_types.ChatCompletionMessageToolCall] = [] for function_call, function_body in zip(function_calls, function_bodies): tool_calls.append( { @@ -2070,6 +2071,12 @@ def create_completion(stop): ) # TODO: support stream mode + function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = { + "function_call": { + "name": tool_calls[0]["function"]["name"], + "arguments": tool_calls[0]["function"]["arguments"], + } + } if len(tool_calls) == 1 else {} return llama_types.CreateChatCompletionResponse( id="chat" + completion["id"], object="chat.completion", @@ -2078,14 +2085,12 @@ def create_completion(stop): choices=[ { "index": 0, + "logprobs": None, "message": { "role": "assistant", "content": None if content == "" else content, - "function_call": { - "name": tool_calls[0]["function"]["name"], - "arguments": tool_calls[0]["function"]["arguments"], - } if len(tool_calls) > 0 else None, - "tool_calls": tool_calls if len(tool_calls) > 0 else None, + "tool_calls": tool_calls, + **function_call_dict, }, "finish_reason": "tool_calls" if len(tool_calls) > 0 else "stop", } @@ -2565,8 +2570,8 @@ def chatml_function_calling( tool_name = text[len("functions.") :] tool = next((tool for tool in tools if tool["function"]["name"] == tool_name), None) if not stream: - completions = [] - completions_tool_name = [] + completions: List[llama_types.CreateCompletionResponse] = [] + completions_tool_name: List[str] = [] while tool is not None: prompt += f"functions.{tool_name}:\n" try: @@ -2603,6 +2608,7 @@ def chatml_function_calling( logits_processor=logits_processor, grammar=grammar, ) + completion_or_chunks = cast(llama_types.CreateCompletionResponse, completion_or_chunks) completions.append(completion_or_chunks) completions_tool_name.append(tool_name) prompt += completion_or_chunks["choices"][0]["text"] @@ -2631,6 +2637,7 @@ def chatml_function_calling( follow_up_gbnf_tool_grammar, verbose=llama.verbose ), ) + response = cast(llama_types.CreateCompletionResponse, response) tool_name = response["choices"][0]["text"][len("functions.") :] tool = next( @@ -2638,7 +2645,7 @@ def chatml_function_calling( ) # Merge completions - function_call = { + function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = { "function_call": { "name": tool_name, "arguments": completions[0]["choices"][0]["text"], @@ -2653,6 +2660,7 @@ def chatml_function_calling( { "finish_reason": "tool_calls", "index": 0, + "logprobs": None, "message": { "role": "assistant", "content": None, @@ -2673,20 +2681,22 @@ def chatml_function_calling( zip(completions_tool_name, completions) ) ], - **function_call + **function_call_dict }, } ], "usage": { "completion_tokens": sum( - completion["usage"]["completion_tokens"] + completion["usage"]["completion_tokens"] if "usage" in completion else 0 for completion in completions ), "prompt_tokens": sum( - completion["usage"]["prompt_tokens"] for completion in completions + completion["usage"]["prompt_tokens"] if "usage" in completion else 0 + for completion in completions ), "total_tokens": sum( - completion["usage"]["total_tokens"] for completion in completions + completion["usage"]["total_tokens"] if "usage" in completion else 0 + for completion in completions ), }, } From f4092e6b46711c92723258fac6326074ad51daa7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 5 Apr 2024 10:59:31 -0400 Subject: [PATCH 056/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a307375c02..1b496a745c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a307375c02cac45cff53cf2520330b43fecc7718 +Subproject commit 1b496a745c315022df2d919374052e6004ced8d3 From b3bfea6dbfb6ed9ce18f9a2723e0a9e4bd1da7ad Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 6 Apr 2024 01:36:53 -0400 Subject: [PATCH 057/561] fix: Always embed metal library. Closes #1332 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7415149076..70f9b99936 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ if (LLAMA_BUILD) set(LLAMA_F16C "Off" CACHE BOOL "llama: enable F16C" FORCE) endif() - if (APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") + if (APPLE) set(LLAMA_METAL_EMBED_LIBRARY "On" CACHE BOOL "llama: embed metal library" FORCE) endif() From 7ca364c8bd8669add75eb4143d36651495d5c743 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 6 Apr 2024 01:37:43 -0400 Subject: [PATCH 058/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1b496a745c..75cd4c7729 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1b496a745c315022df2d919374052e6004ced8d3 +Subproject commit 75cd4c77292034ecec587ecb401366f57338f7c0 From 08b16afe11e7b42adec2fed0a781123383476045 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 6 Apr 2024 01:53:38 -0400 Subject: [PATCH 059/561] chore: Bump version --- CHANGELOG.md | 7 +++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc4e29b38d..dcb7a816cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.60] + +- feat: Update llama.cpp to ggerganov/llama.cpp@75cd4c77292034ecec587ecb401366f57338f7c0 +- fix: Always embed metal library by @abetlen in b3bfea6dbfb6ed9ce18f9a2723e0a9e4bd1da7ad +- fix: missing logprobs in response, incorrect response type for functionary by @abetlen in 1ae3abbcc3af7f4a25a3ffc40b246f18039565e8 +- fix(docs): incorrect tool_choice example by @CISC in #1330 + ## [0.2.59] - feat: Update llama.cpp to ggerganov/llama.cpp@ba0c7c70ab5b15f1f2be7fb0dfbe0366dda30d6c diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 2f5219cc83..d5db993d95 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.59" \ No newline at end of file +__version__ = "0.2.60" \ No newline at end of file From 56071c956a947355ae63fe878448fe03df3b7586 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Apr 2024 09:53:49 -0400 Subject: [PATCH 060/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 237 +++++++++++++++++++++++++++++++++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 231 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index accc02cb85..879308595e 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -237,11 +237,18 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' LLAMA_FILE_MAGIC_GGSN = 0x6767736E +#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq' +LLAMA_FILE_MAGIC_GGSQ = 0x67677371 + # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN # define LLAMA_SESSION_VERSION 5 LLAMA_SESSION_VERSION = 5 +#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ +LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ +#define LLAMA_STATE_SEQ_VERSION 1 +LLAMA_STATE_SEQ_VERSION = 1 # struct llama_model; llama_model_p = NewType("llama_model_p", int) @@ -1467,6 +1474,7 @@ def llama_kv_cache_clear(ctx: llama_context_p, /): # // Removes all tokens that belong to the specified sequence and have positions in [p0, p1) +# // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails # // seq_id < 0 : match any sequence # // p0 < 0 : [0, p1] # // p1 < 0 : [p0, inf) @@ -1493,6 +1501,9 @@ def llama_kv_cache_seq_rm( /, ) -> bool: """Removes all tokens that belong to the specified sequence and have positions in [p0, p1) + + Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails + seq_id < 0 : match any sequence p0 < 0 : [0, p1] p1 < 0 : [p0, inf)""" @@ -1652,7 +1663,16 @@ def llama_kv_cache_update(ctx: llama_context_p, /): # Returns the maximum size in bytes of the state (rng, logits, embedding # and kv_cache) - will often be smaller after compacting tokens -# LLAMA_API size_t llama_get_state_size(const struct llama_context * ctx); +# LLAMA_API size_t llama_state_get_size(const struct llama_context * ctx); +@ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) +def llama_state_get_size(ctx: llama_context_p, /) -> int: + """Returns the maximum size in bytes of the state (rng, logits, embedding + and kv_cache) - will often be smaller after compacting tokens""" + ... + + +# LLAMA_API DEPRECATED(size_t llama_get_state_size(const struct llama_context * ctx), +# "use llama_state_get_size instead"); @ctypes_function("llama_get_state_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_get_state_size(ctx: llama_context_p, /) -> int: """Returns the maximum size in bytes of the state (rng, logits, embedding @@ -1663,9 +1683,30 @@ def llama_get_state_size(ctx: llama_context_p, /) -> int: # Copies the state to the specified destination address. # Destination needs to have allocated enough memory. # Returns the number of bytes copied -# LLAMA_API size_t llama_copy_state_data( +# LLAMA_API size_t llama_state_get_data( # struct llama_context * ctx, # uint8_t * dst); +@ctypes_function( + "llama_state_get_data", + [ + llama_context_p_ctypes, + ctypes.POINTER(ctypes.c_uint8), + ], + ctypes.c_size_t, +) +def llama_state_get_data( + ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], / +) -> int: + """Copies the state to the specified destination address. + Destination needs to have allocated enough memory. + Returns the number of bytes copied""" + ... + + +# LLAMA_API DEPRECATED(size_t llama_copy_state_data( +# struct llama_context * ctx, +# uint8_t * dst), +# "use llama_state_get_data instead"); @ctypes_function( "llama_copy_state_data", [ @@ -1685,9 +1726,26 @@ def llama_copy_state_data( # // Set the state reading from the specified address # // Returns the number of bytes read -# LLAMA_API size_t llama_set_state_data( +# LLAMA_API size_t llama_state_set_data( # struct llama_context * ctx, # const uint8_t * src); +@ctypes_function( + "llama_state_set_data", + [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)], + ctypes.c_size_t, +) +def llama_state_set_data( + ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], / +) -> int: + """Set the state reading from the specified address + Returns the number of bytes read""" + ... + + +# LLAMA_API DEPRECATED(size_t llama_set_state_data( +# struct llama_context * ctx, +# const uint8_t * src), +# "use llama_state_set_data instead"); @ctypes_function( "llama_set_state_data", [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)], @@ -1701,12 +1759,40 @@ def llama_set_state_data( # Save/load session file -# LLAMA_API bool llama_load_session_file( +# LLAMA_API bool llama_state_load_file( # struct llama_context * ctx, # const char * path_session, # llama_token * tokens_out, # size_t n_token_capacity, # size_t * n_token_count_out); +@ctypes_function( + "llama_state_load_file", + [ + llama_context_p_ctypes, + ctypes.c_char_p, + llama_token_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_size_t), + ], + ctypes.c_bool, +) +def llama_state_load_file( + ctx: llama_context_p, + path_session: bytes, + tokens_out: CtypesArray[llama_token], + n_token_capacity: Union[ctypes.c_size_t, int], + n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], + /, +) -> bool: ... + + +# LLAMA_API DEPRECATED(bool llama_load_session_file( +# struct llama_context * ctx, +# const char * path_session, +# llama_token * tokens_out, +# size_t n_token_capacity, +# size_t * n_token_count_out), +# "use llama_state_load_file instead"); @ctypes_function( "llama_load_session_file", [ @@ -1728,11 +1814,36 @@ def llama_load_session_file( ) -> int: ... -# LLAMA_API bool llama_save_session_file( +# LLAMA_API bool llama_state_save_file( # struct llama_context * ctx, # const char * path_session, # const llama_token * tokens, # size_t n_token_count); +@ctypes_function( + "llama_state_save_file", + [ + llama_context_p_ctypes, + ctypes.c_char_p, + llama_token_p, + ctypes.c_size_t, + ], + ctypes.c_bool, +) +def llama_state_save_file( + ctx: llama_context_p, + path_session: bytes, + tokens: CtypesArray[llama_token], + n_token_count: Union[ctypes.c_size_t, int], + /, +) -> bool: ... + + +# LLAMA_API DEPRECATED(bool llama_save_session_file( +# struct llama_context * ctx, +# const char * path_session, +# const llama_token * tokens, +# size_t n_token_count), +# "use llama_state_save_file instead"); @ctypes_function( "llama_save_session_file", [ @@ -1752,6 +1863,116 @@ def llama_save_session_file( ) -> int: ... +# // Get the exact size needed to copy the KV cache of a single sequence +# LLAMA_API size_t llama_state_seq_get_size( +# struct llama_context * ctx, +# llama_seq_id seq_id); +@ctypes_function( + "llama_state_seq_get_size", + [llama_context_p_ctypes, llama_seq_id], + ctypes.c_size_t, +) +def llama_state_seq_get_size(ctx: llama_context_p, seq_id: llama_seq_id, /) -> int: + """Get the exact size needed to copy the KV cache of a single sequence""" + ... + + +# // Copy the KV cache of a single sequence into the specified buffer +# LLAMA_API size_t llama_state_seq_get_data( +# struct llama_context * ctx, +# uint8_t * dst, +# llama_seq_id seq_id); +@ctypes_function( + "llama_state_seq_get_data", + [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), llama_seq_id], + ctypes.c_size_t, +) +def llama_state_seq_get_data( + ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], seq_id: llama_seq_id, / +) -> int: + """Copy the KV cache of a single sequence into the specified buffer""" + ... + + +# // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence +# // Returns: +# // - Positive: Ok +# // - Zero: Failed to load +# LLAMA_API size_t llama_state_seq_set_data( +# struct llama_context * ctx, +# const uint8_t * src, +# llama_seq_id dest_seq_id); +@ctypes_function( + "llama_state_seq_set_data", + [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), llama_seq_id], + ctypes.c_size_t, +) +def llama_state_seq_set_data( + ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], dest_seq_id: llama_seq_id, / +) -> int: + """Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence""" + ... + + +# LLAMA_API size_t llama_state_seq_save_file( +# struct llama_context * ctx, +# const char * filepath, +# llama_seq_id seq_id, +# const llama_token * tokens, +# size_t n_token_count); +@ctypes_function( + "llama_state_seq_save_file", + [ + llama_context_p_ctypes, + ctypes.c_char_p, + llama_seq_id, + llama_token_p, + ctypes.c_size_t, + ], + ctypes.c_size_t, +) +def llama_state_seq_save_file( + ctx: llama_context_p, + filepath: bytes, + seq_id: llama_seq_id, + tokens: CtypesArray[llama_token], + n_token_count: Union[ctypes.c_size_t, int], + /, +) -> int: + ... + + +# LLAMA_API size_t llama_state_seq_load_file( +# struct llama_context * ctx, +# const char * filepath, +# llama_seq_id dest_seq_id, +# llama_token * tokens_out, +# size_t n_token_capacity, +# size_t * n_token_count_out); +@ctypes_function( + "llama_state_seq_load_file", + [ + llama_context_p_ctypes, + ctypes.c_char_p, + llama_seq_id, + llama_token_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_size_t), + ], + ctypes.c_size_t, +) +def llama_state_seq_load_file( + ctx: llama_context_p, + filepath: bytes, + dest_seq_id: llama_seq_id, + tokens_out: CtypesArray[llama_token], + n_token_capacity: Union[ctypes.c_size_t, int], + n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], + /, +) -> int: + ... + + # // # // Decoding # // @@ -1930,8 +2151,9 @@ def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]: ... -# // Logits for the ith token. Equivalent to: +# // Logits for the ith token. For positive indices, Equivalent to: # // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab +# // Negative indicies can be used to access logits in reverse order, -1 is the last logit. # // returns NULL for invalid ids. # LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i); @ctypes_function( @@ -1963,8 +2185,9 @@ def llama_get_embeddings(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float] ... -# // Get the embeddings for the ith token. Equivalent to: +# // Get the embeddings for the ith token. For positive indices, Equivalent to: # // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd +# // Negative indicies can be used to access embeddings in reverse order, -1 is the last embedding. # // shape: [n_embd] (1-dimensional) # // returns NULL for invalid ids. # LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 75cd4c7729..400d5d722d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 75cd4c77292034ecec587ecb401366f57338f7c0 +Subproject commit 400d5d722d7edf7de0cf24a18c42b183c65047d2 From 889d0e8981641b603259e9a9ee40ce3b3ee8db51 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 10 Apr 2024 02:25:58 -0400 Subject: [PATCH 061/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 31 ++++++++++++++++++++++--------- vendor/llama.cpp | 2 +- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 879308595e..99ae7dea88 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -2271,6 +2271,20 @@ def llama_token_eos(model: llama_model_p, /) -> int: ... +# LLAMA_API llama_token llama_token_cls(const struct llama_model * model); // classification +@ctypes_function("llama_token_cls", [llama_model_p_ctypes], llama_token) +def llama_token_cls(model: llama_model_p, /) -> int: + """classification""" + ... + + +# LLAMA_API llama_token llama_token_sep(const struct llama_model * model); // sentence separator +@ctypes_function("llama_token_sep", [llama_model_p_ctypes], llama_token) +def llama_token_sep(model: llama_model_p, /) -> int: + """sentence separator""" + ... + + # LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line @ctypes_function("llama_token_nl", [llama_model_p_ctypes], llama_token) def llama_token_nl(model: llama_model_p, /) -> int: @@ -2326,16 +2340,16 @@ def llama_token_eot(model: llama_model_p, /) -> int: ... # /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. # /// @return Returns the number of tokens on success, no more than n_tokens_max # /// @return Returns a negative number on failure - the number of tokens that would have been returned -# /// @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. -# /// Does not insert a leading space. +# /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated +# /// as plaintext. Does not insert a leading space. # LLAMA_API int32_t llama_tokenize( # const struct llama_model * model, # const char * text, # int32_t text_len, # llama_token * tokens, # int32_t n_tokens_max, -# bool add_bos, -# bool special); +# bool add_special, +# bool parse_special); @ctypes_function( "llama_tokenize", [ @@ -2355,8 +2369,8 @@ def llama_tokenize( text_len: Union[ctypes.c_int, int], tokens: CtypesArray[llama_token], n_tokens_max: Union[ctypes.c_int, int], - add_bos: Union[ctypes.c_bool, bool], - special: Union[ctypes.c_bool, bool], + add_special: Union[ctypes.c_bool, bool], + parse_special: Union[ctypes.c_bool, bool], /, ) -> int: """Convert the provided text into tokens. @@ -2367,9 +2381,8 @@ def llama_tokenize( text_len: The length of the text. tokens: The tokens pointer must be large enough to hold the resulting tokens. n_max_tokens: The maximum number of tokens to return. - add_bos: Whether to add a beginning-of-sentence token. - special: Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. - Does not insert a leading space. + add_special: Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. Does not insert a leading space. + parse_special: Allow parsing special tokens. Returns: Returns the number of tokens on success, no more than n_tokens_max diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 400d5d722d..ba5e134e07 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 400d5d722d7edf7de0cf24a18c42b183c65047d2 +Subproject commit ba5e134e073ec6837078c874aba44a702944a676 From 1347e1d050fc5a9a32ffe0bb3e22858da28003bd Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 10 Apr 2024 02:40:41 -0400 Subject: [PATCH 062/561] feat: Add typechecking for ctypes structure attributes --- llama_cpp/llama_cpp.py | 216 ++++++++++++++++++++++++++++++++++------- 1 file changed, 180 insertions(+), 36 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 99ae7dea88..2450d11e35 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -237,7 +237,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' LLAMA_FILE_MAGIC_GGSN = 0x6767736E -#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq' +# define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq' LLAMA_FILE_MAGIC_GGSQ = 0x67677371 # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN @@ -245,9 +245,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_SESSION_VERSION 5 LLAMA_SESSION_VERSION = 5 -#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ +# define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ -#define LLAMA_STATE_SEQ_VERSION 1 +# define LLAMA_STATE_SEQ_VERSION 1 LLAMA_STATE_SEQ_VERSION = 1 # struct llama_model; @@ -431,6 +431,11 @@ class llama_token_data(ctypes.Structure): logit (float): log-odds of the token p (float): probability of the token""" + if TYPE_CHECKING: + id: llama_token + logit: float + p: float + _fields_ = [ ("id", llama_token), ("logit", ctypes.c_float), @@ -454,6 +459,11 @@ class llama_token_data_array(ctypes.Structure): size (int): size of the array sorted (bool): whether the array is sorted""" + if TYPE_CHECKING: + data: CtypesArray[llama_token_data] + size: int + sorted: bool + _fields_ = [ ("data", llama_token_data_p), ("size", ctypes.c_size_t), @@ -515,6 +525,15 @@ class llama_batch(ctypes.Structure): logits (ctypes.Array[ctypes.ctypes.c_int8]): if zero, the logits for the respective token will not be output """ + if TYPE_CHECKING: + n_tokens: int + token: CtypesArray[llama_token] + embd: CtypesArray[ctypes.c_float] + pos: CtypesArray[CtypesArray[llama_pos]] + n_seq_id: CtypesArray[ctypes.c_int] + seq_id: CtypesArray[CtypesArray[llama_seq_id]] + logits: CtypesArray[ctypes.c_int8] + _fields_ = [ ("n_tokens", ctypes.c_int32), ("token", ctypes.POINTER(llama_token)), @@ -609,6 +628,18 @@ class llama_model_params(ctypes.Structure): use_mmap (bool): use mmap if possible use_mlock (bool): force system to keep model in RAM""" + if TYPE_CHECKING: + n_gpu_layers: int + split_mode: int + main_gpu: int + tensor_split: CtypesArray[ctypes.c_float] + progress_callback: Callable[[float, ctypes.c_void_p], bool] + progress_callback_user_data: ctypes.c_void_p + kv_overrides: CtypesArray[llama_model_kv_override] + vocab_only: bool + use_mmap: bool + use_mlock: bool + _fields_ = [ ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), @@ -696,6 +727,34 @@ class llama_context_params(ctypes.Structure): abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback """ + if TYPE_CHECKING: + seed: int + n_ctx: int + n_batch: int + n_ubatch: int + n_seq_max: int + n_threads: int + n_threads_batch: int + rope_scaling_type: int + pooling_type: int + rope_freq_base: float + rope_freq_scale: float + yarn_ext_factor: float + yarn_attn_factor: float + yarn_beta_fast: float + yarn_beta_slow: float + yarn_orig_ctx: int + defrag_thold: float + cb_eval: Callable[[ctypes.c_void_p, bool], bool] + cb_eval_user_data: ctypes.c_void_p + type_k: int + type_v: int + logits_all: bool + embeddings: bool + offload_kqv: bool + abort_callback: Callable[[ctypes.c_void_p], bool] + abort_callback_data: ctypes.c_void_p + _fields_ = [ ("seed", ctypes.c_uint32), ("n_ctx", ctypes.c_uint32), @@ -771,6 +830,18 @@ class llama_model_quantize_params(ctypes.Structure): kv_overrides (ctypes.c_void_p): pointer to vector containing overrides """ + if TYPE_CHECKING: + nthread: int + ftype: int + output_tensor_type: int + token_embedding_type: int + allow_requantize: bool + quantize_output_tensor: bool + only_copy: bool + pure: bool + imatrix: ctypes.c_void_p + kv_overrides: ctypes.c_void_p + _fields_ = [ ("nthread", ctypes.c_int32), ("ftype", ctypes.c_int), @@ -828,6 +899,10 @@ class llama_model_quantize_params(ctypes.Structure): # uint32_t value; // Unicode code point or rule ID # } llama_grammar_element; class llama_grammar_element(ctypes.Structure): + if TYPE_CHECKING: + type: int + value: int + _fields_ = [ ("type", ctypes.c_int), ("value", ctypes.c_uint32), @@ -851,6 +926,17 @@ class llama_grammar_element(ctypes.Structure): # int32_t n_eval; # }; class llama_timings(ctypes.Structure): + if TYPE_CHECKING: + t_start_ms: float + t_end_ms: float + t_load_ms: float + t_sample_ms: float + t_p_eval_ms: float + t_eval_ms: float + n_sample: int + n_p_eval: int + n_eval: int + _fields_ = [ ("t_start_ms", ctypes.c_double), ("t_end_ms", ctypes.c_double), @@ -951,7 +1037,8 @@ def llama_backend_init(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): ... +def llama_numa_init(numa: int, /): + ... # // Call once at the end of the program - currently only used for MPI @@ -976,7 +1063,8 @@ def llama_backend_free(): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: ... +) -> Optional[llama_model_p]: + ... # LLAMA_API void llama_free_model(struct llama_model * model); @@ -985,7 +1073,8 @@ def llama_load_model_from_file( [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): ... +def llama_free_model(model: llama_model_p, /): + ... # LLAMA_API struct llama_context * llama_new_context_with_model( @@ -998,7 +1087,8 @@ def llama_free_model(model: llama_model_p, /): ... ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: ... +) -> Optional[llama_context_p]: + ... # // Frees all allocated memory @@ -1019,82 +1109,98 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: ... +def llama_time_us() -> int: + ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: ... +def llama_max_devices() -> int: + ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: ... +def llama_supports_mmap() -> bool: + ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: ... +def llama_supports_mlock() -> bool: + ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: ... +def llama_supports_gpu_offload() -> bool: + ... # LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: + ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: ... +def llama_n_batch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); @ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... +def llama_n_ubatch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); @ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... +def llama_n_seq_max(ctx: llama_context_p, /) -> int: + ... # LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: ... +def llama_vocab_type(model: llama_model_p, /) -> int: + ... # LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: ... +def llama_rope_type(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: ... +def llama_n_vocab(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_embd (const struct llama_model * model); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: ... +def llama_n_embd(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_layer (const struct llama_model * model); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: ... +def llama_n_layer(model: llama_model_p, /) -> int: + ... # // Get the model's RoPE frequency scaling factor @@ -1358,6 +1464,9 @@ class llama_kv_cache_view_cell(ctypes.Structure): pos (llama_pos): The position for this cell. Takes KV cache shifts into account. May be negative if the cell is not populated.""" + if TYPE_CHECKING: + pos: llama_pos + _fields_ = [("pos", llama_pos)] @@ -1394,6 +1503,16 @@ class llama_kv_cache_view_cell(ctypes.Structure): # llama_seq_id * cells_sequences; # }; class llama_kv_cache_view(ctypes.Structure): + if TYPE_CHECKING: + n_cells: int + n_max_seq: int + token_count: int + used_cells: int + max_contiguous: int + max_contiguous_idx: int + cells: CtypesArray[llama_kv_cache_view_cell] + cells_sequences: CtypesArray[llama_seq_id] + _fields_ = [ ("n_cells", ctypes.c_int32), ("n_max_seq", ctypes.c_int32), @@ -1783,7 +1902,8 @@ def llama_state_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_load_session_file( @@ -1811,7 +1931,8 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: ... +) -> int: + ... # LLAMA_API bool llama_state_save_file( @@ -1835,7 +1956,8 @@ def llama_state_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_save_session_file( @@ -1860,7 +1982,8 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: ... +) -> int: + ... # // Get the exact size needed to copy the KV cache of a single sequence @@ -2233,7 +2356,8 @@ def llama_get_embeddings_seq( ) def llama_token_get_text( model: llama_model_p, token: Union[llama_token, int], / -) -> bytes: ... +) -> bytes: + ... # LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); @@ -2242,7 +2366,8 @@ def llama_token_get_text( ) def llama_token_get_score( model: llama_model_p, token: Union[llama_token, int], / -) -> float: ... +) -> float: + ... # LLAMA_API enum llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token); @@ -2251,7 +2376,8 @@ def llama_token_get_score( ) def llama_token_get_type( model: llama_model_p, token: Union[llama_token, int], / -) -> int: ... +) -> int: + ... # // Special tokens @@ -2318,17 +2444,20 @@ def llama_token_prefix(model: llama_model_p) -> int: # LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: ... +def llama_token_middle(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: ... +def llama_token_suffix(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle @ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: ... +def llama_token_eot(model: llama_model_p, /) -> int: + ... # // @@ -2459,7 +2588,8 @@ def llama_chat_apply_template( chat: CtypesArray[llama_chat_message], n_msg: int, /, -) -> int: ... +) -> int: + ... # // @@ -2989,6 +3119,12 @@ def llama_grammar_accept_token( # bool eob; // Callback should set this to true when a beam is at end-of-beam. # }; class llama_beam_view(ctypes.Structure): + if TYPE_CHECKING: + tokens: CtypesArray[llama_token] + n_tokens: int + p: float + eob: bool + _fields_ = [ ("tokens", llama_token_p), ("n_tokens", ctypes.c_size_t), @@ -3008,6 +3144,12 @@ class llama_beam_view(ctypes.Structure): # bool last_call; // True iff this is the last callback invocation. # }; class llama_beams_state(ctypes.Structure): + if TYPE_CHECKING: + beam_views: CtypesArray[llama_beam_view] + n_beams: int + common_prefix_length: int + last_call: bool + _fields_ = [ ("beam_views", ctypes.POINTER(llama_beam_view)), ("n_beams", ctypes.c_size_t), @@ -3060,7 +3202,8 @@ def llama_beam_search( n_past: Union[ctypes.c_int, int], n_predict: Union[ctypes.c_int, int], /, -): ... +): + ... # /// @details Build a split GGUF final path for this chunk. @@ -3179,4 +3322,5 @@ def llama_log_set( [ctypes.c_void_p, llama_context_p_ctypes], None, ) -def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): ... +def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): + ... From 060bfa64d529ade2af9b1f4e207a3937bbc4138f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 10 Apr 2024 02:47:01 -0400 Subject: [PATCH 063/561] feat: Add support for yaml based configs --- examples/batch-processing/server.py | 30 +++++++++++++++++++++++++++++ llama_cpp/server/__main__.py | 11 ++++++++++- llama_cpp/server/app.py | 10 +++++++++- pyproject.toml | 1 + 4 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 examples/batch-processing/server.py diff --git a/examples/batch-processing/server.py b/examples/batch-processing/server.py new file mode 100644 index 0000000000..d3536697a2 --- /dev/null +++ b/examples/batch-processing/server.py @@ -0,0 +1,30 @@ +"""llama-cpp-python server from scratch in a single file. +""" + +# import llama_cpp + +# path = b"../../models/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf" + +# model_params = llama_cpp.llama_model_default_params() +# model = llama_cpp.llama_load_model_from_file(path, model_params) + +# if model is None: +# raise RuntimeError(f"Failed to load model from file: {path}") + + +# ctx_params = llama_cpp.llama_context_default_params() +# ctx = llama_cpp.llama_new_context_with_model(model, ctx_params) + +# if ctx is None: +# raise RuntimeError("Failed to create context") + + +from fastapi import FastAPI + +app = FastAPI() + +import openai.types.chat as types + +@app.post("/v1/chat/completions") +def create_chat_completions(): + return {"message": "Hello World"} diff --git a/llama_cpp/server/__main__.py b/llama_cpp/server/__main__.py index fadfc5fb4f..a6f1f4e9ca 100644 --- a/llama_cpp/server/__main__.py +++ b/llama_cpp/server/__main__.py @@ -59,7 +59,16 @@ def main(): if not os.path.exists(config_file): raise ValueError(f"Config file {config_file} not found!") with open(config_file, "rb") as f: - config_file_settings = ConfigFileSettings.model_validate_json(f.read()) + # Check if yaml file + if config_file.endswith(".yaml") or config_file.endswith(".yml"): + import yaml + import json + + config_file_settings = ConfigFileSettings.model_validate_json( + json.dumps(yaml.safe_load(f)) + ) + else: + config_file_settings = ConfigFileSettings.model_validate_json(f.read()) server_settings = ServerSettings.model_validate(config_file_settings) model_settings = config_file_settings.models else: diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 815ed3c5ea..8211323a48 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -97,7 +97,15 @@ def create_app( if not os.path.exists(config_file): raise ValueError(f"Config file {config_file} not found!") with open(config_file, "rb") as f: - config_file_settings = ConfigFileSettings.model_validate_json(f.read()) + # Check if yaml file + if config_file.endswith(".yaml") or config_file.endswith(".yml"): + import yaml + + config_file_settings = ConfigFileSettings.model_validate_json( + json.dumps(yaml.safe_load(f)) + ) + else: + config_file_settings = ConfigFileSettings.model_validate_json(f.read()) server_settings = ServerSettings.model_validate(config_file_settings) model_settings = config_file_settings.models diff --git a/pyproject.toml b/pyproject.toml index 2f3d3ced0f..e2bbb4b0c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ server = [ "pydantic-settings>=2.0.1", "sse-starlette>=1.6.1", "starlette-context>=0.3.6,<0.4", + "PyYAML>=5.1", ] test = [ "pytest>=7.4.0", From bb65b4d76411112c6fb0bf759efd746f99ef3c6b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 10 Apr 2024 03:41:55 -0400 Subject: [PATCH 064/561] fix: pass correct type to chat handlers for chat completion logprobs --- llama_cpp/llama.py | 3 ++- llama_cpp/llama_chat_format.py | 24 ++++++++++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index e07d57ae4b..466dc22c54 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1664,7 +1664,8 @@ def create_chat_completion( top_k=top_k, min_p=min_p, typical_p=typical_p, - logprobs=top_logprobs if logprobs else None, + logprobs=logprobs, + top_logprobs=top_logprobs, stream=stream, stop=stop, seed=seed, diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 705202e8c4..519d2f50af 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -77,6 +77,8 @@ def __call__( mirostat_eta: float = 0.1, logits_processor: Optional[llama.LogitsProcessorList] = None, grammar: Optional[llama.LlamaGrammar] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, @@ -338,7 +340,7 @@ def _convert_completion_to_chat_function( } ], }, - "logprobs": None, + "logprobs": completion["choices"][0]["logprobs"], "finish_reason": "tool_calls", } ], @@ -391,7 +393,7 @@ def _stream_response_to_function_stream( { "index": 0, "finish_reason": None, - "logprobs": None, + "logprobs": chunk["choices"][0]["logprobs"], "delta": { "role": None, "content": None, @@ -426,7 +428,7 @@ def _stream_response_to_function_stream( { "index": 0, "finish_reason": None, - "logprobs": None, + "logprobs": chunk["choices"][0]["logprobs"], "delta": { "role": None, "content": None, @@ -491,7 +493,6 @@ def chat_completion_handler( temperature: float = 0.2, top_p: float = 0.95, top_k: int = 40, - logprobs: int = 0, min_p: float = 0.05, typical_p: float = 1.0, stream: bool = False, @@ -512,6 +513,8 @@ def chat_completion_handler( logits_processor: Optional[llama.LogitsProcessorList] = None, grammar: Optional[llama.LlamaGrammar] = None, logit_bias: Optional[Dict[str, float]] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, @@ -581,7 +584,7 @@ def chat_completion_handler( top_k=top_k, min_p=min_p, typical_p=typical_p, - logprobs=logprobs, + logprobs=top_logprobs if logprobs else None, stream=stream, stop=stop, seed=seed, @@ -1628,7 +1631,7 @@ def message_to_str(msg: llama_types.ChatCompletionRequestMessage): } ], }, - "logprobs": None, + "logprobs": completion["choices"][0]["logprobs"], "finish_reason": "tool_calls", } ], @@ -2085,7 +2088,7 @@ def create_completion(stop): choices=[ { "index": 0, - "logprobs": None, + "logprobs": completion["choices"][0]["logprobs"], "message": { "role": "assistant", "content": None if content == "" else content, @@ -2311,11 +2314,14 @@ def chatml_function_calling( model: Optional[str] = None, logits_processor: Optional[llama.LogitsProcessorList] = None, grammar: Optional[llama.LlamaGrammar] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, Iterator[llama_types.CreateChatCompletionStreamResponse], ]: + print(logprobs) function_calling_template = ( "{% for message in messages %}" "<|im_start|>{{ message.role }}\n" @@ -2437,6 +2443,7 @@ def chatml_function_calling( model=model, logits_processor=logits_processor, grammar=grammar, + logprobs=top_logprobs if logprobs else None, ), stream=stream, ) @@ -2549,6 +2556,7 @@ def chatml_function_calling( typical_p=typical_p, stream=stream, stop=["<|im_end|>"], + logprobs=top_logprobs if logprobs else None, max_tokens=None, presence_penalty=presence_penalty, frequency_penalty=frequency_penalty, @@ -2660,7 +2668,7 @@ def chatml_function_calling( { "finish_reason": "tool_calls", "index": 0, - "logprobs": None, + "logprobs": completion["choices"][0]["logprobs"], "message": { "role": "assistant", "content": None, From ef29235d453ecf552f015f6ec04270b233bb22c9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 10 Apr 2024 03:44:46 -0400 Subject: [PATCH 065/561] chore: Bump version --- CHANGELOG.md | 7 +++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcb7a816cd..c67498ecda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.61] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ba5e134e073ec6837078c874aba44a702944a676 +- fix: pass correct type to chat handlers for chat completion logprobs by @abetlen in bb65b4d76411112c6fb0bf759efd746f99ef3c6b +- feat: Add support for yaml based server configs by @abetlen in 060bfa64d529ade2af9b1f4e207a3937bbc4138f +- feat: Add typechecking for ctypes structure attributes by @abetlen in 1347e1d050fc5a9a32ffe0bb3e22858da28003bd + ## [0.2.60] - feat: Update llama.cpp to ggerganov/llama.cpp@75cd4c77292034ecec587ecb401366f57338f7c0 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index d5db993d95..2382db9b29 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.60" \ No newline at end of file +__version__ = "0.2.61" \ No newline at end of file From 2e9ffd28fd99958a58fccad6db4614d8c6b555a5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 12 Apr 2024 21:09:12 -0400 Subject: [PATCH 066/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ba5e134e07..ab9a3240a9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ba5e134e073ec6837078c874aba44a702944a676 +Subproject commit ab9a3240a9da941fdef5cd4a25f2b97c2f5a67aa From 90dceaba8a9938ca4d7e7d2cb0997e613bad7040 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 14 Apr 2024 11:35:57 -0400 Subject: [PATCH 067/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ab9a3240a9..f184dd9208 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ab9a3240a9da941fdef5cd4a25f2b97c2f5a67aa +Subproject commit f184dd920852d6d372b754f871ee06cfe6f977ad From a420f9608bbd3b76e8bfbb6cdcf4d3fa69efe5c0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 14 Apr 2024 19:14:09 -0400 Subject: [PATCH 068/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f184dd9208..1958f7e06c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f184dd920852d6d372b754f871ee06cfe6f977ad +Subproject commit 1958f7e06ca2d2e3ab5698cc67513ba359144d8e From c96b2daebf7b5dbe3cfd9194b249ab579a28d633 Mon Sep 17 00:00:00 2001 From: ddh0 <40664579+ddh0@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:04:33 -0500 Subject: [PATCH 069/561] feat: Use all available CPUs for batch processing (#1345) --- llama_cpp/llama.py | 4 +--- llama_cpp/server/settings.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 466dc22c54..dfac9bb8a1 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -262,9 +262,7 @@ def __init__( self.n_batch = min(n_ctx, n_batch) # ??? self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1) - self.n_threads_batch = n_threads_batch or max( - multiprocessing.cpu_count() // 2, 1 - ) + self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count() # Context Params self.context_params = llama_cpp.llama_context_default_params() diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 9ebdd0d8c5..811c6ca660 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -70,7 +70,7 @@ class ModelSettings(BaseSettings): description="The number of threads to use.", ) n_threads_batch: int = Field( - default=max(multiprocessing.cpu_count() // 2, 1), + default=max(multiprocessing.cpu_count(), 1), ge=0, description="The number of threads to use when batch processing.", ) From 9842cbf99d048a121cff03f01340592ad4b7fc1e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 17 Apr 2024 10:06:15 -0400 Subject: [PATCH 070/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1958f7e06c..8dd1ec8b3f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1958f7e06ca2d2e3ab5698cc67513ba359144d8e +Subproject commit 8dd1ec8b3ffbfa2d26e82e672cea89f5eeb2f141 From 4924455decd79273c8c695a8ff796306ac0df30d Mon Sep 17 00:00:00 2001 From: tc-wolf <50339167+tc-wolf@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:06:50 -0500 Subject: [PATCH 071/561] feat: Make saved state more compact on-disk (#1296) * State load/save changes - Only store up to `n_tokens` logits instead of full `(n_ctx, n_vocab)` sized array. - Difference between ~350MB and ~1500MB for example prompt with ~300 tokens (makes sense lol) - Auto-formatting changes * Back out formatting changes --- llama_cpp/llama.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index dfac9bb8a1..5a0111bbe6 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -18,6 +18,7 @@ Iterator, Deque, Callable, + Dict, ) from collections import deque from pathlib import Path @@ -1791,7 +1792,7 @@ def save_state(self) -> LlamaState: file=sys.stderr, ) return LlamaState( - scores=self.scores.copy(), + scores=self._scores.copy(), input_ids=self.input_ids.copy(), n_tokens=self.n_tokens, llama_state=bytes(llama_state_compact), @@ -1800,7 +1801,9 @@ def save_state(self) -> LlamaState: def load_state(self, state: LlamaState) -> None: assert self._ctx.ctx is not None - self.scores = state.scores.copy() + # Only filling in up to `n_tokens` and then zero-ing out the rest + self.scores[: state.n_tokens, :] = state.scores.copy() + self.scores[state.n_tokens :, :] = 0.0 self.input_ids = state.input_ids.copy() self.n_tokens = state.n_tokens state_size = state.llama_state_size @@ -1951,7 +1954,6 @@ def from_pretrained( local_dir_use_symlinks=local_dir_use_symlinks, cache_dir=cache_dir, local_files_only=True, - ) else: model_path = os.path.join(local_dir, filename) From b73c73c0c67f559fd9c7d620ad3d4e24d5c4bc4c Mon Sep 17 00:00:00 2001 From: khimaros Date: Wed, 17 Apr 2024 14:08:19 +0000 Subject: [PATCH 072/561] feat: add `disable_ping_events` flag (#1257) for backward compatibility, this is false by default it can be set to true to disable EventSource pings which are not supported by some OpenAI clients. fixes https://github.com/abetlen/llama-cpp-python/issues/1256 --- llama_cpp/server/app.py | 12 ++++++++++++ llama_cpp/server/settings.py | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 8211323a48..b6ed9b1b6b 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -87,6 +87,13 @@ def get_llama_proxy(): llama_outer_lock.release() +_ping_message_factory = None + +def set_ping_message_factory(factory): + global _ping_message_factory + _ping_message_factory = factory + + def create_app( settings: Settings | None = None, server_settings: ServerSettings | None = None, @@ -138,6 +145,9 @@ def create_app( assert model_settings is not None set_llama_proxy(model_settings=model_settings) + if server_settings.disable_ping_events: + set_ping_message_factory(lambda: bytes()) + return app @@ -302,6 +312,7 @@ def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: iterator=iterator(), ), sep="\n", + ping_message_factory=_ping_message_factory, ) else: return iterator_or_completion @@ -470,6 +481,7 @@ def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: iterator=iterator(), ), sep="\n", + ping_message_factory=_ping_message_factory, ) else: return iterator_or_completion diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 811c6ca660..934aecd919 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -195,6 +195,10 @@ class ServerSettings(BaseSettings): default=True, description="Whether to interrupt requests when a new request is received.", ) + disable_ping_events: bool = Field( + default=False, + description="Disable EventSource pings (may be needed for some clients).", + ) class Settings(ServerSettings, ModelSettings): From 610a592f708f2a5a8b0e1d6f0900f6337b9beb39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucca=20Zen=C3=B3bio?= Date: Wed, 17 Apr 2024 11:10:21 -0300 Subject: [PATCH 073/561] feat: Update json to grammar (#1350) * feat: improve function calling * feat:grammar --- llama_cpp/llama_chat_format.py | 2 +- llama_cpp/llama_grammar.py | 612 +++++++++++++++++++++++++++------ 2 files changed, 514 insertions(+), 100 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 519d2f50af..eb98cbf391 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2709,4 +2709,4 @@ def chatml_function_calling( }, } - raise ValueError("Automatic streaming tool choice is not supported") + raise ValueError("Automatic streaming tool choice is not supported") \ No newline at end of file diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 9cc48a93b3..8c0f8aa091 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -5,11 +5,12 @@ import sys from ctypes import * # type: ignore from enum import Enum -from itertools import islice +from itertools import islice, groupby from typing import ( Any, Callable, Dict, + Set, Generic, List, Optional, @@ -1391,139 +1392,552 @@ def print_grammar(file: TextIO, state: parse_state) -> None: # whitespace. Also maybe improves generation quality? SPACE_RULE = '" "?' -PRIMITIVE_RULES = { - "boolean": '("true" | "false") space', - "number": '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space', - "integer": '("-"? ([0-9] | [1-9] [0-9]*)) space', - "string": r""" "\"" ( - [^"\\] | - "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) - )* "\"" space """, - "null": '"null" space', -} INVALID_RULE_CHARS_RE = re.compile(r"[^a-zA-Z0-9-]+") GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"]') GRAMMAR_LITERAL_ESCAPES = {"\r": "\\r", "\n": "\\n", '"': '\\"'} +# whitespace is constrained to a single space char to prevent model "running away" in +# whitespace. Also maybe improves generation quality? +SPACE_RULE = '" "?' + + +def _build_repetition(item_rule, min_items, max_items, separator_rule=None, item_rule_is_literal=False): + if not separator_rule: + if min_items == 0 and max_items == 1: + return f'{item_rule}?' + elif min_items == 1 and max_items is None: + return f'{item_rule}+' + + result = '' + + if min_items > 0: + if item_rule_is_literal and separator_rule is None: + result = '"' + (item_rule[1:-1] * min_items) + '"' + else: + result = (f' {separator_rule} ' if separator_rule else ' ').join([item_rule] * min_items) + + def opt_repetitions(up_to_n, prefix_with_sep=False): + ''' + - n=4, no sep: '(a (a (a (a)?)?)?)?' + - n=4, sep=',', prefix: '("," a ("," a ("," a ("," a)?)?)?)?' + - n=4, sep=',', no prefix: '(a ("," a ("," a ("," a)?)?)?)?' + ''' + + content = f'{separator_rule} {item_rule}' if prefix_with_sep and separator_rule else item_rule + if up_to_n == 0: + return '' + elif up_to_n == 1: + return f'({content})?' + elif separator_rule and not prefix_with_sep: + return f'({content} {opt_repetitions(up_to_n - 1, prefix_with_sep=True)})?' + else: + return (f'({content} ' * up_to_n).rstrip() + (')?' * up_to_n) + + if min_items > 0 and max_items != min_items: + result += ' ' + + if max_items is not None: + result += opt_repetitions(max_items - min_items, prefix_with_sep=min_items > 0) + else: + item_operator = f'({separator_rule + " " if separator_rule else ""}{item_rule})' + + if min_items == 0 and separator_rule: + result = f'({item_rule} {item_operator}*)?' + else: + result += f'{item_operator}*' + + return result + + + +class BuiltinRule: + def __init__(self, content: str, deps: list = None): + self.content = content + self.deps = deps or [] + +_up_to_15_digits = _build_repetition('[0-9]', 0, 15) + +PRIMITIVE_RULES = { + 'boolean' : BuiltinRule('("true" | "false") space', []), + 'decimal-part' : BuiltinRule('[0-9] ' + _up_to_15_digits, []), + 'integral-part': BuiltinRule('[0-9] | [1-9] ' + _up_to_15_digits, []), + 'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space', ['integral-part', 'decimal-part']), + 'integer' : BuiltinRule('("-"? integral-part) space', ['integral-part']), + 'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']), + 'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', ['string', 'value']), + 'array' : BuiltinRule('"[" space ( value ("," space value)* )? "]" space', ['value']), + 'uuid' : BuiltinRule(r'"\"" ' + ' "-" '.join('[0-9a-fA-F]' * n for n in [8, 4, 4, 4, 12]) + r' "\"" space', []), + 'char' : BuiltinRule(r'[^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])', []), + 'string' : BuiltinRule(r'"\"" char* "\"" space', ['char']), + 'null' : BuiltinRule('"null" space', []), +} + +# TODO: support "uri", "email" string formats +STRING_FORMAT_RULES = { + 'date' : BuiltinRule('[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []), + 'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []), + 'date-time' : BuiltinRule('date "T" time', ['date', 'time']), + 'date-string' : BuiltinRule('"\\"" date "\\"" space', ['date']), + 'time-string' : BuiltinRule('"\\"" time "\\"" space', ['time']), + 'date-time-string': BuiltinRule('"\\"" date-time "\\"" space', ['date-time']), +} + +DOTALL = '[\\U00000000-\\U0010FFFF]' +DOT = '[^\\x0A\\x0D]' + +RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()]) + + +NON_LITERAL_SET = set('|.()[]{}*+?') +ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('[]()|{}*+?') + + + class SchemaConverter: def __init__(self, prop_order): self._prop_order = prop_order self._rules = {"space": SPACE_RULE} self._defs: Dict[str, Any] = {} + self._refs = {} + self._refs_being_resolved = set() - def _format_literal(self, literal: str): - escaped: str = GRAMMAR_LITERAL_ESCAPE_RE.sub( - lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)), json.dumps(literal) + def _format_literal(self, literal): + escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub( + lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)), literal ) return f'"{escaped}"' - def _add_rule(self, name: str, rule: str): - esc_name = INVALID_RULE_CHARS_RE.sub("-", name) + def not_literal(self, literal: str, dotall: bool = True, maybe_escaped_underscores = False) -> str: + ''' + not_literal('a') -> '[^a]' + not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?' + ''' + assert len(literal) > 0, 'Empty literal not supported' + def recurse(i: int): + c = literal[i] + if maybe_escaped_underscores and c == '_': + yield f'[^{c}\\\\]' + yield ' | ' + yield f'"\\\\"? "{c}"' + else: + yield f'[^{c}]' + if i < len(literal) - 1: + yield ' | ' + yield self._format_literal(c) + yield ' (' + yield from recurse(i + 1) + yield ')?' + + return ''.join(('(', *recurse(0), ')')) + + def _add_rule(self, name, rule): + esc_name = INVALID_RULE_CHARS_RE.sub('-', name) if esc_name not in self._rules or self._rules[esc_name] == rule: key = esc_name else: i = 0 - while f"{esc_name}{i}" in self._rules: + while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule: i += 1 - key = f"{esc_name}{i}" + key = f'{esc_name}{i}' self._rules[key] = rule return key - def visit(self, schema: Dict[str, Any], name: str) -> str: - rule_name = name or "root" - - if "$defs" in schema: - # add defs to self._defs for later inlining - for def_name, def_schema in schema["$defs"].items(): - self._defs[def_name] = def_schema + def resolve_refs(self, schema: dict, url: str): + ''' + Resolves all $ref fields in the given schema, fetching any remote schemas, + replacing $ref with absolute reference URL and populating self._refs with the + respective referenced (sub)schema dictionaries. + ''' + def visit(n: dict): + if isinstance(n, list): + return [visit(x) for x in n] + elif isinstance(n, dict): + ref = n.get('$ref') + if ref is not None and ref not in self._refs: + if ref.startswith('https://'): + assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)' + import requests + + frag_split = ref.split('#') + base_url = frag_split[0] + + target = self._refs.get(base_url) + if target is None: + target = self.resolve_refs(requests.get(ref).json(), base_url) + self._refs[base_url] = target + + if len(frag_split) == 1 or frag_split[-1] == '': + return target + elif ref.startswith('#/'): + target = schema + ref = f'{url}{ref}' + n['$ref'] = ref + else: + raise ValueError(f'Unsupported ref {ref}') + + for sel in ref.split('#')[-1].split('/')[1:]: + assert target is not None and sel in target, f'Error resolving ref {ref}: {sel} not in {target}' + target = target[sel] + + self._refs[ref] = target + else: + for v in n.values(): + visit(v) + + return n + return visit(schema) + + def _generate_union_rule(self, name, alt_schemas): + return ' | '.join(( + self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') + for i, alt_schema in enumerate(alt_schemas) + )) + + def _visit_pattern(self, pattern, name): + ''' + Transforms a regular expression pattern into a GBNF rule. + + Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions + Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md + + Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. + + Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which + we define sub-rules to keep the output lean. + ''' + + assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"' + pattern = pattern[1:-1] + sub_rule_ids = {} + + i = 0 + length = len(pattern) + + def to_rule(s: Tuple[str, bool]) -> str: + (txt, is_literal) = s + return "\"" + txt + "\"" if is_literal else txt + + def transform() -> Tuple[str, bool]: + ''' + Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. + ''' + nonlocal i + nonlocal pattern + nonlocal sub_rule_ids + + start = i + # For each component of this sequence, store its string representation and whether it's a literal. + # We only need a flat structure here to apply repetition operators to the last item, and + # to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially + # (GBNF's syntax is luckily very close to regular expressions!) + seq: list[Tuple[str, bool]] = [] + + def get_dot(): + if self._dotall: + rule = DOTALL + else: + # Accept any character... except \n and \r line break chars (\x0A and \xOD) + rule = DOT + return self._add_rule(f'dot', rule) + + def join_seq(): + nonlocal seq + ret = [] + for is_literal, g in groupby(seq, lambda x: x[1]): + if is_literal: + ret.append((''.join(x[0] for x in g), True)) + else: + ret.extend(g) + if len(ret) == 1: + return ret[0] + return (' '.join(to_rule(x) for x in seq), False) + + while i < length: + c = pattern[i] + if c == '.': + seq.append((get_dot(), False)) + i += 1 + elif c == '(': + i += 1 + if i < length: + assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' + seq.append((f'({to_rule(transform())})', False)) + elif c == ')': + i += 1 + assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}' + return join_seq() + elif c == '[': + square_brackets = c + i += 1 + while i < length and pattern[i] != ']': + if pattern[i] == '\\': + square_brackets += pattern[i:i+2] + i += 2 + else: + square_brackets += pattern[i] + i += 1 + assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}' + square_brackets += ']' + i += 1 + seq.append((square_brackets, False)) + elif c == '|': + seq.append(('|', False)) + i += 1 + elif c in ('*', '+', '?'): + seq[-1] = (to_rule(seq[-1]) + c, False) + i += 1 + elif c == '{': + curly_brackets = c + i += 1 + while i < length and pattern[i] != '}': + curly_brackets += pattern[i] + i += 1 + assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}' + curly_brackets += '}' + i += 1 + nums = [s.strip() for s in curly_brackets[1:-1].split(',')] + min_times = 0 + max_times = None + try: + if len(nums) == 1: + min_times = int(nums[0]) + max_times = min_times + else: + assert len(nums) == 2 + min_times = int(nums[0]) if nums[0] else 0 + max_times = int(nums[1]) if nums[1] else None + except ValueError: + raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/') + + (sub, sub_is_literal) = seq[-1] + + if not sub_is_literal: + id = sub_rule_ids.get(sub) + if id is None: + id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub) + sub_rule_ids[sub] = id + sub = id + + seq[-1] = (_build_repetition(f'"{sub}"' if sub_is_literal else sub, min_times, max_times, item_rule_is_literal=sub_is_literal), False) + else: + literal = '' + while i < length: + if pattern[i] == '\\' and i < length - 1: + next = pattern[i + 1] + if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS: + i += 1 + literal += pattern[i] + i += 1 + else: + literal += pattern[i:i+2] + i += 2 + elif pattern[i] == '"' and not self._raw_pattern: + literal += '\\"' + i += 1 + elif pattern[i] not in NON_LITERAL_SET and \ + (i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET): + literal += pattern[i] + i += 1 + else: + break + if literal: + seq.append((literal, True)) + + return join_seq() + + return self._add_rule( + name, + to_rule(transform()) if self._raw_pattern \ + else "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space") + + + def _resolve_ref(self, ref): + ref_name = ref.split('/')[-1] + if ref_name not in self._rules and ref not in self._refs_being_resolved: + self._refs_being_resolved.add(ref) + resolved = self._refs[ref] + ref_name = self.visit(resolved, ref_name) + self._refs_being_resolved.remove(ref) + return ref_name + + def _generate_constant_rule(self, value): + return self._format_literal(json.dumps(value)) + + def visit(self, schema, name): + schema_type = schema.get('type') + schema_format = schema.get('format') + rule_name = name + '-' if name in RESERVED_NAMES else name or 'root' + + if (ref := schema.get('$ref')) is not None: + return self._add_rule(rule_name, self._resolve_ref(ref)) + + elif 'oneOf' in schema or 'anyOf' in schema: + return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf'])) + + elif isinstance(schema_type, list): + return self._add_rule(rule_name, self._generate_union_rule(name, [{'type': t} for t in schema_type])) + + elif 'const' in schema: + return self._add_rule(rule_name, self._generate_constant_rule(schema['const'])) + + elif 'enum' in schema: + rule = ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + return self._add_rule(rule_name, rule) - if "oneOf" in schema or "anyOf" in schema: - rule = " | ".join( - ( - self.visit(alt_schema, f'{name}{"-" if name else ""}{i}') - for i, alt_schema in enumerate( - schema.get("oneOf") or schema["anyOf"] - ) - ) + elif schema_type in (None, 'object') and \ + ('properties' in schema or \ + ('additionalProperties' in schema and schema['additionalProperties'] is not True)): + required = set(schema.get('required', [])) + properties = list(schema.get('properties', {}).items()) + return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties'))) + + elif schema_type in (None, 'object') and 'allOf' in schema: + required = set() + properties = [] + hybrid_name = name + def add_component(comp_schema, is_required): + if (ref := comp_schema.get('$ref')) is not None: + comp_schema = self._refs[ref] + + if 'properties' in comp_schema: + for prop_name, prop_schema in comp_schema['properties'].items(): + properties.append((prop_name, prop_schema)) + if is_required: + required.add(prop_name) + + for t in schema['allOf']: + if 'anyOf' in t: + for tt in t['anyOf']: + add_component(tt, is_required=False) + else: + add_component(t, is_required=True) + + return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=[])) + + elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema): + items = schema.get('items') or schema['prefixItems'] + if isinstance(items, list): + return self._add_rule( + rule_name, + '"[" space ' + + ' "," space '.join( + self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') + for i, item in enumerate(items)) + + ' "]" space') + else: + item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') + min_items = schema.get("minItems", 0) + max_items = schema.get("maxItems") + return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' "]" space') + + elif schema_type in (None, 'string') and 'pattern' in schema: + return self._visit_pattern(schema['pattern'], rule_name) + + elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''): + return self._add_primitive( + 'root' if rule_name == 'root' else schema_format, + PRIMITIVE_RULES['uuid'] ) - return self._add_rule(rule_name, rule) - elif "const" in schema: - return self._add_rule(rule_name, self._format_literal(schema["const"])) + elif schema_type in (None, 'string') and f'{schema_format}-string' in STRING_FORMAT_RULES: + prim_name = f'{schema_format}-string' + return self._add_rule(rule_name, self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name])) - elif "enum" in schema: - rule = " | ".join((self._format_literal(v) for v in schema["enum"])) - return self._add_rule(rule_name, rule) + elif schema_type == 'string' and ('minLength' in schema or 'maxLength' in schema): + char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) + min_len = schema.get('minLength', 0) + max_len = schema.get('maxLength') - elif "$ref" in schema: - ref = schema["$ref"] - assert ref.startswith("#/$defs/"), f"Unrecognized schema: {schema}" - # inline $defs - def_name = ref[len("#/$defs/") :] - def_schema = self._defs[def_name] - return self.visit(def_schema, f'{name}{"-" if name else ""}{def_name}') - - - schema_type: Optional[str] = schema.get("type") # type: ignore - assert isinstance(schema_type, str), f"Unrecognized schema: {schema}" - - if schema_type == "object" and "properties" in schema: - # TODO: `required` keyword - if self._prop_order: - prop_order = self._prop_order - prop_pairs = sorted( - schema["properties"].items(), - # sort by position in prop_order (if specified) then by key - key=lambda kv: (prop_order.get(kv[0], len(prop_order)), kv[0]), - ) - else: - prop_pairs = schema["properties"].items() + return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\"" space') - rule = '"{" space' - for i, (prop_name, prop_schema) in enumerate(prop_pairs): - prop_rule_name = self.visit( - prop_schema, f'{name}{"-" if name else ""}{prop_name}' - ) - if i > 0: - rule += ' "," space' - rule += rf' {self._format_literal(prop_name)} space ":" space {prop_rule_name}' - rule += ' "}" space' - - return self._add_rule(rule_name, rule) + elif (schema_type == 'object') or (len(schema) == 0): + return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object'])) - elif schema_type == "array" and "items" in schema: - # TODO `prefixItems` keyword - item_rule_name = self.visit( - schema["items"], f'{name}{"-" if name else ""}item' + else: + assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}' + # TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero + return self._add_primitive('root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type]) + + def _add_primitive(self, name: str, rule: BuiltinRule): + n = self._add_rule(name, rule.content) + + for dep in rule.deps: + dep_rule = PRIMITIVE_RULES.get(dep) or STRING_FORMAT_RULES.get(dep) + assert dep_rule, f'Rule {dep} not known' + if dep not in self._rules: + self._add_primitive(dep, dep_rule) + return n + + def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Union[bool, Any]): + prop_order = self._prop_order + # sort by position in prop_order (if specified) then by original order + sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))] + + prop_kv_rule_names = {} + for prop_name, prop_schema in properties: + prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') + prop_kv_rule_names[prop_name] = self._add_rule( + f'{name}{"-" if name else ""}{prop_name}-kv', + fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}' ) - list_item_operator = f'("," space {item_rule_name})' - successive_items = "" - min_items = schema.get("minItems", 0) - if min_items > 0: - first_item = f"({item_rule_name})" - successive_items = list_item_operator * (min_items - 1) - min_items -= 1 - else: - first_item = f"({item_rule_name})?" - max_items = schema.get("maxItems") - if max_items is not None and max_items > min_items: - successive_items += (list_item_operator + "?") * (max_items - min_items - 1) - else: - successive_items += list_item_operator + "*" - rule = f'"[" space {first_item} {successive_items} "]" space' - return self._add_rule(rule_name, rule) + required_props = [k for k in sorted_props if k in required] + optional_props = [k for k in sorted_props if k not in required] + + if additional_properties == True or isinstance(additional_properties, dict): + sub_name = f'{name}{"-" if name else ""}additional' + value_rule = self.visit({} if additional_properties == True else additional_properties, f'{sub_name}-value') + prop_kv_rule_names["*"] = self._add_rule( + f'{sub_name}-kv', + self._add_primitive('string', PRIMITIVE_RULES['string']) + f' ":" space {value_rule}' + ) + optional_props.append("*") + + rule = '"{" space ' + rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props) + + if optional_props: + rule += ' (' + if required_props: + rule += ' "," space ( ' + + def get_recursive_refs(ks, first_is_optional): + [k, *rest] = ks + kv_rule_name = prop_kv_rule_names[k] + if k == '*': + res = self._add_rule( + f'{name}{"-" if name else ""}additional-kvs', + f'{kv_rule_name} ( "," space ' + kv_rule_name + ' )*' + ) + elif first_is_optional: + res = f'( "," space {kv_rule_name} )?' + else: + res = kv_rule_name + if len(rest) > 0: + res += ' ' + self._add_rule( + f'{name}{"-" if name else ""}{k}-rest', + get_recursive_refs(rest, first_is_optional=True) + ) + return res - else: - assert schema_type in PRIMITIVE_RULES, f"Unrecognized schema: {schema}" - return self._add_rule( - "root" if rule_name == "root" else schema_type, - PRIMITIVE_RULES[schema_type], + rule += ' | '.join( + get_recursive_refs(optional_props[i:], first_is_optional=False) + for i in range(len(optional_props)) ) + if required_props: + rule += ' )' + rule += ' )?' + + rule += ' "}" space' + + return rule def format_grammar(self): - return "\n".join((f"{name} ::= {rule}" for name, rule in self._rules.items())) + return '\n'.join( + f'{name} ::= {rule}' + for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0]) + ) def json_schema_to_gbnf(schema: str, prop_order: Optional[List[str]] = None): From fa4bb0cf81594d3c5bfd0cdf883fc88f820cbc54 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 17 Apr 2024 16:18:16 -0400 Subject: [PATCH 074/561] Revert "feat: Update json to grammar (#1350)" This reverts commit 610a592f708f2a5a8b0e1d6f0900f6337b9beb39. --- llama_cpp/llama_chat_format.py | 2 +- llama_cpp/llama_grammar.py | 612 ++++++--------------------------- 2 files changed, 100 insertions(+), 514 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index eb98cbf391..519d2f50af 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2709,4 +2709,4 @@ def chatml_function_calling( }, } - raise ValueError("Automatic streaming tool choice is not supported") \ No newline at end of file + raise ValueError("Automatic streaming tool choice is not supported") diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 8c0f8aa091..9cc48a93b3 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -5,12 +5,11 @@ import sys from ctypes import * # type: ignore from enum import Enum -from itertools import islice, groupby +from itertools import islice from typing import ( Any, Callable, Dict, - Set, Generic, List, Optional, @@ -1392,552 +1391,139 @@ def print_grammar(file: TextIO, state: parse_state) -> None: # whitespace. Also maybe improves generation quality? SPACE_RULE = '" "?' +PRIMITIVE_RULES = { + "boolean": '("true" | "false") space', + "number": '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space', + "integer": '("-"? ([0-9] | [1-9] [0-9]*)) space', + "string": r""" "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) + )* "\"" space """, + "null": '"null" space', +} INVALID_RULE_CHARS_RE = re.compile(r"[^a-zA-Z0-9-]+") GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"]') GRAMMAR_LITERAL_ESCAPES = {"\r": "\\r", "\n": "\\n", '"': '\\"'} -# whitespace is constrained to a single space char to prevent model "running away" in -# whitespace. Also maybe improves generation quality? -SPACE_RULE = '" "?' - - -def _build_repetition(item_rule, min_items, max_items, separator_rule=None, item_rule_is_literal=False): - if not separator_rule: - if min_items == 0 and max_items == 1: - return f'{item_rule}?' - elif min_items == 1 and max_items is None: - return f'{item_rule}+' - - result = '' - - if min_items > 0: - if item_rule_is_literal and separator_rule is None: - result = '"' + (item_rule[1:-1] * min_items) + '"' - else: - result = (f' {separator_rule} ' if separator_rule else ' ').join([item_rule] * min_items) - - def opt_repetitions(up_to_n, prefix_with_sep=False): - ''' - - n=4, no sep: '(a (a (a (a)?)?)?)?' - - n=4, sep=',', prefix: '("," a ("," a ("," a ("," a)?)?)?)?' - - n=4, sep=',', no prefix: '(a ("," a ("," a ("," a)?)?)?)?' - ''' - - content = f'{separator_rule} {item_rule}' if prefix_with_sep and separator_rule else item_rule - if up_to_n == 0: - return '' - elif up_to_n == 1: - return f'({content})?' - elif separator_rule and not prefix_with_sep: - return f'({content} {opt_repetitions(up_to_n - 1, prefix_with_sep=True)})?' - else: - return (f'({content} ' * up_to_n).rstrip() + (')?' * up_to_n) - - if min_items > 0 and max_items != min_items: - result += ' ' - - if max_items is not None: - result += opt_repetitions(max_items - min_items, prefix_with_sep=min_items > 0) - else: - item_operator = f'({separator_rule + " " if separator_rule else ""}{item_rule})' - - if min_items == 0 and separator_rule: - result = f'({item_rule} {item_operator}*)?' - else: - result += f'{item_operator}*' - - return result - - - -class BuiltinRule: - def __init__(self, content: str, deps: list = None): - self.content = content - self.deps = deps or [] - -_up_to_15_digits = _build_repetition('[0-9]', 0, 15) - -PRIMITIVE_RULES = { - 'boolean' : BuiltinRule('("true" | "false") space', []), - 'decimal-part' : BuiltinRule('[0-9] ' + _up_to_15_digits, []), - 'integral-part': BuiltinRule('[0-9] | [1-9] ' + _up_to_15_digits, []), - 'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space', ['integral-part', 'decimal-part']), - 'integer' : BuiltinRule('("-"? integral-part) space', ['integral-part']), - 'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']), - 'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', ['string', 'value']), - 'array' : BuiltinRule('"[" space ( value ("," space value)* )? "]" space', ['value']), - 'uuid' : BuiltinRule(r'"\"" ' + ' "-" '.join('[0-9a-fA-F]' * n for n in [8, 4, 4, 4, 12]) + r' "\"" space', []), - 'char' : BuiltinRule(r'[^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])', []), - 'string' : BuiltinRule(r'"\"" char* "\"" space', ['char']), - 'null' : BuiltinRule('"null" space', []), -} - -# TODO: support "uri", "email" string formats -STRING_FORMAT_RULES = { - 'date' : BuiltinRule('[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []), - 'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []), - 'date-time' : BuiltinRule('date "T" time', ['date', 'time']), - 'date-string' : BuiltinRule('"\\"" date "\\"" space', ['date']), - 'time-string' : BuiltinRule('"\\"" time "\\"" space', ['time']), - 'date-time-string': BuiltinRule('"\\"" date-time "\\"" space', ['date-time']), -} - -DOTALL = '[\\U00000000-\\U0010FFFF]' -DOT = '[^\\x0A\\x0D]' - -RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()]) - - -NON_LITERAL_SET = set('|.()[]{}*+?') -ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('[]()|{}*+?') - - - class SchemaConverter: def __init__(self, prop_order): self._prop_order = prop_order self._rules = {"space": SPACE_RULE} self._defs: Dict[str, Any] = {} - self._refs = {} - self._refs_being_resolved = set() - def _format_literal(self, literal): - escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub( - lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)), literal + def _format_literal(self, literal: str): + escaped: str = GRAMMAR_LITERAL_ESCAPE_RE.sub( + lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)), json.dumps(literal) ) return f'"{escaped}"' - def not_literal(self, literal: str, dotall: bool = True, maybe_escaped_underscores = False) -> str: - ''' - not_literal('a') -> '[^a]' - not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?' - ''' - assert len(literal) > 0, 'Empty literal not supported' - def recurse(i: int): - c = literal[i] - if maybe_escaped_underscores and c == '_': - yield f'[^{c}\\\\]' - yield ' | ' - yield f'"\\\\"? "{c}"' - else: - yield f'[^{c}]' - if i < len(literal) - 1: - yield ' | ' - yield self._format_literal(c) - yield ' (' - yield from recurse(i + 1) - yield ')?' - - return ''.join(('(', *recurse(0), ')')) - - def _add_rule(self, name, rule): - esc_name = INVALID_RULE_CHARS_RE.sub('-', name) + def _add_rule(self, name: str, rule: str): + esc_name = INVALID_RULE_CHARS_RE.sub("-", name) if esc_name not in self._rules or self._rules[esc_name] == rule: key = esc_name else: i = 0 - while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule: + while f"{esc_name}{i}" in self._rules: i += 1 - key = f'{esc_name}{i}' + key = f"{esc_name}{i}" self._rules[key] = rule return key - def resolve_refs(self, schema: dict, url: str): - ''' - Resolves all $ref fields in the given schema, fetching any remote schemas, - replacing $ref with absolute reference URL and populating self._refs with the - respective referenced (sub)schema dictionaries. - ''' - def visit(n: dict): - if isinstance(n, list): - return [visit(x) for x in n] - elif isinstance(n, dict): - ref = n.get('$ref') - if ref is not None and ref not in self._refs: - if ref.startswith('https://'): - assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)' - import requests - - frag_split = ref.split('#') - base_url = frag_split[0] - - target = self._refs.get(base_url) - if target is None: - target = self.resolve_refs(requests.get(ref).json(), base_url) - self._refs[base_url] = target - - if len(frag_split) == 1 or frag_split[-1] == '': - return target - elif ref.startswith('#/'): - target = schema - ref = f'{url}{ref}' - n['$ref'] = ref - else: - raise ValueError(f'Unsupported ref {ref}') - - for sel in ref.split('#')[-1].split('/')[1:]: - assert target is not None and sel in target, f'Error resolving ref {ref}: {sel} not in {target}' - target = target[sel] - - self._refs[ref] = target - else: - for v in n.values(): - visit(v) - - return n - return visit(schema) - - def _generate_union_rule(self, name, alt_schemas): - return ' | '.join(( - self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') - for i, alt_schema in enumerate(alt_schemas) - )) - - def _visit_pattern(self, pattern, name): - ''' - Transforms a regular expression pattern into a GBNF rule. - - Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions - Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md - - Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. - - Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which - we define sub-rules to keep the output lean. - ''' - - assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"' - pattern = pattern[1:-1] - sub_rule_ids = {} - - i = 0 - length = len(pattern) - - def to_rule(s: Tuple[str, bool]) -> str: - (txt, is_literal) = s - return "\"" + txt + "\"" if is_literal else txt - - def transform() -> Tuple[str, bool]: - ''' - Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. - ''' - nonlocal i - nonlocal pattern - nonlocal sub_rule_ids - - start = i - # For each component of this sequence, store its string representation and whether it's a literal. - # We only need a flat structure here to apply repetition operators to the last item, and - # to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially - # (GBNF's syntax is luckily very close to regular expressions!) - seq: list[Tuple[str, bool]] = [] - - def get_dot(): - if self._dotall: - rule = DOTALL - else: - # Accept any character... except \n and \r line break chars (\x0A and \xOD) - rule = DOT - return self._add_rule(f'dot', rule) - - def join_seq(): - nonlocal seq - ret = [] - for is_literal, g in groupby(seq, lambda x: x[1]): - if is_literal: - ret.append((''.join(x[0] for x in g), True)) - else: - ret.extend(g) - if len(ret) == 1: - return ret[0] - return (' '.join(to_rule(x) for x in seq), False) - - while i < length: - c = pattern[i] - if c == '.': - seq.append((get_dot(), False)) - i += 1 - elif c == '(': - i += 1 - if i < length: - assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' - seq.append((f'({to_rule(transform())})', False)) - elif c == ')': - i += 1 - assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}' - return join_seq() - elif c == '[': - square_brackets = c - i += 1 - while i < length and pattern[i] != ']': - if pattern[i] == '\\': - square_brackets += pattern[i:i+2] - i += 2 - else: - square_brackets += pattern[i] - i += 1 - assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}' - square_brackets += ']' - i += 1 - seq.append((square_brackets, False)) - elif c == '|': - seq.append(('|', False)) - i += 1 - elif c in ('*', '+', '?'): - seq[-1] = (to_rule(seq[-1]) + c, False) - i += 1 - elif c == '{': - curly_brackets = c - i += 1 - while i < length and pattern[i] != '}': - curly_brackets += pattern[i] - i += 1 - assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}' - curly_brackets += '}' - i += 1 - nums = [s.strip() for s in curly_brackets[1:-1].split(',')] - min_times = 0 - max_times = None - try: - if len(nums) == 1: - min_times = int(nums[0]) - max_times = min_times - else: - assert len(nums) == 2 - min_times = int(nums[0]) if nums[0] else 0 - max_times = int(nums[1]) if nums[1] else None - except ValueError: - raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/') - - (sub, sub_is_literal) = seq[-1] - - if not sub_is_literal: - id = sub_rule_ids.get(sub) - if id is None: - id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub) - sub_rule_ids[sub] = id - sub = id - - seq[-1] = (_build_repetition(f'"{sub}"' if sub_is_literal else sub, min_times, max_times, item_rule_is_literal=sub_is_literal), False) - else: - literal = '' - while i < length: - if pattern[i] == '\\' and i < length - 1: - next = pattern[i + 1] - if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS: - i += 1 - literal += pattern[i] - i += 1 - else: - literal += pattern[i:i+2] - i += 2 - elif pattern[i] == '"' and not self._raw_pattern: - literal += '\\"' - i += 1 - elif pattern[i] not in NON_LITERAL_SET and \ - (i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET): - literal += pattern[i] - i += 1 - else: - break - if literal: - seq.append((literal, True)) - - return join_seq() - - return self._add_rule( - name, - to_rule(transform()) if self._raw_pattern \ - else "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space") - - - def _resolve_ref(self, ref): - ref_name = ref.split('/')[-1] - if ref_name not in self._rules and ref not in self._refs_being_resolved: - self._refs_being_resolved.add(ref) - resolved = self._refs[ref] - ref_name = self.visit(resolved, ref_name) - self._refs_being_resolved.remove(ref) - return ref_name - - def _generate_constant_rule(self, value): - return self._format_literal(json.dumps(value)) - - def visit(self, schema, name): - schema_type = schema.get('type') - schema_format = schema.get('format') - rule_name = name + '-' if name in RESERVED_NAMES else name or 'root' - - if (ref := schema.get('$ref')) is not None: - return self._add_rule(rule_name, self._resolve_ref(ref)) - - elif 'oneOf' in schema or 'anyOf' in schema: - return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf'])) - - elif isinstance(schema_type, list): - return self._add_rule(rule_name, self._generate_union_rule(name, [{'type': t} for t in schema_type])) - - elif 'const' in schema: - return self._add_rule(rule_name, self._generate_constant_rule(schema['const'])) - - elif 'enum' in schema: - rule = ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) - return self._add_rule(rule_name, rule) + def visit(self, schema: Dict[str, Any], name: str) -> str: + rule_name = name or "root" - elif schema_type in (None, 'object') and \ - ('properties' in schema or \ - ('additionalProperties' in schema and schema['additionalProperties'] is not True)): - required = set(schema.get('required', [])) - properties = list(schema.get('properties', {}).items()) - return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties'))) - - elif schema_type in (None, 'object') and 'allOf' in schema: - required = set() - properties = [] - hybrid_name = name - def add_component(comp_schema, is_required): - if (ref := comp_schema.get('$ref')) is not None: - comp_schema = self._refs[ref] - - if 'properties' in comp_schema: - for prop_name, prop_schema in comp_schema['properties'].items(): - properties.append((prop_name, prop_schema)) - if is_required: - required.add(prop_name) - - for t in schema['allOf']: - if 'anyOf' in t: - for tt in t['anyOf']: - add_component(tt, is_required=False) - else: - add_component(t, is_required=True) - - return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=[])) - - elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema): - items = schema.get('items') or schema['prefixItems'] - if isinstance(items, list): - return self._add_rule( - rule_name, - '"[" space ' + - ' "," space '.join( - self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') - for i, item in enumerate(items)) + - ' "]" space') - else: - item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') - min_items = schema.get("minItems", 0) - max_items = schema.get("maxItems") - return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' "]" space') - - elif schema_type in (None, 'string') and 'pattern' in schema: - return self._visit_pattern(schema['pattern'], rule_name) - - elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''): - return self._add_primitive( - 'root' if rule_name == 'root' else schema_format, - PRIMITIVE_RULES['uuid'] + if "$defs" in schema: + # add defs to self._defs for later inlining + for def_name, def_schema in schema["$defs"].items(): + self._defs[def_name] = def_schema + + if "oneOf" in schema or "anyOf" in schema: + rule = " | ".join( + ( + self.visit(alt_schema, f'{name}{"-" if name else ""}{i}') + for i, alt_schema in enumerate( + schema.get("oneOf") or schema["anyOf"] + ) + ) ) + return self._add_rule(rule_name, rule) - elif schema_type in (None, 'string') and f'{schema_format}-string' in STRING_FORMAT_RULES: - prim_name = f'{schema_format}-string' - return self._add_rule(rule_name, self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name])) + elif "const" in schema: + return self._add_rule(rule_name, self._format_literal(schema["const"])) - elif schema_type == 'string' and ('minLength' in schema or 'maxLength' in schema): - char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) - min_len = schema.get('minLength', 0) - max_len = schema.get('maxLength') + elif "enum" in schema: + rule = " | ".join((self._format_literal(v) for v in schema["enum"])) + return self._add_rule(rule_name, rule) - return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\"" space') + elif "$ref" in schema: + ref = schema["$ref"] + assert ref.startswith("#/$defs/"), f"Unrecognized schema: {schema}" + # inline $defs + def_name = ref[len("#/$defs/") :] + def_schema = self._defs[def_name] + return self.visit(def_schema, f'{name}{"-" if name else ""}{def_name}') + + + schema_type: Optional[str] = schema.get("type") # type: ignore + assert isinstance(schema_type, str), f"Unrecognized schema: {schema}" + + if schema_type == "object" and "properties" in schema: + # TODO: `required` keyword + if self._prop_order: + prop_order = self._prop_order + prop_pairs = sorted( + schema["properties"].items(), + # sort by position in prop_order (if specified) then by key + key=lambda kv: (prop_order.get(kv[0], len(prop_order)), kv[0]), + ) + else: + prop_pairs = schema["properties"].items() - elif (schema_type == 'object') or (len(schema) == 0): - return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object'])) + rule = '"{" space' + for i, (prop_name, prop_schema) in enumerate(prop_pairs): + prop_rule_name = self.visit( + prop_schema, f'{name}{"-" if name else ""}{prop_name}' + ) + if i > 0: + rule += ' "," space' + rule += rf' {self._format_literal(prop_name)} space ":" space {prop_rule_name}' + rule += ' "}" space' - else: - assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}' - # TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero - return self._add_primitive('root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type]) - - def _add_primitive(self, name: str, rule: BuiltinRule): - n = self._add_rule(name, rule.content) - - for dep in rule.deps: - dep_rule = PRIMITIVE_RULES.get(dep) or STRING_FORMAT_RULES.get(dep) - assert dep_rule, f'Rule {dep} not known' - if dep not in self._rules: - self._add_primitive(dep, dep_rule) - return n - - def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Union[bool, Any]): - prop_order = self._prop_order - # sort by position in prop_order (if specified) then by original order - sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))] - - prop_kv_rule_names = {} - for prop_name, prop_schema in properties: - prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') - prop_kv_rule_names[prop_name] = self._add_rule( - f'{name}{"-" if name else ""}{prop_name}-kv', - fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}' - ) - required_props = [k for k in sorted_props if k in required] - optional_props = [k for k in sorted_props if k not in required] - - if additional_properties == True or isinstance(additional_properties, dict): - sub_name = f'{name}{"-" if name else ""}additional' - value_rule = self.visit({} if additional_properties == True else additional_properties, f'{sub_name}-value') - prop_kv_rule_names["*"] = self._add_rule( - f'{sub_name}-kv', - self._add_primitive('string', PRIMITIVE_RULES['string']) + f' ":" space {value_rule}' - ) - optional_props.append("*") - - rule = '"{" space ' - rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props) - - if optional_props: - rule += ' (' - if required_props: - rule += ' "," space ( ' - - def get_recursive_refs(ks, first_is_optional): - [k, *rest] = ks - kv_rule_name = prop_kv_rule_names[k] - if k == '*': - res = self._add_rule( - f'{name}{"-" if name else ""}additional-kvs', - f'{kv_rule_name} ( "," space ' + kv_rule_name + ' )*' - ) - elif first_is_optional: - res = f'( "," space {kv_rule_name} )?' - else: - res = kv_rule_name - if len(rest) > 0: - res += ' ' + self._add_rule( - f'{name}{"-" if name else ""}{k}-rest', - get_recursive_refs(rest, first_is_optional=True) - ) - return res + return self._add_rule(rule_name, rule) - rule += ' | '.join( - get_recursive_refs(optional_props[i:], first_is_optional=False) - for i in range(len(optional_props)) + elif schema_type == "array" and "items" in schema: + # TODO `prefixItems` keyword + item_rule_name = self.visit( + schema["items"], f'{name}{"-" if name else ""}item' ) - if required_props: - rule += ' )' - rule += ' )?' - - rule += ' "}" space' + list_item_operator = f'("," space {item_rule_name})' + successive_items = "" + min_items = schema.get("minItems", 0) + if min_items > 0: + first_item = f"({item_rule_name})" + successive_items = list_item_operator * (min_items - 1) + min_items -= 1 + else: + first_item = f"({item_rule_name})?" + max_items = schema.get("maxItems") + if max_items is not None and max_items > min_items: + successive_items += (list_item_operator + "?") * (max_items - min_items - 1) + else: + successive_items += list_item_operator + "*" + rule = f'"[" space {first_item} {successive_items} "]" space' + return self._add_rule(rule_name, rule) - return rule + else: + assert schema_type in PRIMITIVE_RULES, f"Unrecognized schema: {schema}" + return self._add_rule( + "root" if rule_name == "root" else schema_type, + PRIMITIVE_RULES[schema_type], + ) def format_grammar(self): - return '\n'.join( - f'{name} ::= {rule}' - for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0]) - ) + return "\n".join((f"{name} ::= {rule}" for name, rule in self._rules.items())) def json_schema_to_gbnf(schema: str, prop_order: Optional[List[str]] = None): From 4f4266495549f691194abb357b792898fd12695e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucca=20Zen=C3=B3bio?= Date: Thu, 18 Apr 2024 02:36:25 -0300 Subject: [PATCH 075/561] feat: update grammar schema converter to match llama.cpp (#1353) * feat: improve function calling * feat:grammar * fix * fix * fix --- llama_cpp/llama_chat_format.py | 2 +- llama_cpp/llama_grammar.py | 627 +++++++++++++++++++++++++++------ 2 files changed, 523 insertions(+), 106 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 519d2f50af..eb98cbf391 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2709,4 +2709,4 @@ def chatml_function_calling( }, } - raise ValueError("Automatic streaming tool choice is not supported") + raise ValueError("Automatic streaming tool choice is not supported") \ No newline at end of file diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 9cc48a93b3..6c7b57a591 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -5,11 +5,12 @@ import sys from ctypes import * # type: ignore from enum import Enum -from itertools import islice +from itertools import islice, groupby from typing import ( Any, Callable, Dict, + Set, Generic, List, Optional, @@ -1391,145 +1392,561 @@ def print_grammar(file: TextIO, state: parse_state) -> None: # whitespace. Also maybe improves generation quality? SPACE_RULE = '" "?' -PRIMITIVE_RULES = { - "boolean": '("true" | "false") space', - "number": '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space', - "integer": '("-"? ([0-9] | [1-9] [0-9]*)) space', - "string": r""" "\"" ( - [^"\\] | - "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) - )* "\"" space """, - "null": '"null" space', -} INVALID_RULE_CHARS_RE = re.compile(r"[^a-zA-Z0-9-]+") GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"]') GRAMMAR_LITERAL_ESCAPES = {"\r": "\\r", "\n": "\\n", '"': '\\"'} +# whitespace is constrained to a single space char to prevent model "running away" in +# whitespace. Also maybe improves generation quality? +SPACE_RULE = '" "?' + + +def _build_repetition(item_rule, min_items, max_items, separator_rule=None, item_rule_is_literal=False): + if not separator_rule: + if min_items == 0 and max_items == 1: + return f'{item_rule}?' + elif min_items == 1 and max_items is None: + return f'{item_rule}+' + + result = '' + + if min_items > 0: + if item_rule_is_literal and separator_rule is None: + result = '"' + (item_rule[1:-1] * min_items) + '"' + else: + result = (f' {separator_rule} ' if separator_rule else ' ').join([item_rule] * min_items) + + def opt_repetitions(up_to_n, prefix_with_sep=False): + ''' + - n=4, no sep: '(a (a (a (a)?)?)?)?' + - n=4, sep=',', prefix: '("," a ("," a ("," a ("," a)?)?)?)?' + - n=4, sep=',', no prefix: '(a ("," a ("," a ("," a)?)?)?)?' + ''' + + content = f'{separator_rule} {item_rule}' if prefix_with_sep and separator_rule else item_rule + if up_to_n == 0: + return '' + elif up_to_n == 1: + return f'({content})?' + elif separator_rule and not prefix_with_sep: + return f'({content} {opt_repetitions(up_to_n - 1, prefix_with_sep=True)})?' + else: + return (f'({content} ' * up_to_n).rstrip() + (')?' * up_to_n) + + if min_items > 0 and max_items != min_items: + result += ' ' + + if max_items is not None: + result += opt_repetitions(max_items - min_items, prefix_with_sep=min_items > 0) + else: + item_operator = f'({separator_rule + " " if separator_rule else ""}{item_rule})' + + if min_items == 0 and separator_rule: + result = f'({item_rule} {item_operator}*)?' + else: + result += f'{item_operator}*' + + return result + + + +class BuiltinRule: + def __init__(self, content: str, deps: list = None): + self.content = content + self.deps = deps or [] + +_up_to_15_digits = _build_repetition('[0-9]', 0, 15) + +PRIMITIVE_RULES = { + 'boolean' : BuiltinRule('("true" | "false") space', []), + 'decimal-part' : BuiltinRule('[0-9] ' + _up_to_15_digits, []), + 'integral-part': BuiltinRule('[0-9] | [1-9] ' + _up_to_15_digits, []), + 'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space', ['integral-part', 'decimal-part']), + 'integer' : BuiltinRule('("-"? integral-part) space', ['integral-part']), + 'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']), + 'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', ['string', 'value']), + 'array' : BuiltinRule('"[" space ( value ("," space value)* )? "]" space', ['value']), + 'uuid' : BuiltinRule(r'"\"" ' + ' "-" '.join('[0-9a-fA-F]' * n for n in [8, 4, 4, 4, 12]) + r' "\"" space', []), + 'char' : BuiltinRule(r'[^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])', []), + 'string' : BuiltinRule(r'"\"" char* "\"" space', ['char']), + 'null' : BuiltinRule('"null" space', []), +} + +# TODO: support "uri", "email" string formats +STRING_FORMAT_RULES = { + 'date' : BuiltinRule('[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []), + 'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []), + 'date-time' : BuiltinRule('date "T" time', ['date', 'time']), + 'date-string' : BuiltinRule('"\\"" date "\\"" space', ['date']), + 'time-string' : BuiltinRule('"\\"" time "\\"" space', ['time']), + 'date-time-string': BuiltinRule('"\\"" date-time "\\"" space', ['date-time']), +} + +DOTALL = '[\\U00000000-\\U0010FFFF]' +DOT = '[^\\x0A\\x0D]' + +RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()]) + + +NON_LITERAL_SET = set('|.()[]{}*+?') +ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('[]()|{}*+?') + + + class SchemaConverter: - def __init__(self, prop_order): + def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern): self._prop_order = prop_order - self._rules = {"space": SPACE_RULE} - self._defs: Dict[str, Any] = {} - - def _format_literal(self, literal: str): - escaped: str = GRAMMAR_LITERAL_ESCAPE_RE.sub( - lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)), json.dumps(literal) + self._allow_fetch = allow_fetch + self._dotall = dotall + self._raw_pattern = raw_pattern + self._rules = { + 'space': SPACE_RULE, + } + self._refs = {} + self._refs_being_resolved = set() + + def _format_literal(self, literal): + escaped = GRAMMAR_LITERAL_ESCAPE_RE.sub( + lambda m: GRAMMAR_LITERAL_ESCAPES.get(m.group(0)), literal ) return f'"{escaped}"' - def _add_rule(self, name: str, rule: str): - esc_name = INVALID_RULE_CHARS_RE.sub("-", name) + def not_literal(self, literal: str, dotall: bool = True, maybe_escaped_underscores = False) -> str: + ''' + not_literal('a') -> '[^a]' + not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?' + ''' + assert len(literal) > 0, 'Empty literal not supported' + def recurse(i: int): + c = literal[i] + if maybe_escaped_underscores and c == '_': + yield f'[^{c}\\\\]' + yield ' | ' + yield f'"\\\\"? "{c}"' + else: + yield f'[^{c}]' + if i < len(literal) - 1: + yield ' | ' + yield self._format_literal(c) + yield ' (' + yield from recurse(i + 1) + yield ')?' + + return ''.join(('(', *recurse(0), ')')) + + def _add_rule(self, name, rule): + esc_name = INVALID_RULE_CHARS_RE.sub('-', name) if esc_name not in self._rules or self._rules[esc_name] == rule: key = esc_name else: i = 0 - while f"{esc_name}{i}" in self._rules: + while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule: i += 1 - key = f"{esc_name}{i}" + key = f'{esc_name}{i}' self._rules[key] = rule return key - def visit(self, schema: Dict[str, Any], name: str) -> str: - rule_name = name or "root" - - if "$defs" in schema: - # add defs to self._defs for later inlining - for def_name, def_schema in schema["$defs"].items(): - self._defs[def_name] = def_schema + def resolve_refs(self, schema: dict, url: str): + ''' + Resolves all $ref fields in the given schema, fetching any remote schemas, + replacing $ref with absolute reference URL and populating self._refs with the + respective referenced (sub)schema dictionaries. + ''' + def visit(n: dict): + if isinstance(n, list): + return [visit(x) for x in n] + elif isinstance(n, dict): + ref = n.get('$ref') + if ref is not None and ref not in self._refs: + if ref.startswith('https://'): + assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)' + import requests + + frag_split = ref.split('#') + base_url = frag_split[0] + + target = self._refs.get(base_url) + if target is None: + target = self.resolve_refs(requests.get(ref).json(), base_url) + self._refs[base_url] = target + + if len(frag_split) == 1 or frag_split[-1] == '': + return target + elif ref.startswith('#/'): + target = schema + ref = f'{url}{ref}' + n['$ref'] = ref + else: + raise ValueError(f'Unsupported ref {ref}') + + for sel in ref.split('#')[-1].split('/')[1:]: + assert target is not None and sel in target, f'Error resolving ref {ref}: {sel} not in {target}' + target = target[sel] + + self._refs[ref] = target + else: + for v in n.values(): + visit(v) + + return n + return visit(schema) + + def _generate_union_rule(self, name, alt_schemas): + return ' | '.join(( + self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') + for i, alt_schema in enumerate(alt_schemas) + )) + + def _visit_pattern(self, pattern, name): + ''' + Transforms a regular expression pattern into a GBNF rule. + + Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions + Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md + + Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. + + Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which + we define sub-rules to keep the output lean. + ''' + + assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"' + pattern = pattern[1:-1] + sub_rule_ids = {} + + i = 0 + length = len(pattern) + + def to_rule(s: Tuple[str, bool]) -> str: + (txt, is_literal) = s + return "\"" + txt + "\"" if is_literal else txt + + def transform() -> Tuple[str, bool]: + ''' + Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. + ''' + nonlocal i + nonlocal pattern + nonlocal sub_rule_ids + + start = i + # For each component of this sequence, store its string representation and whether it's a literal. + # We only need a flat structure here to apply repetition operators to the last item, and + # to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially + # (GBNF's syntax is luckily very close to regular expressions!) + seq: list[Tuple[str, bool]] = [] + + def get_dot(): + if self._dotall: + rule = DOTALL + else: + # Accept any character... except \n and \r line break chars (\x0A and \xOD) + rule = DOT + return self._add_rule(f'dot', rule) + + def join_seq(): + nonlocal seq + ret = [] + for is_literal, g in groupby(seq, lambda x: x[1]): + if is_literal: + ret.append((''.join(x[0] for x in g), True)) + else: + ret.extend(g) + if len(ret) == 1: + return ret[0] + return (' '.join(to_rule(x) for x in seq), False) + + while i < length: + c = pattern[i] + if c == '.': + seq.append((get_dot(), False)) + i += 1 + elif c == '(': + i += 1 + if i < length: + assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' + seq.append((f'({to_rule(transform())})', False)) + elif c == ')': + i += 1 + assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}' + return join_seq() + elif c == '[': + square_brackets = c + i += 1 + while i < length and pattern[i] != ']': + if pattern[i] == '\\': + square_brackets += pattern[i:i+2] + i += 2 + else: + square_brackets += pattern[i] + i += 1 + assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}' + square_brackets += ']' + i += 1 + seq.append((square_brackets, False)) + elif c == '|': + seq.append(('|', False)) + i += 1 + elif c in ('*', '+', '?'): + seq[-1] = (to_rule(seq[-1]) + c, False) + i += 1 + elif c == '{': + curly_brackets = c + i += 1 + while i < length and pattern[i] != '}': + curly_brackets += pattern[i] + i += 1 + assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}' + curly_brackets += '}' + i += 1 + nums = [s.strip() for s in curly_brackets[1:-1].split(',')] + min_times = 0 + max_times = None + try: + if len(nums) == 1: + min_times = int(nums[0]) + max_times = min_times + else: + assert len(nums) == 2 + min_times = int(nums[0]) if nums[0] else 0 + max_times = int(nums[1]) if nums[1] else None + except ValueError: + raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/') + + (sub, sub_is_literal) = seq[-1] + + if not sub_is_literal: + id = sub_rule_ids.get(sub) + if id is None: + id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub) + sub_rule_ids[sub] = id + sub = id + + seq[-1] = (_build_repetition(f'"{sub}"' if sub_is_literal else sub, min_times, max_times, item_rule_is_literal=sub_is_literal), False) + else: + literal = '' + while i < length: + if pattern[i] == '\\' and i < length - 1: + next = pattern[i + 1] + if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS: + i += 1 + literal += pattern[i] + i += 1 + else: + literal += pattern[i:i+2] + i += 2 + elif pattern[i] == '"' and not self._raw_pattern: + literal += '\\"' + i += 1 + elif pattern[i] not in NON_LITERAL_SET and \ + (i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET): + literal += pattern[i] + i += 1 + else: + break + if literal: + seq.append((literal, True)) + + return join_seq() + + return self._add_rule( + name, + to_rule(transform()) if self._raw_pattern \ + else "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space") + + + def _resolve_ref(self, ref): + ref_name = ref.split('/')[-1] + if ref_name not in self._rules and ref not in self._refs_being_resolved: + self._refs_being_resolved.add(ref) + resolved = self._refs[ref] + ref_name = self.visit(resolved, ref_name) + self._refs_being_resolved.remove(ref) + return ref_name + + def _generate_constant_rule(self, value): + return self._format_literal(json.dumps(value)) + + def visit(self, schema, name): + schema_type = schema.get('type') + schema_format = schema.get('format') + rule_name = name + '-' if name in RESERVED_NAMES else name or 'root' + + if (ref := schema.get('$ref')) is not None: + return self._add_rule(rule_name, self._resolve_ref(ref)) + + elif 'oneOf' in schema or 'anyOf' in schema: + return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf'])) + + elif isinstance(schema_type, list): + return self._add_rule(rule_name, self._generate_union_rule(name, [{'type': t} for t in schema_type])) + + elif 'const' in schema: + return self._add_rule(rule_name, self._generate_constant_rule(schema['const'])) + + elif 'enum' in schema: + rule = ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + return self._add_rule(rule_name, rule) - if "oneOf" in schema or "anyOf" in schema: - rule = " | ".join( - ( - self.visit(alt_schema, f'{name}{"-" if name else ""}{i}') - for i, alt_schema in enumerate( - schema.get("oneOf") or schema["anyOf"] - ) - ) + elif schema_type in (None, 'object') and \ + ('properties' in schema or \ + ('additionalProperties' in schema and schema['additionalProperties'] is not True)): + required = set(schema.get('required', [])) + properties = list(schema.get('properties', {}).items()) + return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties'))) + + elif schema_type in (None, 'object') and 'allOf' in schema: + required = set() + properties = [] + hybrid_name = name + def add_component(comp_schema, is_required): + if (ref := comp_schema.get('$ref')) is not None: + comp_schema = self._refs[ref] + + if 'properties' in comp_schema: + for prop_name, prop_schema in comp_schema['properties'].items(): + properties.append((prop_name, prop_schema)) + if is_required: + required.add(prop_name) + + for t in schema['allOf']: + if 'anyOf' in t: + for tt in t['anyOf']: + add_component(tt, is_required=False) + else: + add_component(t, is_required=True) + + return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=[])) + + elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema): + items = schema.get('items') or schema['prefixItems'] + if isinstance(items, list): + return self._add_rule( + rule_name, + '"[" space ' + + ' "," space '.join( + self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') + for i, item in enumerate(items)) + + ' "]" space') + else: + item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') + min_items = schema.get("minItems", 0) + max_items = schema.get("maxItems") + return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' "]" space') + + elif schema_type in (None, 'string') and 'pattern' in schema: + return self._visit_pattern(schema['pattern'], rule_name) + + elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''): + return self._add_primitive( + 'root' if rule_name == 'root' else schema_format, + PRIMITIVE_RULES['uuid'] ) - return self._add_rule(rule_name, rule) - elif "const" in schema: - return self._add_rule(rule_name, self._format_literal(schema["const"])) + elif schema_type in (None, 'string') and f'{schema_format}-string' in STRING_FORMAT_RULES: + prim_name = f'{schema_format}-string' + return self._add_rule(rule_name, self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name])) - elif "enum" in schema: - rule = " | ".join((self._format_literal(v) for v in schema["enum"])) - return self._add_rule(rule_name, rule) + elif schema_type == 'string' and ('minLength' in schema or 'maxLength' in schema): + char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) + min_len = schema.get('minLength', 0) + max_len = schema.get('maxLength') - elif "$ref" in schema: - ref = schema["$ref"] - assert ref.startswith("#/$defs/"), f"Unrecognized schema: {schema}" - # inline $defs - def_name = ref[len("#/$defs/") :] - def_schema = self._defs[def_name] - return self.visit(def_schema, f'{name}{"-" if name else ""}{def_name}') - - - schema_type: Optional[str] = schema.get("type") # type: ignore - assert isinstance(schema_type, str), f"Unrecognized schema: {schema}" - - if schema_type == "object" and "properties" in schema: - # TODO: `required` keyword - if self._prop_order: - prop_order = self._prop_order - prop_pairs = sorted( - schema["properties"].items(), - # sort by position in prop_order (if specified) then by key - key=lambda kv: (prop_order.get(kv[0], len(prop_order)), kv[0]), - ) - else: - prop_pairs = schema["properties"].items() + return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\"" space') - rule = '"{" space' - for i, (prop_name, prop_schema) in enumerate(prop_pairs): - prop_rule_name = self.visit( - prop_schema, f'{name}{"-" if name else ""}{prop_name}' - ) - if i > 0: - rule += ' "," space' - rule += rf' {self._format_literal(prop_name)} space ":" space {prop_rule_name}' - rule += ' "}" space' - - return self._add_rule(rule_name, rule) + elif (schema_type == 'object') or (len(schema) == 0): + return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object'])) - elif schema_type == "array" and "items" in schema: - # TODO `prefixItems` keyword - item_rule_name = self.visit( - schema["items"], f'{name}{"-" if name else ""}item' + else: + assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}' + # TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero + return self._add_primitive('root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type]) + + def _add_primitive(self, name: str, rule: BuiltinRule): + n = self._add_rule(name, rule.content) + + for dep in rule.deps: + dep_rule = PRIMITIVE_RULES.get(dep) or STRING_FORMAT_RULES.get(dep) + assert dep_rule, f'Rule {dep} not known' + if dep not in self._rules: + self._add_primitive(dep, dep_rule) + return n + + def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Union[bool, Any]): + prop_order = self._prop_order + # sort by position in prop_order (if specified) then by original order + sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))] + + prop_kv_rule_names = {} + for prop_name, prop_schema in properties: + prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') + prop_kv_rule_names[prop_name] = self._add_rule( + f'{name}{"-" if name else ""}{prop_name}-kv', + fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}' ) - list_item_operator = f'("," space {item_rule_name})' - successive_items = "" - min_items = schema.get("minItems", 0) - if min_items > 0: - first_item = f"({item_rule_name})" - successive_items = list_item_operator * (min_items - 1) - min_items -= 1 - else: - first_item = f"({item_rule_name})?" - max_items = schema.get("maxItems") - if max_items is not None and max_items > min_items: - successive_items += (list_item_operator + "?") * (max_items - min_items - 1) - else: - successive_items += list_item_operator + "*" - rule = f'"[" space {first_item} {successive_items} "]" space' - return self._add_rule(rule_name, rule) + required_props = [k for k in sorted_props if k in required] + optional_props = [k for k in sorted_props if k not in required] + + if additional_properties == True or isinstance(additional_properties, dict): + sub_name = f'{name}{"-" if name else ""}additional' + value_rule = self.visit({} if additional_properties == True else additional_properties, f'{sub_name}-value') + prop_kv_rule_names["*"] = self._add_rule( + f'{sub_name}-kv', + self._add_primitive('string', PRIMITIVE_RULES['string']) + f' ":" space {value_rule}' + ) + optional_props.append("*") + + rule = '"{" space ' + rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props) + + if optional_props: + rule += ' (' + if required_props: + rule += ' "," space ( ' + + def get_recursive_refs(ks, first_is_optional): + [k, *rest] = ks + kv_rule_name = prop_kv_rule_names[k] + if k == '*': + res = self._add_rule( + f'{name}{"-" if name else ""}additional-kvs', + f'{kv_rule_name} ( "," space ' + kv_rule_name + ' )*' + ) + elif first_is_optional: + res = f'( "," space {kv_rule_name} )?' + else: + res = kv_rule_name + if len(rest) > 0: + res += ' ' + self._add_rule( + f'{name}{"-" if name else ""}{k}-rest', + get_recursive_refs(rest, first_is_optional=True) + ) + return res - else: - assert schema_type in PRIMITIVE_RULES, f"Unrecognized schema: {schema}" - return self._add_rule( - "root" if rule_name == "root" else schema_type, - PRIMITIVE_RULES[schema_type], + rule += ' | '.join( + get_recursive_refs(optional_props[i:], first_is_optional=False) + for i in range(len(optional_props)) ) + if required_props: + rule += ' )' + rule += ' )?' - def format_grammar(self): - return "\n".join((f"{name} ::= {rule}" for name, rule in self._rules.items())) + rule += ' "}" space' + return rule + def format_grammar(self): + return '\n'.join( + f'{name} ::= {rule}' + for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0]) + ) def json_schema_to_gbnf(schema: str, prop_order: Optional[List[str]] = None): prop_order = prop_order or [] schema = json.loads(schema) prop_order = {name: idx for idx, name in enumerate(prop_order)} - converter = SchemaConverter(prop_order) + converter = SchemaConverter(prop_order=prop_order, allow_fetch=False, dotall=False, raw_pattern=False) + schema = converter.resolve_refs(schema, "stdin") converter.visit(schema, "") return converter.format_grammar() From a128c80500046f31d6411f75438d0caf66060ace Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 18 Apr 2024 01:39:45 -0400 Subject: [PATCH 076/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8dd1ec8b3f..3b8f1ec4b1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8dd1ec8b3ffbfa2d26e82e672cea89f5eeb2f141 +Subproject commit 3b8f1ec4b18770531d0b1d792f3edf08254e4f0c From 893a27a7364366cc195dcc50aadfa75d95bb9319 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 18 Apr 2024 01:43:39 -0400 Subject: [PATCH 077/561] chore: Bump version --- CHANGELOG.md | 8 ++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c67498ecda..cf1efeec4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.62] + +- feat: Update llama.cpp to ggerganov/llama.cpp@3b8f1ec4b18770531d0b1d792f3edf08254e4f0c +- feat: update grammar schema converter to match llama.cpp by @themrzmaster in #1353 +- feat: add disable_ping_events flag by @khimaros in #1257 +- feat: Make saved state more compact on-disk by @tc-wolf in #1296 +- feat: Use all available CPUs for batch processing by @ddh0 in #1345 + ## [0.2.61] - feat: Update llama.cpp to ggerganov/llama.cpp@ba5e134e073ec6837078c874aba44a702944a676 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 2382db9b29..828390a7c4 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.61" \ No newline at end of file +__version__ = "0.2.62" \ No newline at end of file From d17c1887a364201d292e2959c45a7f144ff240b2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 19 Apr 2024 23:58:16 -0400 Subject: [PATCH 078/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 3b8f1ec4b1..0e4802b2ec 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 3b8f1ec4b18770531d0b1d792f3edf08254e4f0c +Subproject commit 0e4802b2ecbaab04b4f829fde4a3096ca19c84b5 From cc81afebf04d26ca1ac3cf72f23f18da6ab58588 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 20 Apr 2024 00:00:53 -0400 Subject: [PATCH 079/561] feat: Add stopping_criteria to ChatFormatter, allow stopping on arbitrary token ids, fixes llama3 instruct --- llama_cpp/llama.py | 5 ++++- llama_cpp/llama_chat_format.py | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 5a0111bbe6..818be827da 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -426,7 +426,10 @@ def __init__( print(f"Using chat bos_token: {bos_token}", file=sys.stderr) self.chat_handler = llama_chat_format.Jinja2ChatFormatter( - template=template, eos_token=eos_token, bos_token=bos_token + template=template, + eos_token=eos_token, + bos_token=bos_token, + stop_token_ids=[eos_token_id], ).to_chat_handler() if self.chat_format is None and self.chat_handler is None: diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index eb98cbf391..189ccb0c6a 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -10,6 +10,9 @@ import jinja2 +import numpy as np +import numpy.typing as npt + import llama_cpp.llama as llama import llama_cpp.llama_types as llama_types import llama_cpp.llama_grammar as llama_grammar @@ -150,6 +153,7 @@ class ChatFormatterResponse: prompt: str stop: Optional[Union[str, List[str]]] = None + stopping_criteria: Optional[llama.StoppingCriteriaList] = None class ChatFormatter(Protocol): @@ -173,12 +177,14 @@ def __init__( eos_token: str, bos_token: str, add_generation_prompt: bool = True, + stop_token_ids: Optional[List[int]] = None, ): """A chat formatter that uses jinja2 templates to format the prompt.""" self.template = template self.eos_token = eos_token self.bos_token = bos_token self.add_generation_prompt = add_generation_prompt + self.stop_token_ids = set(stop_token_ids) if stop_token_ids is not None else None self._environment = jinja2.Environment( loader=jinja2.BaseLoader(), @@ -211,7 +217,16 @@ def raise_exception(message: str): tool_choice=tool_choice, ) - return ChatFormatterResponse(prompt=prompt, stop=[self.eos_token]) + stopping_criteria = None + if self.stop_token_ids is not None: + def stop_on_last_token( + tokens: npt.NDArray[np.intc], + logits: npt.NDArray[np.single] + ) -> bool: + return tokens[-1] in self.stop_token_ids + stopping_criteria = llama.StoppingCriteriaList([stop_on_last_token]) + + return ChatFormatterResponse(prompt=prompt, stop=[self.eos_token], stopping_criteria=stopping_criteria) def to_chat_handler(self) -> LlamaChatCompletionHandler: return chat_formatter_to_chat_completion_handler(self) @@ -533,6 +548,10 @@ def chat_completion_handler( rstop = result.stop if isinstance(result.stop, list) else [result.stop] stop = stop + rstop + stopping_criteria = None + if result.stopping_criteria is not None: + stopping_criteria = result.stopping_criteria + if response_format is not None and response_format["type"] == "json_object": grammar = _grammar_for_response_format(response_format, verbose=llama.verbose) @@ -598,6 +617,7 @@ def chat_completion_handler( mirostat_eta=mirostat_eta, model=model, logits_processor=logits_processor, + stopping_criteria=stopping_criteria, grammar=grammar, logit_bias=logit_bias, ) From 02812148635bf6337ffc7d1abb34093f4065df88 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 20 Apr 2024 00:09:37 -0400 Subject: [PATCH 080/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf1efeec4d..4c7e267fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.63] + +- feat: Update llama.cpp to ggerganov/llama.cpp@0e4802b2ecbaab04b4f829fde4a3096ca19c84b5 +- feat: Add stopping_criteria to ChatFormatter, allow stopping on arbitrary token ids, fixes llama3 instruct by @abetlen in cc81afebf04d26ca1ac3cf72f23f18da6ab58588 + ## [0.2.62] - feat: Update llama.cpp to ggerganov/llama.cpp@3b8f1ec4b18770531d0b1d792f3edf08254e4f0c diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 828390a7c4..f2c7aba5b2 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.62" \ No newline at end of file +__version__ = "0.2.63" \ No newline at end of file From 159cc4e5d924804c9776355011d5378dc8d5d9f4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 21 Apr 2024 20:46:40 -0400 Subject: [PATCH 081/561] feat: Update llama.cpp --- llama_cpp/_internals.py | 14 +++++++------- llama_cpp/llama_cpp.py | 28 +++++++++++++++++++++++++--- vendor/llama.cpp | 2 +- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 79f6543a89..ff2d657e41 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -181,20 +181,20 @@ def tokenize(self, text: bytes, add_bos: bool, special: bool): ) return list(tokens[:n_tokens]) - def token_to_piece(self, token: int) -> bytes: + def token_to_piece(self, token: int, special: bool = False) -> bytes: assert self.model is not None buf = ctypes.create_string_buffer(32) - llama_cpp.llama_token_to_piece(self.model, token, buf, 32) + llama_cpp.llama_token_to_piece(self.model, token, buf, 32, special) return bytes(buf) - def detokenize(self, tokens: List[int]) -> bytes: + def detokenize(self, tokens: List[int], special: bool = False) -> bytes: assert self.model is not None output = b"" size = 32 buffer = (ctypes.c_char * size)() for token in tokens: n = llama_cpp.llama_token_to_piece( - self.model, llama_cpp.llama_token(token), buffer, size + self.model, llama_cpp.llama_token(token), buffer, size, special ) assert n <= size output += bytes(buffer[:n]) @@ -597,13 +597,13 @@ def _tokenize(model: _LlamaModel, text: str, add_bos: bool, special: bool) -> li return list(result) -def _token_to_piece(model: _LlamaModel, token: int) -> str: +def _token_to_piece(model: _LlamaModel, token: int, special: bool = False) -> str: assert model.model is not None result = (ctypes.c_char * 8)(0) - n_tokens = llama_cpp.llama_token_to_piece(model.model, token, result, len(result)) + n_tokens = llama_cpp.llama_token_to_piece(model.model, token, result, len(result), special) if n_tokens < 0: result = (ctypes.c_char * -n_tokens)(0) - check = llama_cpp.llama_token_to_piece(model.model, token, result, len(result)) + check = llama_cpp.llama_token_to_piece(model.model, token, result, len(result), special) if check != -n_tokens: raise RuntimeError(f"Failed to get piece: token={token}") else: diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2450d11e35..c2b909ec4a 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -2380,6 +2380,18 @@ def llama_token_get_type( ... +# // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) +# LLAMA_API bool llama_token_is_eog(const struct llama_model * model, llama_token token); +@ctypes_function( + "llama_token_is_eog", [llama_model_p_ctypes, llama_token], ctypes.c_bool +) +def llama_token_is_eog( + model: llama_model_p, token: Union[llama_token, int], / +) -> bool: + """Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)""" + ... + + # // Special tokens @@ -2434,7 +2446,7 @@ def llama_add_eos_token(model: llama_model_p, /) -> int: ... -# // codellama infill tokens +# // Codellama infill tokens # LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix @ctypes_function("llama_token_prefix", [llama_model_p_ctypes], llama_token) def llama_token_prefix(model: llama_model_p) -> int: @@ -2524,11 +2536,13 @@ def llama_tokenize( # // Uses the vocabulary in the provided context. # // Does not write null terminator to the buffer. # // User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens. +# // @param special If true, special tokens are rendered in the output. # LLAMA_API int32_t llama_token_to_piece( # const struct llama_model * model, # llama_token token, # char * buf, -# int32_t length); +# int32_t length, +# bool special); @ctypes_function( "llama_token_to_piece", [ @@ -2536,6 +2550,7 @@ def llama_tokenize( llama_token, ctypes.c_char_p, ctypes.c_int32, + ctypes.c_bool, ], ctypes.c_int32, ) @@ -2544,13 +2559,20 @@ def llama_token_to_piece( token: Union[llama_token, int], buf: Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]], length: Union[ctypes.c_int, int], + special: Union[ctypes.c_bool, bool], /, ) -> int: """Token Id -> Piece. Uses the vocabulary in the provided context. Does not write null terminator to the buffer. User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens. - """ + + Args: + model: The model to use for tokenization. + token: The token to convert. + buf: The buffer to write the token to. + length: The length of the buffer. + special: If true, special tokens are rendered in the output.""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 3b8f1ec4b1..5cf5e7d490 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 3b8f1ec4b18770531d0b1d792f3edf08254e4f0c +Subproject commit 5cf5e7d490dfdd2e70bface2d35dfd14aa44b4fb From d40a250ef3cfaa8224d12c83776a2f1de96ae3d1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 22 Apr 2024 00:35:47 -0400 Subject: [PATCH 082/561] feat: Use new llama_token_is_eog in create_completions --- llama_cpp/llama.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 818be827da..0a576d4cfb 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1034,7 +1034,8 @@ def logit_bias_processor( logits_processor=logits_processor, grammar=grammar, ): - if token == self._token_eos: + assert self._model.model is not None + if llama_cpp.llama_token_is_eog(self._model.model, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) finish_reason = "stop" break From 617d536e1c4f9e84e551828897f5e2b8004539cb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 23 Apr 2024 02:31:40 -0400 Subject: [PATCH 083/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 5cf5e7d490..4e96a812b3 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 5cf5e7d490dfdd2e70bface2d35dfd14aa44b4fb +Subproject commit 4e96a812b3ce7322a29a3008db2ed73d9087b176 From 8559e8ce88b7c7343004eeccb7333b806034b01c Mon Sep 17 00:00:00 2001 From: abk16 Date: Tue, 23 Apr 2024 06:33:29 +0000 Subject: [PATCH 084/561] feat: Add Llama-3 chat format (#1371) * feat: Add Llama-3 chat format * feat: Auto-detect Llama-3 chat format from gguf template * feat: Update llama.cpp to b2715 Includes proper Llama-3 <|eot_id|> token handling. --------- Co-authored-by: Andrei Betlen --- llama_cpp/llama_chat_format.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 189ccb0c6a..17b570a0fd 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -35,6 +35,9 @@ # Source: https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1/blob/main/tokenizer_config.json MIXTRAL_INSTRUCT_CHAT_TEMPLATE = "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}" +# Source: https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct/blob/main/tokenizer_config.json +LLAMA3_INSTRUCT_CHAT_TEMPLATE = "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}" + ### Chat Completion Handler ### @@ -729,6 +732,9 @@ def guess_chat_format_from_gguf_metadata(metadata: Dict[str, str]) -> Optional[s metadata["tokenizer.chat_template"] == MIXTRAL_INSTRUCT_CHAT_TEMPLATE): return "mistral-instruct" + if metadata["tokenizer.chat_template"] == LLAMA3_INSTRUCT_CHAT_TEMPLATE: + return "llama-3" + return None @@ -920,6 +926,26 @@ def format_llama2( return ChatFormatterResponse(prompt=_prompt) +# Chat format for Llama-3 models, see more details at: +# https://github.com/meta-llama/llama3/blob/main/llama/tokenizer.py#L202-L229 +@register_chat_format("llama-3") +def format_llama3( + messages: List[llama_types.ChatCompletionRequestMessage], + **kwargs: Any, +) -> ChatFormatterResponse: + _roles = dict( + system="<|start_header_id|>system<|end_header_id|>\n\n", + user="<|start_header_id|>user<|end_header_id|>\n\n", + assistant="<|start_header_id|>assistant<|end_header_id|>\n\n", + ) + _begin_token = "<|begin_of_text|>" + _sep = "<|eot_id|>" + _messages = _map_roles(messages, _roles) + _messages.append((_roles["assistant"], None)) + _prompt = _format_no_colon_single(_begin_token, _messages, _sep) + return ChatFormatterResponse(prompt=_prompt, stop=_sep) + + @register_chat_format("alpaca") def format_alpaca( messages: List[llama_types.ChatCompletionRequestMessage], From 507c1da0663e80d01a5a608355f523c0a791971d Mon Sep 17 00:00:00 2001 From: Geza Velkey Date: Tue, 23 Apr 2024 08:34:15 +0200 Subject: [PATCH 085/561] fix: Update scikit-build-core build dependency avoid bug in 0.9.1 (#1370) cmake [options] cmake [options] cmake [options] -S -B Specify a source directory to (re-)generate a build system for it in the current working directory. Specify an existing build directory to re-generate its build system. Run 'cmake --help' for more information. issue --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e2bbb4b0c2..8345cb1f09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["scikit-build-core[pyproject]>=0.5.1"] +requires = ["scikit-build-core[pyproject]>=0.9.2"] build-backend = "scikit_build_core.build" [project] From 53ebcc8bb5beb912bda4baa44ae27a1e3a7eabd0 Mon Sep 17 00:00:00 2001 From: Sean Bailey <34511443+sean-bailey@users.noreply.github.com> Date: Tue, 23 Apr 2024 02:35:38 -0400 Subject: [PATCH 086/561] feat(server): Provide ability to dynamically allocate all threads if desired using `-1` (#1364) --- llama_cpp/server/settings.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 934aecd919..eab5a8aacb 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -3,7 +3,7 @@ import multiprocessing from typing import Optional, List, Literal, Union -from pydantic import Field +from pydantic import Field, root_validator from pydantic_settings import BaseSettings import llama_cpp @@ -67,12 +67,12 @@ class ModelSettings(BaseSettings): n_threads: int = Field( default=max(multiprocessing.cpu_count() // 2, 1), ge=1, - description="The number of threads to use.", + description="The number of threads to use. Use -1 for max cpu threads", ) n_threads_batch: int = Field( default=max(multiprocessing.cpu_count(), 1), ge=0, - description="The number of threads to use when batch processing.", + description="The number of threads to use when batch processing. Use -1 for max cpu threads", ) rope_scaling_type: int = Field( default=llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED @@ -173,6 +173,16 @@ class ModelSettings(BaseSettings): default=True, description="Whether to print debug information." ) + @root_validator(pre=True) # pre=True to ensure this runs before any other validation + def set_dynamic_defaults(cls, values): + # If n_threads or n_threads_batch is -1, set it to multiprocessing.cpu_count() + cpu_count = multiprocessing.cpu_count() + if values.get('n_threads', 0) == -1: + values['n_threads'] = cpu_count + if values.get('n_threads_batch', 0) == -1: + values['n_threads_batch'] = cpu_count + return values + class ServerSettings(BaseSettings): """Server settings used to configure the FastAPI and Uvicorn server.""" From 611781f5319719a3d05fefccbbf0cc321742a026 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 23 Apr 2024 02:48:09 -0400 Subject: [PATCH 087/561] ci: Build arm64 wheels. Closes #1342 --- .github/workflows/build-and-release.yaml | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 76b5f7fcf0..07742f1dce 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -41,6 +41,35 @@ jobs: with: path: ./wheelhouse/*.whl + build_arm64_wheels: + name: Build arm64 wheels + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/arm64 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + env: + CIBW_SKIP: "*musllinux* pp*" + CIBW_REPAIR_WHEEL_COMMAND: "" + CIBW_ARCHS: "aarch64" + CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" + with: + output-dir: wheelhouse/ + + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.version }} + path: wheelhouse/*.whl + build_sdist: name: Build source distribution runs-on: ubuntu-latest @@ -65,7 +94,7 @@ jobs: release: name: Release - needs: [build_wheels, build_sdist] + needs: [build_wheels, build_arm64_wheels, build_sdist] runs-on: ubuntu-latest steps: From c50d3300d2a09c98765be7f2c05b7e4fd0b4232e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 23 Apr 2024 02:53:20 -0400 Subject: [PATCH 088/561] chore: Bump version --- CHANGELOG.md | 9 +++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c7e267fed..25b8835528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.64] + +- feat: Update llama.cpp to ggerganov/llama.cpp@4e96a812b3ce7322a29a3008db2ed73d9087b176 +- feat: Add `llama-3` chat format by @andreabak in #1371 +- feat: Use new llama_token_is_eog in create_completions by @abetlen in d40a250ef3cfaa8224d12c83776a2f1de96ae3d1 +- feat(server): Provide ability to dynamically allocate all threads if desired using -1 by @sean-bailey in #1364 +- ci: Build arm64 wheels by @gaby in 611781f5319719a3d05fefccbbf0cc321742a026 +- fix: Update scikit-build-core build dependency avoid bug in 0.9.1 by @evelkey in #1370 + ## [0.2.63] - feat: Update llama.cpp to ggerganov/llama.cpp@0e4802b2ecbaab04b4f829fde4a3096ca19c84b5 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index f2c7aba5b2..f736458741 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.63" \ No newline at end of file +__version__ = "0.2.64" \ No newline at end of file From 2a9979fce16ea647918473b85b6f03effb7e1e4c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 25 Apr 2024 02:48:26 -0400 Subject: [PATCH 089/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 12 +++++++++--- vendor/llama.cpp | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index c2b909ec4a..7f5d2658d9 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1167,13 +1167,19 @@ def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... -# LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model); +# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); +@ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) +def llama_pooling_type(ctx: llama_context_p, /) -> int: + ... + + +# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) def llama_vocab_type(model: llama_model_p, /) -> int: ... -# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); +# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) def llama_rope_type(model: llama_model_p, /) -> int: ... @@ -3091,7 +3097,7 @@ def llama_sample_token_greedy( ... -# /// @details Randomly selects a token from the candidates based on their probabilities. +# /// @details Randomly selects a token from the candidates based on their probabilities using the RNG of ctx. # LLAMA_API llama_token llama_sample_token( # struct llama_context * ctx, # llama_token_data_array * candidates); diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4e96a812b3..784e11dea1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4e96a812b3ce7322a29a3008db2ed73d9087b176 +Subproject commit 784e11dea1f5ce9638851b2b0dddb107e2a609c8 From de37420fcf52f759768cec51ddbd5cc768cd9ef3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 25 Apr 2024 03:08:32 -0400 Subject: [PATCH 090/561] fix(ci): Fix python macos test runners issue --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 77df546973..06af61f00d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -57,7 +57,7 @@ jobs: build-macos: - runs-on: macos-latest + runs-on: macos-13 strategy: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] From 266abfc1a36a4b33c0da3f3689676827df3e6a88 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 25 Apr 2024 03:09:46 -0400 Subject: [PATCH 091/561] fix(ci): Fix metal tests as well --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 06af61f00d..6b0e7e8c67 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -107,7 +107,7 @@ jobs: build-macos-metal: - runs-on: macos-latest + runs-on: macos-13 steps: - uses: actions/checkout@v3 From 7f52335c50be425d7dce6302ae38ecff87b0ee74 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 25 Apr 2024 21:21:29 -0400 Subject: [PATCH 092/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 4 ++++ vendor/llama.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 7f5d2658d9..3b96adc4c7 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -811,6 +811,7 @@ class llama_context_params(ctypes.Structure): # bool quantize_output_tensor; // quantize output.weight # bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored # bool pure; // quantize all tensors to the default type +# bool keep_split; // quantize to the same number of shards # void * imatrix; // pointer to importance matrix data # void * kv_overrides; // pointer to vector containing overrides # } llama_model_quantize_params; @@ -826,6 +827,7 @@ class llama_model_quantize_params(ctypes.Structure): quantize_output_tensor (bool): quantize output.weight only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored pure (bool): quantize all tensors to the default type + keep_split (bool): quantize to the same number of shards imatrix (ctypes.c_void_p): pointer to importance matrix data kv_overrides (ctypes.c_void_p): pointer to vector containing overrides """ @@ -839,6 +841,7 @@ class llama_model_quantize_params(ctypes.Structure): quantize_output_tensor: bool only_copy: bool pure: bool + keep_split: bool imatrix: ctypes.c_void_p kv_overrides: ctypes.c_void_p @@ -851,6 +854,7 @@ class llama_model_quantize_params(ctypes.Structure): ("quantize_output_tensor", ctypes.c_bool), ("only_copy", ctypes.c_bool), ("pure", ctypes.c_bool), + ("keep_split", ctypes.c_bool), ("imatrix", ctypes.c_void_p), ("kv_overrides", ctypes.c_void_p), ] diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 784e11dea1..46e12c4692 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 784e11dea1f5ce9638851b2b0dddb107e2a609c8 +Subproject commit 46e12c4692a37bdd31a0432fc5153d7d22bc7f72 From fcfea66857c73c9b95326835debb80dbc0b76f17 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 25 Apr 2024 21:21:48 -0400 Subject: [PATCH 093/561] fix: pydantic deprecation warning --- llama_cpp/server/settings.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index eab5a8aacb..0c858f9fe9 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -2,8 +2,10 @@ import multiprocessing -from typing import Optional, List, Literal, Union -from pydantic import Field, root_validator +from typing import Optional, List, Literal, Union, Dict, cast +from typing_extensions import Self + +from pydantic import Field, model_validator from pydantic_settings import BaseSettings import llama_cpp @@ -173,15 +175,16 @@ class ModelSettings(BaseSettings): default=True, description="Whether to print debug information." ) - @root_validator(pre=True) # pre=True to ensure this runs before any other validation - def set_dynamic_defaults(cls, values): + @model_validator(mode="before") # pre=True to ensure this runs before any other validation + def set_dynamic_defaults(self) -> Self: # If n_threads or n_threads_batch is -1, set it to multiprocessing.cpu_count() cpu_count = multiprocessing.cpu_count() + values = cast(Dict[str, int], self) if values.get('n_threads', 0) == -1: values['n_threads'] = cpu_count if values.get('n_threads_batch', 0) == -1: values['n_threads_batch'] = cpu_count - return values + return self class ServerSettings(BaseSettings): From f6ed21f9a26874625b020996fed1f107c1990f50 Mon Sep 17 00:00:00 2001 From: Douglas Hanley Date: Thu, 25 Apr 2024 20:32:44 -0500 Subject: [PATCH 094/561] feat: Allow for possibly non-pooled embeddings (#1380) * allow for possibly non-pooled embeddings * add more to embeddings section in README.md --------- Co-authored-by: Andrei --- README.md | 8 +++++- llama_cpp/_internals.py | 14 ++++++++++ llama_cpp/llama.py | 57 +++++++++++++++++++++++++++------------- llama_cpp/llama_cpp.py | 6 +++++ llama_cpp/llama_types.py | 2 +- 5 files changed, 67 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index cd41a3804e..b5e7d20350 100644 --- a/README.md +++ b/README.md @@ -575,7 +575,7 @@ llama = Llama( ### Embeddings -To generate text embeddings use [`create_embedding`](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_embedding). +To generate text embeddings use [`create_embedding`](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.create_embedding) or [`embed`](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.embed). Note that you must pass `embedding=True` to the constructor upon model creation for these to work properly. ```python import llama_cpp @@ -589,6 +589,12 @@ embeddings = llm.create_embedding("Hello, world!") embeddings = llm.create_embedding(["Hello, world!", "Goodbye, world!"]) ``` +There are two primary notions of embeddings in a Transformer-style model: *token level* and *sequence level*. Sequence level embeddings are produced by "pooling" token level embeddings together, usually by averaging them or using the first token. + +Models that are explicitly geared towards embeddings will usually return sequence level embeddings by default, one for each input string. Non-embedding models such as those designed for text generation will typically return only token level embeddings, one for each token in each sequence. Thus the dimensionality of the return type will be one higher for token level embeddings. + +It is possible to control pooling behavior in some cases using the `pooling_type` flag on model creation. You can ensure token level embeddings from any model using `LLAMA_POOLING_TYPE_NONE`. The reverse, getting a generation oriented model to yield sequence level embeddings is currently not possible, but you can always do the pooling manually. + ### Adjusting the Context Window The context window of the Llama models determines the maximum number of tokens that can be processed at once. By default, this is set to 512 tokens, but can be adjusted based on your requirements. diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index ff2d657e41..cc3d989287 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -273,6 +273,10 @@ def n_ctx(self) -> int: assert self.ctx is not None return llama_cpp.llama_n_ctx(self.ctx) + def pooling_type(self) -> int: + assert self.ctx is not None + return llama_cpp.llama_pooling_type(self.ctx) + def kv_cache_clear(self): assert self.ctx is not None llama_cpp.llama_kv_cache_clear(self.ctx) @@ -641,6 +645,16 @@ def _should_add_bos(model: _LlamaModel) -> bool: return llama_cpp.llama_vocab_type(model.model) == llama_cpp.LLAMA_VOCAB_TYPE_SPM +# Embedding functions + + +def _normalize_embedding(embedding): + norm = float(np.linalg.norm(embedding)) + if norm == 0.0: + return embedding + return [v / norm for v in embedding] + + # Python wrappers over common/sampling structs diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 0a576d4cfb..481842b650 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -50,6 +50,7 @@ _LlamaTokenDataArray, # type: ignore _LlamaSamplingParams, # type: ignore _LlamaSamplingContext, # type: ignore + _normalize_embedding, # type: ignore ) from ._logger import set_verbose from ._utils import suppress_stdout_stderr @@ -760,7 +761,7 @@ def create_embedding( input = input if isinstance(input, list) else [input] # get numeric embeddings - embeds: List[List[float]] + embeds: Union[List[List[float]], List[List[List[float]]]] total_tokens: int embeds, total_tokens = self.embed(input, return_count=True) # type: ignore @@ -787,7 +788,7 @@ def create_embedding( def embed( self, input: Union[str, List[str]], - normalize: bool = True, + normalize: bool = False, truncate: bool = True, return_count: bool = False, ): @@ -803,6 +804,10 @@ def embed( n_embd = self.n_embd() n_batch = self.n_batch + # get pooling information + pooling_type = self.pooling_type() + logits_all = pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE + if self.context_params.embeddings == False: raise RuntimeError( "Llama model must be created with embedding=True to call this method" @@ -820,29 +825,37 @@ def embed( self._batch.reset() # decode and fetch embeddings - data: List[List[float]] = [] + data: Union[List[List[float]], List[List[List[float]]]] = [] - def decode_batch(n_seq: int): + def decode_batch(seq_sizes: List[int]): assert self._ctx.ctx is not None llama_cpp.llama_kv_cache_clear(self._ctx.ctx) self._ctx.decode(self._batch) self._batch.reset() # store embeddings - for i in range(n_seq): - ptr = llama_cpp.llama_get_embeddings_seq( - self._ctx.ctx, i - ) - if not ptr: - raise RuntimeError("Failed to get embeddings from sequence pooling type is not set") - embedding: List[float] = ptr[:n_embd] - if normalize: - norm = float(np.linalg.norm(embedding)) - embedding = [v / norm for v in embedding] - data.append(embedding) + if pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE: + pos: int = 0 + for i, size in enumerate(seq_sizes): + ptr = llama_cpp.llama_get_embeddings(self._ctx.ctx) + embedding: List[List[float]] = [ + ptr[pos + j * n_embd : pos + (j + 1) * n_embd] for j in range(size) + ] + if normalize: + embedding = [_normalize_embedding(e) for e in embedding] + data.append(embedding) + pos += size + else: + for i in range(len(seq_sizes)): + ptr = llama_cpp.llama_get_embeddings_seq(self._ctx.ctx, i) + embedding: List[float] = ptr[:n_embd] + if normalize: + embedding = _normalize_embedding(embedding) + data.append(embedding) # init state total_tokens = 0 + s_batch = [] t_batch = 0 p_batch = 0 @@ -863,17 +876,21 @@ def decode_batch(n_seq: int): # time to eval batch if t_batch + n_tokens > n_batch: - decode_batch(p_batch) + decode_batch(s_batch) + s_batch = [] t_batch = 0 p_batch = 0 # add to batch - self._batch.add_sequence(tokens, p_batch, False) + self._batch.add_sequence(tokens, p_batch, logits_all) + + # update batch stats + s_batch.append(n_tokens) t_batch += n_tokens p_batch += 1 # hanlde last batch - decode_batch(p_batch) + decode_batch(s_batch) if self.verbose: llama_cpp.llama_print_timings(self._ctx.ctx) @@ -1845,6 +1862,10 @@ def token_nl(self) -> int: """Return the newline token.""" return self._model.token_nl() + def pooling_type(self) -> str: + """Return the pooling type.""" + return self._ctx.pooling_type() + @staticmethod def logits_to_logprobs( logits: Union[npt.NDArray[np.single], List], axis: int = -1 diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 3b96adc4c7..609dd369ff 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1189,6 +1189,12 @@ def llama_rope_type(model: llama_model_p, /) -> int: ... +# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_model * model); +@ctypes_function("llama_pooling_type", [llama_model_p_ctypes], ctypes.c_int) +def llama_pooling_type(model: llama_model_p, /) -> int: + ... + + # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_vocab(model: llama_model_p, /) -> int: diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 87e000f140..4677785ae1 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -24,7 +24,7 @@ class EmbeddingUsage(TypedDict): class Embedding(TypedDict): index: int object: str - embedding: List[float] + embedding: Union[List[float], List[List[float]]] class CreateEmbeddingResponse(TypedDict): From 173ebc78782b2eaed40b3b29989502cb6ecdea44 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 25 Apr 2024 21:36:09 -0400 Subject: [PATCH 095/561] fix: Remove duplicate pooling_type definition and add misisng n_vocab definition in bindings --- llama_cpp/llama_cpp.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 609dd369ff..3b96adc4c7 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1189,12 +1189,6 @@ def llama_rope_type(model: llama_model_p, /) -> int: ... -# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_model * model); -@ctypes_function("llama_pooling_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_pooling_type(model: llama_model_p, /) -> int: - ... - - # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_vocab(model: llama_model_p, /) -> int: From 65edc906712eb205afaedc7d69f27e33be162228 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 26 Apr 2024 10:11:31 -0400 Subject: [PATCH 096/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25b8835528..806876ce2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.65] + +- feat: Update llama.cpp to ggerganov/llama.cpp@46e12c4692a37bdd31a0432fc5153d7d22bc7f72 +- feat: Allow for possibly non-pooled embeddings by @iamlemec in #1380 + ## [0.2.64] - feat: Update llama.cpp to ggerganov/llama.cpp@4e96a812b3ce7322a29a3008db2ed73d9087b176 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index f736458741..6db6333ba5 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.64" \ No newline at end of file +__version__ = "0.2.65" \ No newline at end of file From 9e7f738220a28de2c0b3fdac8ceceb5a2c4d02ec Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 28 Apr 2024 02:47:07 +0200 Subject: [PATCH 097/561] ci: Update dependabot.yml (#1391) Add github-actions update --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 91abb11fdf..c58c9ae570 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,7 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From c58b56123d8659195609098e14eacd13b7374742 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 28 Apr 2024 02:47:49 +0200 Subject: [PATCH 098/561] ci: Update action versions in build-wheels-metal.yaml (#1390) * Bump actions/setup-python@v4 to v5 * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml --- .github/workflows/build-wheels-metal.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 2cca477860..fc798c84e7 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -41,7 +41,7 @@ jobs: with: submodules: "recursive" - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.pyver }} @@ -78,7 +78,7 @@ jobs: VERBOSE=1 python -m build --wheel fi - - uses: softprops/action-gh-release@v1 + - uses: softprops/action-gh-release@v2 with: files: dist/* # set release name to -metal From e6bbfb863c38e7575e4fe87a823ac6ce2e15c27c Mon Sep 17 00:00:00 2001 From: iyubondyrev <76585902+iyubondyrev@users.noreply.github.com> Date: Sun, 28 Apr 2024 02:48:47 +0200 Subject: [PATCH 099/561] examples: fix quantize example (#1387) @iyubondyrev thank you! --- examples/low_level_api/quantize.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/low_level_api/quantize.py b/examples/low_level_api/quantize.py index 8bd03f88a1..cdb038a712 100644 --- a/examples/low_level_api/quantize.py +++ b/examples/low_level_api/quantize.py @@ -4,14 +4,16 @@ def main(args): + fname_inp = args.fname_inp.encode("utf-8") + fname_out = args.fname_out.encode("utf-8") if not os.path.exists(fname_inp): raise RuntimeError(f"Input file does not exist ({fname_inp})") if os.path.exists(fname_out): raise RuntimeError(f"Output file already exists ({fname_out})") - fname_inp = args.fname_inp.encode("utf-8") - fname_out = args.fname_out.encode("utf-8") - itype = args.itype - return_code = llama_cpp.llama_model_quantize(fname_inp, fname_out, itype) + ftype = args.type + args = llama_cpp.llama_model_quantize_default_params() + args.ftype = ftype + return_code = llama_cpp.llama_model_quantize(fname_inp, fname_out, args) if return_code != 0: raise RuntimeError("Failed to quantize model") @@ -20,6 +22,7 @@ def main(args): parser = argparse.ArgumentParser() parser.add_argument("fname_inp", type=str, help="Path to input model") parser.add_argument("fname_out", type=str, help="Path to output model") - parser.add_argument("type", type=int, help="Type of quantization (2: q4_0, 3: q4_1)") + parser.add_argument("type", type=int, help="Type of quantization (2: q4_0, 3: q4_1), see llama_cpp.py for enum") args = parser.parse_args() main(args) + From f178636e1b2a30b12498aa656011779796b9ba11 Mon Sep 17 00:00:00 2001 From: Jeffrey Fong Date: Sun, 28 Apr 2024 08:49:52 +0800 Subject: [PATCH 100/561] fix: Functionary bug fixes (#1385) * fix completion tokens tracking, prompt forming * fix 'function_call' and 'tool_calls' depending on 'functions' and 'tools', incompatibility with python 3.8 * Updated README * fix for openai server compatibility --------- Co-authored-by: Andrei --- README.md | 2 + llama_cpp/llama_chat_format.py | 96 +++++++++++++++++++--------------- 2 files changed, 57 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index b5e7d20350..a33524cabf 100644 --- a/README.md +++ b/README.md @@ -484,6 +484,8 @@ Due to discrepancies between llama.cpp and HuggingFace's tokenizers, it is requi tokenizer=LlamaHFTokenizer.from_pretrained("meetkai/functionary-small-v2.2-GGUF") ) ``` + +**NOTE**: There is no need to provide the default system messages used in Functionary as they are added automatically in the Functionary chat handler. Thus, the messages should contain just the chat messages and/or system messages that provide additional context for the model (e.g.: datetime, etc.).
### Multi-modal Models diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 17b570a0fd..71aac80614 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1828,27 +1828,35 @@ def prepare_messages_for_inference( version: Literal["v1", "v2"], functions: Optional[List[llama_types.ChatCompletionFunctions]] = None, tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Union[Dict, str] = "auto", ): all_messages: List[llama_types.ChatCompletionRequestMessage] = [] - if functions is not None: + if tool_choice == "none": all_messages.append( llama_types.ChatCompletionRequestSystemMessage( - role="system", content=generate_schema_from_functions(functions) + role="system", content=generate_schema_from_functions([]) ) ) - elif tools is not None: - all_messages.append( - llama_types.ChatCompletionRequestSystemMessage( - role="system", - content=generate_schema_from_functions( - [ - tool["function"] - for tool in tools - if tool["type"] == "function" - ] - ), + else: + if functions is not None: + all_messages.append( + llama_types.ChatCompletionRequestSystemMessage( + role="system", content=generate_schema_from_functions(functions) + ) + ) + elif tools is not None and tool_choice != "none": + all_messages.append( + llama_types.ChatCompletionRequestSystemMessage( + role="system", + content=generate_schema_from_functions( + [ + tool["function"] + for tool in tools + if tool["type"] == "function" + ] + ), + ) ) - ) all_messages.append( llama_types.ChatCompletionRequestSystemMessage( @@ -1888,7 +1896,7 @@ def prepare_messages_for_inference( function_call = "auto" prompt = prepare_messages_for_inference( - messages, tokenizer, version, functions, tools + messages, tokenizer, version, functions, tools, function_call ) # If no tools/functions are provided @@ -1985,17 +1993,12 @@ def create_completion(stop): content = "" function_calls, function_bodies = [], [] + completion_tokens = 0 if version == "v1": # If no or "auto" tool_choice/function_call if isinstance(function_call, str) and function_call == "auto": stops = ["\n", END_ASSISTANT_TOKEN] - # If tool_choice/function_call is "none" - elif isinstance(function_call, str) and function_call == "none": - prompt = prepare_messages_for_inference( - messages, tokenizer, version, [], [] - ) - stops = END_ASSISTANT_TOKEN # If tool_choice/function_call is provided elif isinstance(function_call, dict): prompt += f"{START_FUNCTION_CALL_TOKEN}{function_call['name']}:\n" @@ -2009,12 +2012,15 @@ def create_completion(stop): completion = create_completion(stop=stops) completion_text = completion["choices"][0]["text"] + completion_tokens += completion["usage"]["completion_tokens"] + # If the generation does not involve a function call if ( START_FUNCTION_CALL_TOKEN not in prompt and START_FUNCTION_CALL_TOKEN not in completion_text ): + completion["usage"]["completion_tokens"] = completion_tokens return _convert_completion_to_chat(completion, stream=stream) # type: ignore # If the generation involves a function call in completion, generate the parameters elif ( @@ -2032,23 +2038,14 @@ def create_completion(stop): ) grammar = get_grammar(function_calls[-1]) completion = create_completion(stop=END_FUNCTION_CALL_TOKEN) + completion_tokens += completion["usage"]["completion_tokens"] function_bodies.append(completion["choices"][0]["text"].strip()) # If the prompt involves a function call, just append generated parameters to function_bodies else: function_bodies.append(completion_text.strip()) else: - # If tool_choice/function_call is "none" - if isinstance(function_call, str) and function_call == "none": - prompt = ( - prepare_messages_for_inference(messages, tokenizer, version, [], []) - + "all\n<|content|>" - ) - stops = [STOP_TOKEN, FROM_TOKEN] - completion = create_completion(stop=stops) - completion["choices"][0]["text"] = completion["choices"][0]["text"].strip() - return _convert_completion_to_chat(completion, stream=stream) # type: ignore # If tool_choice/function_call is provided - elif isinstance(function_call, dict): + if isinstance(function_call, dict): prompt += f"{function_call['name']}\n{CONTENT_TOKEN}" function_call = function_call["name"] function_calls.append(function_call) @@ -2056,6 +2053,7 @@ def create_completion(stop): stops = [STOP_TOKEN, FROM_TOKEN] completion = create_completion(stop=stops) completion_text = completion["choices"][0]["text"] + completion_tokens += completion["usage"]["completion_tokens"] function_bodies.append(completion_text.strip()) # If "auto" or no tool_choice/function_call elif isinstance(function_call, str) and function_call == "auto": @@ -2065,6 +2063,7 @@ def create_completion(stop): stops = CONTENT_TOKEN completion = create_completion(stop=stops) completion_text = completion["choices"][0]["text"] + completion_tokens += completion["usage"]["completion_tokens"] function_name = completion_text.strip() if function_name == "all": prompt += "all\n<|content|>" @@ -2077,12 +2076,23 @@ def create_completion(stop): stops = [RECIPIENT_TOKEN, STOP_TOKEN] completion = create_completion(stop=stops) completion_text = completion["choices"][0]["text"] + completion_tokens += completion["usage"]["completion_tokens"] if function_name == "all": - content += completion_text.removesuffix("\n<|from|>assistant\n").removesuffix("\n<|from|> assistant\n") + if completion_text.endswith("\n<|from|>assistant\n"): + content += completion_text[:-len("\n<|from|>assistant\n")] + if completion_text.endswith("\n<|from|> assistant\n"): + content += completion_text[-len("\n<|from|> assistant\n")] + else: + content += completion_text content = content.lstrip() # Check whether the model wants to generate another turn if "<|from|> assistant" in completion_text or "<|from|>assistant" in completion_text: - cleaned_completion_text = completion_text.removesuffix("\n<|from|>assistant\n").removesuffix("\n<|from|> assistant\n").strip() + if completion_text.endswith("\n<|from|>assistant\n"): + cleaned_completion_text = completion_text[:-len("\n<|from|>assistant\n")].strip() + elif completion_text.endswith("\n<|from|> assistant\n"): + cleaned_completion_text = completion_text[-len("\n<|from|> assistant\n")].strip() + else: + cleaned_completion_text = completion_text.strip() prompt += f"{cleaned_completion_text}\n<|from|>assistant\n<|recipient|>" else: break @@ -2092,6 +2102,7 @@ def create_completion(stop): prompt += completion_text.strip() grammar = None completion = create_completion(stop=stops) + completion_tokens += completion["usage"]["completion_tokens"] if "<|from|> assistant" in completion["choices"][0]["text"] or "<|from|>assistant" in completion["choices"][0]["text"]: prompt += "\n<|from|>assistant\n<|recipient|>" else: @@ -2120,12 +2131,16 @@ def create_completion(stop): ) # TODO: support stream mode - function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = { - "function_call": { - "name": tool_calls[0]["function"]["name"], - "arguments": tool_calls[0]["function"]["arguments"], - } - } if len(tool_calls) == 1 else {} + function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = {} + if len(tool_calls) > 0: + if tools is not None: + function_call_dict["tool_calls"] = tool_calls + else: + function_call_dict["function_call"] = { + "name": tool_calls[0]["function"]["name"], + "arguments": tool_calls[0]["function"]["arguments"], + } + completion["usage"]["completion_tokens"] = completion_tokens return llama_types.CreateChatCompletionResponse( id="chat" + completion["id"], object="chat.completion", @@ -2138,7 +2153,6 @@ def create_completion(stop): "message": { "role": "assistant", "content": None if content == "" else content, - "tool_calls": tool_calls, **function_call_dict, }, "finish_reason": "tool_calls" if len(tool_calls) > 0 else "stop", From 17bdfc818f50988ddb6dfc5a30cbec68571e4c56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:50:28 -0400 Subject: [PATCH 101/561] chore(deps): bump conda-incubator/setup-miniconda from 2.2.0 to 3.0.4 (#1397) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2.2.0 to 3.0.4. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v2.2.0...v3.0.4) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index a222dceb41..9647036b81 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -52,7 +52,7 @@ jobs: python-version: ${{ matrix.pyver }} - name: Setup Mamba - uses: conda-incubator/setup-miniconda@v2.2.0 + uses: conda-incubator/setup-miniconda@v3.0.4 with: activate-environment: "build" python-version: ${{ matrix.pyver }} From 27038db3d6cbab301dc96e21d7f24fc53c5ccbf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:50:39 -0400 Subject: [PATCH 102/561] chore(deps): bump actions/cache from 3.3.2 to 4.0.2 (#1398) Bumps [actions/cache](https://github.com/actions/cache) from 3.3.2 to 4.0.2. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3.3.2...v4.0.2) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 9647036b81..274b946697 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -65,7 +65,7 @@ jobs: - name: VS Integration Cache id: vs-integration-cache if: runner.os == 'Windows' - uses: actions/cache@v3.3.2 + uses: actions/cache@v4.0.2 with: path: ./MSBuildExtensions key: cuda-${{ matrix.cuda }}-vs-integration From 79318ba1d181f5765d3abfab00d27ec5c42a3e8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:50:50 -0400 Subject: [PATCH 103/561] chore(deps): bump docker/login-action from 2 to 3 (#1399) Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 750b91e1f0..726f872337 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -23,7 +23,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} From 7074c4d256e06c31a6386bb099bad1e891284334 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:51:02 -0400 Subject: [PATCH 104/561] chore(deps): bump docker/build-push-action from 4 to 5 (#1400) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 5. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 726f872337..68cc06f156 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -31,7 +31,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . file: "docker/simple/Dockerfile" From c07db99e5b34b9d791ac9d14cdde46928f4694ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:51:13 -0400 Subject: [PATCH 105/561] chore(deps): bump pypa/cibuildwheel from 2.16.5 to 2.17.0 (#1401) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.16.5 to 2.17.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.16.5...v2.17.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 07742f1dce..2137da350c 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -29,7 +29,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -55,7 +55,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.17.0 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" From c9b85bf09859161d3ab9de49ed448297488e0c89 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 27 Apr 2024 23:41:54 -0400 Subject: [PATCH 106/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 152 ++++++++++++++++++----------------------- vendor/llama.cpp | 2 +- 2 files changed, 69 insertions(+), 85 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 3b96adc4c7..d00dfcb955 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -552,19 +552,25 @@ class llama_batch(ctypes.Structure): # LLAMA_KV_OVERRIDE_TYPE_INT, # LLAMA_KV_OVERRIDE_TYPE_FLOAT, # LLAMA_KV_OVERRIDE_TYPE_BOOL, +# LLAMA_KV_OVERRIDE_TYPE_STR, # }; LLAMA_KV_OVERRIDE_TYPE_INT = 0 LLAMA_KV_OVERRIDE_TYPE_FLOAT = 1 LLAMA_KV_OVERRIDE_TYPE_BOOL = 2 +LLAMA_KV_OVERRIDE_TYPE_STR = 3 # struct llama_model_kv_override { -# char key[128]; # enum llama_model_kv_override_type tag; + +# char key[128]; + + # union { -# int64_t int_value; -# double float_value; -# bool bool_value; +# int64_t val_i64; +# double val_f64; +# bool val_bool; +# char val_str[128]; # }; # }; class llama_model_kv_override_value(ctypes.Union): @@ -572,16 +578,28 @@ class llama_model_kv_override_value(ctypes.Union): ("int_value", ctypes.c_int64), ("float_value", ctypes.c_double), ("bool_value", ctypes.c_bool), + ("str_value", ctypes.c_char * 128), ] + if TYPE_CHECKING: + int_value: int + float_value: float + bool_value: bool + str_value: bytes + class llama_model_kv_override(ctypes.Structure): _fields_ = [ - ("key", ctypes.c_char * 128), ("tag", ctypes.c_int), + ("key", ctypes.c_char * 128), ("value", llama_model_kv_override_value), ] + if TYPE_CHECKING: + tag: int + key: bytes + value: Union[int, float, bool, bytes] + # struct llama_model_params { # int32_t n_gpu_layers; // number of layers to store in VRAM @@ -609,9 +627,10 @@ class llama_model_kv_override(ctypes.Structure): # // Keep the booleans together to avoid misalignment during copy-by-value. -# bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_mlock; // force system to keep model in RAM +# bool vocab_only; // only load the vocabulary, no weights +# bool use_mmap; // use mmap if possible +# bool use_mlock; // force system to keep model in RAM +# bool check_tensors; // validate model tensor data # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -626,7 +645,8 @@ class llama_model_params(ctypes.Structure): kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights use_mmap (bool): use mmap if possible - use_mlock (bool): force system to keep model in RAM""" + use_mlock (bool): force system to keep model in RAM + check_tensors (bool): validate model tensor data""" if TYPE_CHECKING: n_gpu_layers: int @@ -639,6 +659,7 @@ class llama_model_params(ctypes.Structure): vocab_only: bool use_mmap: bool use_mlock: bool + check_tensors: bool _fields_ = [ ("n_gpu_layers", ctypes.c_int32), @@ -651,6 +672,7 @@ class llama_model_params(ctypes.Structure): ("vocab_only", ctypes.c_bool), ("use_mmap", ctypes.c_bool), ("use_mlock", ctypes.c_bool), + ("check_tensors", ctypes.c_bool), ] @@ -1041,8 +1063,7 @@ def llama_backend_init(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): - ... +def llama_numa_init(numa: int, /): ... # // Call once at the end of the program - currently only used for MPI @@ -1067,8 +1088,7 @@ def llama_backend_free(): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: - ... +) -> Optional[llama_model_p]: ... # LLAMA_API void llama_free_model(struct llama_model * model); @@ -1077,8 +1097,7 @@ def llama_load_model_from_file( [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): - ... +def llama_free_model(model: llama_model_p, /): ... # LLAMA_API struct llama_context * llama_new_context_with_model( @@ -1091,8 +1110,7 @@ def llama_free_model(model: llama_model_p, /): ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: - ... +) -> Optional[llama_context_p]: ... # // Frees all allocated memory @@ -1113,104 +1131,87 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: - ... +def llama_time_us() -> int: ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: - ... +def llama_max_devices() -> int: ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: - ... +def llama_supports_mmap() -> bool: ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: - ... +def llama_supports_mlock() -> bool: ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: - ... +def llama_supports_gpu_offload() -> bool: ... # LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: - ... +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: - ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: - ... +def llama_n_batch(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); @ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ubatch(ctx: llama_context_p, /) -> int: - ... +def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); @ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_seq_max(ctx: llama_context_p, /) -> int: - ... +def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... # LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) -def llama_pooling_type(ctx: llama_context_p, /) -> int: - ... +def llama_pooling_type(ctx: llama_context_p, /) -> int: ... # LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: - ... +def llama_vocab_type(model: llama_model_p, /) -> int: ... # LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: - ... +def llama_rope_type(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: - ... +def llama_n_vocab(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: - ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_embd (const struct llama_model * model); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: - ... +def llama_n_embd(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_layer (const struct llama_model * model); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: - ... +def llama_n_layer(model: llama_model_p, /) -> int: ... # // Get the model's RoPE frequency scaling factor @@ -1912,8 +1913,7 @@ def llama_state_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: - ... +) -> bool: ... # LLAMA_API DEPRECATED(bool llama_load_session_file( @@ -1941,8 +1941,7 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: - ... +) -> int: ... # LLAMA_API bool llama_state_save_file( @@ -1966,8 +1965,7 @@ def llama_state_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: - ... +) -> bool: ... # LLAMA_API DEPRECATED(bool llama_save_session_file( @@ -1992,8 +1990,7 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: - ... +) -> int: ... # // Get the exact size needed to copy the KV cache of a single sequence @@ -2071,8 +2068,7 @@ def llama_state_seq_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: - ... +) -> int: ... # LLAMA_API size_t llama_state_seq_load_file( @@ -2102,8 +2098,7 @@ def llama_state_seq_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: - ... +) -> int: ... # // @@ -2366,8 +2361,7 @@ def llama_get_embeddings_seq( ) def llama_token_get_text( model: llama_model_p, token: Union[llama_token, int], / -) -> bytes: - ... +) -> bytes: ... # LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); @@ -2376,8 +2370,7 @@ def llama_token_get_text( ) def llama_token_get_score( model: llama_model_p, token: Union[llama_token, int], / -) -> float: - ... +) -> float: ... # LLAMA_API enum llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token); @@ -2386,8 +2379,7 @@ def llama_token_get_score( ) def llama_token_get_type( model: llama_model_p, token: Union[llama_token, int], / -) -> int: - ... +) -> int: ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) @@ -2395,9 +2387,7 @@ def llama_token_get_type( @ctypes_function( "llama_token_is_eog", [llama_model_p_ctypes, llama_token], ctypes.c_bool ) -def llama_token_is_eog( - model: llama_model_p, token: Union[llama_token, int], / -) -> bool: +def llama_token_is_eog(model: llama_model_p, token: Union[llama_token, int], /) -> bool: """Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)""" ... @@ -2466,20 +2456,17 @@ def llama_token_prefix(model: llama_model_p) -> int: # LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: - ... +def llama_token_middle(model: llama_model_p, /) -> int: ... # LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: - ... +def llama_token_suffix(model: llama_model_p, /) -> int: ... # LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle @ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: - ... +def llama_token_eot(model: llama_model_p, /) -> int: ... # // @@ -2620,8 +2607,7 @@ def llama_chat_apply_template( chat: CtypesArray[llama_chat_message], n_msg: int, /, -) -> int: - ... +) -> int: ... # // @@ -3234,8 +3220,7 @@ def llama_beam_search( n_past: Union[ctypes.c_int, int], n_predict: Union[ctypes.c_int, int], /, -): - ... +): ... # /// @details Build a split GGUF final path for this chunk. @@ -3354,5 +3339,4 @@ def llama_log_set( [ctypes.c_void_p, llama_context_p_ctypes], None, ) -def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): - ... +def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 46e12c4692..4dba7e8114 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 46e12c4692a37bdd31a0432fc5153d7d22bc7f72 +Subproject commit 4dba7e8114d84241c842b986e008af8b88d1a019 From a411612b385cef100d76145da1fbd02a7b7cc894 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 27 Apr 2024 23:42:19 -0400 Subject: [PATCH 107/561] feat: Add support for str type kv_overrides --- llama_cpp/llama.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 481842b650..96aac6668b 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -73,7 +73,7 @@ def __init__( vocab_only: bool = False, use_mmap: bool = True, use_mlock: bool = False, - kv_overrides: Optional[Dict[str, Union[bool, int, float]]] = None, + kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, # Context Params seed: int = llama_cpp.LLAMA_DEFAULT_SEED, n_ctx: int = 512, @@ -254,6 +254,13 @@ def __init__( elif isinstance(v, float): self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT self._kv_overrides_array[i].value.float_value = v + elif isinstance(v, str): # type: ignore + v_bytes = v.encode("utf-8") + if len(v_bytes) > 128: # TODO: Make this a constant + raise ValueError(f"Value for {k} is too long: {v}") + v_bytes = v_bytes.ljust(128, b"\0") + self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR + self._kv_overrides_array[i].value.str_value[:128] = v_bytes else: raise ValueError(f"Unknown value type for {k}: {v}") From 2355ce2227476ea69f1d6142dedd2f4b7f27a86b Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 28 Apr 2024 05:44:47 +0200 Subject: [PATCH 108/561] ci: Add support for pre-built cuda 12.4.1 wheels (#1388) * Add support for cuda 12.4.1 * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml Revert --- .github/workflows/build-wheels-cuda.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 274b946697..ae9e8632c0 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -22,7 +22,7 @@ jobs: $matrix = @{ 'os' = @('ubuntu-20.04', 'windows-latest') 'pyver' = @("3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") 'releasetag' = @("basic") } @@ -47,7 +47,7 @@ jobs: with: submodules: "recursive" - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.pyver }} @@ -74,7 +74,7 @@ jobs: if: runner.os == 'Windows' && steps.vs-integration-cache.outputs.cache-hit != 'true' run: | if ($env:CUDAVER -eq '12.1.1') {$x = '12.1.0'} else {$x = $env:CUDAVER} - $links = (Invoke-RestMethod 'https://github.com/Jimver/cuda-toolkit/raw/dc0ca7bb29c5a92f7a963d3d5c93f8d59765136a/src/links/windows-links.ts').Trim().split().where({$_ -ne ''}) + $links = (Invoke-RestMethod 'https://raw.githubusercontent.com/Jimver/cuda-toolkit/master/src/links/windows-links.ts').Trim().split().where({$_ -ne ''}) for ($i=$q=0;$i -lt $links.count -and $q -lt 2;$i++) {if ($links[$i] -eq "'$x',") {$q++}} Invoke-RestMethod $links[$i].Trim("'") -OutFile 'cudainstaller.zip' & 'C:\Program Files\7-Zip\7z.exe' e cudainstaller.zip -oMSBuildExtensions -r *\MSBuildExtensions\* > $null @@ -122,7 +122,7 @@ jobs: # write the build tag to the output Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV - - uses: softprops/action-gh-release@v1 + - uses: softprops/action-gh-release@v2 with: files: dist/* # Set tag_name to -cu From 0c3bc4b92813381f3ffddcc9b579e221ee48667b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 29 Apr 2024 12:37:22 -0400 Subject: [PATCH 109/561] fix(ci): Update generate wheel index script to include cu12.3 and cu12.4 Closes #1406 --- .github/workflows/generate-index-from-release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 9042d6cfea..858e50711b 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -37,6 +37,8 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$' ./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$' ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' + ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' + ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 From 03c654a3d9889bcf8a0402c3b7478ad85fbaf111 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Tue, 30 Apr 2024 04:52:23 +0200 Subject: [PATCH 110/561] ci(fix): Workflow actions updates and fix arm64 wheels not included in release (#1392) * Update test.yaml Bump actions/checkout@v3 to v4 Bump action/setup-python@v4 to v5 * Update test-pypi.yaml Bum actions/setup-python@v4 to v5 * Update build-and-release.yaml Bump softprops/action-gh-release@v1 to v2 Bump actions/checkout@v3 to v4 Bump actions/setup-python@v3 to v5 * Update publish.yaml Bump actions/checkout@v3 to v4 Bump actions/sertup-python@v4 to v5 * Update publish-to-test.yaml Bump actions/checkout@v3 to v4 Bump actions/setup-python @v4 to v5 * Update test-pypi.yaml Add Python 3.12 * Update build-and-release.yaml * Update build-docker.yaml Bump docker/setup-qemu-action@v2 to v3 Bump docker/setup-buildx-action@v2 to v3 * Update build-and-release.yaml * Update build-and-release.yaml --- .github/workflows/build-and-release.yaml | 19 +++++++++---------- .github/workflows/build-docker.yaml | 6 +++--- .github/workflows/publish-to-test.yaml | 6 +++--- .github/workflows/publish.yaml | 4 ++-- .github/workflows/test-pypi.yaml | 14 +++++++------- .github/workflows/test.yaml | 18 +++++++++--------- 6 files changed, 33 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 2137da350c..76bf708fea 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -14,12 +14,12 @@ jobs: os: [ubuntu-20.04, windows-2019, macos-11] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" # Used to host cibuildwheel - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 with: python-version: "3.8" @@ -62,23 +62,22 @@ jobs: CIBW_ARCHS: "aarch64" CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" with: - output-dir: wheelhouse/ + output-dir: wheelhouse - name: Upload wheels as artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: - name: wheels-${{ matrix.version }} - path: wheelhouse/*.whl + path: ./wheelhouse/*.whl build_sdist: name: Build source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 with: python-version: "3.8" - name: Install dependencies @@ -102,8 +101,8 @@ jobs: with: name: artifact path: dist - - uses: softprops/action-gh-release@v1 + - uses: softprops/action-gh-release@v2 with: files: dist/* env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 68cc06f156..4ebe3bb6d7 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -12,15 +12,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: "recursive" - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v3 diff --git a/.github/workflows/publish-to-test.yaml b/.github/workflows/publish-to-test.yaml index 47e7c40b15..2bf0ea9ba4 100644 --- a/.github/workflows/publish-to-test.yaml +++ b/.github/workflows/publish-to-test.yaml @@ -16,11 +16,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.8" - name: Append Dev Version to __version__ @@ -40,4 +40,4 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository-url: https://test.pypi.org/legacy/ \ No newline at end of file + repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 1afdd667d7..bc4ec9063b 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -10,11 +10,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.8" - name: Install dependencies diff --git a/.github/workflows/test-pypi.yaml b/.github/workflows/test-pypi.yaml index cc6a3a7252..aa8e8fa0b0 100644 --- a/.github/workflows/test-pypi.yaml +++ b/.github/workflows/test-pypi.yaml @@ -8,11 +8,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -28,11 +28,11 @@ jobs: runs-on: windows-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -48,11 +48,11 @@ jobs: runs-on: macos-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -61,4 +61,4 @@ jobs: python3 -m pip install --verbose llama-cpp-python[all] - name: Test with pytest run: | - python3 -c "import llama_cpp" \ No newline at end of file + python3 -c "import llama_cpp" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6b0e7e8c67..292343c2bf 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -21,7 +21,7 @@ jobs: with: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -40,11 +40,11 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -63,11 +63,11 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -83,11 +83,11 @@ jobs: # runs-on: ubuntu-latest # steps: - # - uses: actions/checkout@v3 + # - uses: actions/checkout@v4 # with: # submodules: "recursive" # - name: Set up Python 3.8 - # uses: actions/setup-python@v4 + # uses: actions/setup-python@v5 # with: # python-version: "3.8" # - name: Set up OpenCL & CLBlast @@ -110,11 +110,11 @@ jobs: runs-on: macos-13 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" - name: Set up Python 3.8 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.8" - name: Install dependencies From 32c000f3ec5d0417612256a92297362c4c21c4ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:52:58 -0400 Subject: [PATCH 111/561] chore(deps): bump softprops/action-gh-release from 1 to 2 (#1408) Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 1 to 2. - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/v1...v2) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From be43018e09ef795fc8b214a4d13fd702dc670a74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:53:21 -0400 Subject: [PATCH 112/561] chore(deps): bump actions/configure-pages from 4 to 5 (#1411) Bumps [actions/configure-pages](https://github.com/actions/configure-pages) from 4 to 5. - [Release notes](https://github.com/actions/configure-pages/releases) - [Commits](https://github.com/actions/configure-pages/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/configure-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generate-index-from-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 858e50711b..500c4613ca 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -31,7 +31,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Setup Pages - uses: actions/configure-pages@v4 + uses: actions/configure-pages@v5 - name: Build run: | ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$' From df2b5b5d4485c485b6043d193f10092452d760d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:53:42 -0400 Subject: [PATCH 113/561] chore(deps): bump actions/upload-artifact from 3 to 4 (#1412) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3 to 4. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 76bf708fea..78bcb87d4e 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -37,7 +37,7 @@ jobs: package-dir: . output-dir: wheelhouse - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl @@ -65,7 +65,7 @@ jobs: output-dir: wheelhouse - name: Upload wheels as artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl @@ -87,7 +87,7 @@ jobs: - name: Build source distribution run: | python -m build --sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: ./dist/*.tar.gz From 97fb860eba42e1018abc9b603f8e4602e84dd153 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 29 Apr 2024 23:34:55 -0400 Subject: [PATCH 114/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 21 +++++++++++++++++++++ tests/test_llama.py | 2 +- vendor/llama.cpp | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d00dfcb955..9c8f77889e 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -284,6 +284,27 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa """BERT tokenizer based on WordPiece""" +# // pre-tokenization types +# enum llama_vocab_pre_type { +# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, +# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, +# LLAMA_VOCAB_PRE_TYPE_FALCON = 4, +# LLAMA_VOCAB_PRE_TYPE_MPT = 5, +# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, +# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, +# }; +LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 +LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 +LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2 +LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3 +LLAMA_VOCAB_PRE_TYPE_FALCON = 4 +LLAMA_VOCAB_PRE_TYPE_MPT = 5 +LLAMA_VOCAB_PRE_TYPE_STARCODER = 6 +LLAMA_VOCAB_PRE_TYPE_GPT2 = 7 + + # // note: these values should be synchronized with ggml_rope # // TODO: maybe move this enum to ggml.h (ggml_rope_type) # enum llama_rope_type { diff --git a/tests/test_llama.py b/tests/test_llama.py index fa2f6dfc16..469ef91cab 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -6,7 +6,7 @@ import llama_cpp -MODEL = "./vendor/llama.cpp/models/ggml-vocab-llama.gguf" +MODEL = "./vendor/llama.cpp/models/ggml-vocab-llama-spm.gguf" def test_llama_cpp_tokenization(): diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4dba7e8114..8843a98c2b 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4dba7e8114d84241c842b986e008af8b88d1a019 +Subproject commit 8843a98c2ba97a25e93319a104f9ddfaf83ce4c4 From fe2da0953895000e20655d2c991308f702508494 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 30 Apr 2024 01:35:38 -0400 Subject: [PATCH 115/561] feat: Generic Chat Formats, Tool Calling, and Huggingface Pull Support for Multimodal Models (Obsidian, LLaVA1.6, Moondream) (#1147) * Test dummy image tags in chat templates * Format and improve types for llava_cpp.py * Add from_pretrained support to llava chat format. * Refactor llava chat format to use a jinja2 * Revert chat format test * Add moondream support (wip) * Update moondream chat format * Update moondream chat format * Update moondream prompt * Add function calling support * Cache last image embed * Add Llava1.6 support * Add nanollava support * Add obisidian support * Remove unnecessary import * Re-order multimodal chat formats * Logits all no longer required for multi-modal models * Update README.md * Update docs * Update README * Fix typo * Update README * Fix typo --- README.md | 41 ++- docs/server.md | 2 + llama_cpp/llama_chat_format.py | 631 +++++++++++++++++++++++++++------ llama_cpp/llava_cpp.py | 111 ++++-- llama_cpp/server/model.py | 71 +++- 5 files changed, 711 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index a33524cabf..d34caf409a 100644 --- a/README.md +++ b/README.md @@ -490,14 +490,15 @@ Due to discrepancies between llama.cpp and HuggingFace's tokenizers, it is requi ### Multi-modal Models -`llama-cpp-python` supports the llava1.5 family of multi-modal models which allow the language model to -read information from both text and images. +`llama-cpp-python` supports such as llava1.5 which allow the language model to read information from both text and images. You'll first need to download one of the available multi-modal models in GGUF format: - [llava-v1.5-7b](https://huggingface.co/mys/ggml_llava-v1.5-7b) - [llava-v1.5-13b](https://huggingface.co/mys/ggml_llava-v1.5-13b) - [bakllava-1-7b](https://huggingface.co/mys/ggml_bakllava-1) +- [llava-v1.6-34b](https://huggingface.co/cjpais/llava-v1.6-34B-gguf) +- [moondream2](https://huggingface.co/vikhyatk/moondream2) Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. @@ -509,7 +510,6 @@ Then you'll need to use a custom chat handler to load the clip model and process model_path="./path/to/llava/llama-model.gguf", chat_handler=chat_handler, n_ctx=2048, # n_ctx should be increased to accomodate the image embedding - logits_all=True,# needed to make llava work ) >>> llm.create_chat_completion( messages = [ @@ -517,14 +517,45 @@ Then you'll need to use a custom chat handler to load the clip model and process { "role": "user", "content": [ - {"type": "image_url", "image_url": {"url": "https://.../image.png"}}, - {"type" : "text", "text": "Describe this image in detail please."} + {"type" : "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" } } ] } ] ) ``` +You can also pull the model from the Hugging Face Hub using the `from_pretrained` method. + +```python +>>> from llama_cpp import Llama +>>> from llama_cpp.llama_chat_format import MoondreamChatHandler +>>> chat_handler = MoondreamChatHandler.from_pretrained( + repo_id="vikhyatk/moondream2", + filename="*mmproj*", +) +>>> llm = Llama.from_pretrained( + repo_id="vikhyatk/moondream2" + filename="*text-model*", + chat_handler=chat_handler, + n_ctx=2048, # n_ctx should be increased to accomodate the image embedding +) +>>> llm.create_chat_completion( + messages = [ + { + "role": "user", + "content": [ + {"type" : "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" } } + + ] + } + ] +) +``` + +**Note**: Multi-modal models also support tool calling and JSON mode. +
Loading a Local Image diff --git a/docs/server.md b/docs/server.md index c66c9cce00..cd6f86c513 100644 --- a/docs/server.md +++ b/docs/server.md @@ -98,6 +98,8 @@ You'll first need to download one of the available multi-modal models in GGUF fo - [llava-v1.5-7b](https://huggingface.co/mys/ggml_llava-v1.5-7b) - [llava-v1.5-13b](https://huggingface.co/mys/ggml_llava-v1.5-13b) - [bakllava-1-7b](https://huggingface.co/mys/ggml_bakllava-1) +- [llava-v1.6-34b](https://huggingface.co/cjpais/llava-v1.6-34B-gguf) +- [moondream2](https://huggingface.co/vikhyatk/moondream2) Then when you run the server you'll need to also specify the path to the clip model used for image embedding and the `llava-1-5` chat_format diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 71aac80614..cbd8d203ce 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -6,6 +6,8 @@ import dataclasses import random import string + +from contextlib import ExitStack from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Union, Protocol, cast import jinja2 @@ -2163,42 +2165,80 @@ def create_completion(stop): class Llava15ChatHandler: - _clip_free = None + DEFAULT_SYSTEM_MESSAGE = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." + + CHAT_FORMAT = ( + "{% for message in messages %}" + "{% if message.role == 'system' %}" + "{{ message.content }}" + "{% endif %}" + "{% if message.role == 'user' %}" + "{% if message.content is string %}" + "\nUSER: {{ message.content }}" + "{% elif message.content is iterable %}" + "\nUSER: " + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% if content.type == 'image_url' and content.image_url is string %}" + "{{ content.image_url }}" + "{% endif %}" + "{% if content.type == 'image_url' and content.image_url is mapping %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "{% endif %}" + "{% if message.role == 'assistant' and message.content is not none %}" + "\nASSISTANT: {{ message.content }}" + "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "\nASSISTANT: " + "{% endif %}" + ) def __init__(self, clip_model_path: str, verbose: bool = False): import llama_cpp.llava_cpp as llava_cpp - self._llava_cpp = llava_cpp self.clip_model_path = clip_model_path self.verbose = verbose - self._clip_free = self._llava_cpp._libllava.clip_free # type: ignore + + self._llava_cpp = llava_cpp # TODO: Fix + self._exit_stack = ExitStack() + self._last_image_embed: Optional[llava_cpp.CtypesPointer[llava_cpp.llava_image_embed]] = None + self._last_image_hash: Optional[int] = None if not os.path.exists(clip_model_path): raise ValueError(f"Clip model path does not exist: {clip_model_path}") with suppress_stdout_stderr(disable=self.verbose): - self.clip_ctx = self._llava_cpp.clip_model_load( + clip_ctx = self._llava_cpp.clip_model_load( self.clip_model_path.encode(), 0 ) - def __del__(self): - with suppress_stdout_stderr(disable=self.verbose): - if self.clip_ctx is not None and self._clip_free is not None: - self._clip_free(self.clip_ctx) - self.clip_ctx = None + if clip_ctx is None: + raise ValueError(f"Failed to load clip model: {clip_model_path}") + + self.clip_ctx = clip_ctx - def load_image(self, image_url: str) -> bytes: - if image_url.startswith("data:"): - import base64 + def clip_free(): + with suppress_stdout_stderr(disable=self.verbose): + self._llava_cpp.clip_free(self.clip_ctx) + + self._exit_stack.callback(clip_free) + + def last_image_embed_free(): + with suppress_stdout_stderr(disable=self.verbose): + if self._last_image_embed is not None: + self._llava_cpp.llava_image_embed_free(self._last_image_embed) + self._last_image_embed = None - image_bytes = base64.b64decode(image_url.split(",")[1]) - return image_bytes - else: - import urllib.request + self._exit_stack.callback(last_image_embed_free) - with urllib.request.urlopen(image_url) as f: - image_bytes = f.read() - return image_bytes + def load_image(self, image_url: str) -> bytes: + return self._load_image(image_url) def __call__( self, @@ -2216,6 +2256,7 @@ def __call__( typical_p: float = 1.0, stream: bool = False, stop: Optional[Union[str, List[str]]] = [], + seed: Optional[int] = None, response_format: Optional[ llama_types.ChatCompletionRequestResponseFormat ] = None, @@ -2230,121 +2271,477 @@ def __call__( model: Optional[str] = None, logits_processor: Optional[llama.LogitsProcessorList] = None, grammar: Optional[llama.LlamaGrammar] = None, + logit_bias: Optional[Dict[str, float]] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, Iterator[llama_types.CreateChatCompletionStreamResponse], ]: - assert ( - llama.context_params.logits_all is True - ) # BUG: logits_all=True is required for llava assert self.clip_ctx is not None + system_prompt = _get_system_message(messages) - system_prompt = ( - system_prompt - if system_prompt != "" - else "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." - ) - user_role = "\nUSER:" - assistant_role = "\nASSISTANT:" - llama.reset() - llama.eval(llama.tokenize(system_prompt.encode("utf8"), add_bos=True)) - for message in messages: - if message["role"] == "user" and message["content"] is not None: - if isinstance(message["content"], str): - llama.eval( - llama.tokenize( - f"{user_role} {message['content']}".encode("utf8"), - add_bos=False, - ) - ) - else: - assert isinstance(message["content"], list) - llama.eval( - llama.tokenize(f"{user_role} ".encode("utf8"), add_bos=False) - ) - for content in message["content"]: - if content["type"] == "text": - llama.eval( - llama.tokenize( - f"{content['text']}".encode("utf8"), add_bos=False - ) - ) - if content["type"] == "image_url": - image_bytes = ( - self.load_image(content["image_url"]["url"]) - if isinstance(content["image_url"], dict) - else self.load_image(content["image_url"]) - ) - import array - - data_array = array.array("B", image_bytes) - c_ubyte_ptr = ( - ctypes.c_ubyte * len(data_array) - ).from_buffer(data_array) - with suppress_stdout_stderr(disable=self.verbose): - embed = ( - self._llava_cpp.llava_image_embed_make_with_bytes( - self.clip_ctx, - llama.context_params.n_threads, - c_ubyte_ptr, - len(image_bytes), - ) - ) - try: - n_past = ctypes.c_int(llama.n_tokens) - n_past_p = ctypes.pointer(n_past) - with suppress_stdout_stderr(disable=self.verbose): - self._llava_cpp.llava_eval_image_embed( - llama.ctx, - embed, - llama.n_batch, - n_past_p, - ) - assert llama.n_ctx() >= n_past.value - llama.n_tokens = n_past.value - finally: - with suppress_stdout_stderr(disable=self.verbose): - self._llava_cpp.llava_image_embed_free(embed) - if message["role"] == "assistant" and message["content"] is not None: - llama.eval( - llama.tokenize( - f"ASSISTANT: {message['content']}".encode("utf8"), add_bos=False + if system_prompt == "": + messages = [llama_types.ChatCompletionRequestSystemMessage(role="system", content=self.DEFAULT_SYSTEM_MESSAGE)] + messages + + image_urls = self.get_image_urls(messages) + template = jinja2.Template(self.CHAT_FORMAT) + text = template.render(messages=messages, add_generation_prompt=True) + split_text = self.split_text_on_image_urls(text, image_urls) + + def embed_image_bytes(image_bytes: bytes): + if self._last_image_embed is not None and self._last_image_hash is not None and hash(image_bytes) == self._last_image_hash: + return self._last_image_embed + with suppress_stdout_stderr(disable=self.verbose): + embed = ( + self._llava_cpp.llava_image_embed_make_with_bytes( + self.clip_ctx, + llama.context_params.n_threads_batch, + (ctypes.c_uint8 * len(image_bytes)).from_buffer(bytearray(image_bytes)), + len(image_bytes), ) ) - assert llama.n_ctx() >= llama.n_tokens - llama.eval(llama.tokenize(f"{assistant_role}".encode("utf8"), add_bos=False)) - assert llama.n_ctx() >= llama.n_tokens + self._last_image_embed = embed + self._last_image_hash = hash(image_bytes) + return embed + + # Evaluate prompt + llama.reset() + for i, (type_, value) in enumerate(split_text): + if type_ == "text": + tokens = llama.tokenize(value.encode("utf8"), add_bos=i == 0) + if llama.n_tokens + len(tokens) > llama.n_ctx(): + raise ValueError("Prompt exceeds n_ctx") # TODO: Fix + llama.eval(tokens) + else: + image_bytes = self.load_image(value) + embed = embed_image_bytes(image_bytes) + if llama.n_tokens + embed.contents.n_image_pos > llama.n_ctx(): + raise ValueError("Prompt exceeds n_ctx") # TODO: Fix + n_past = ctypes.c_int(llama.n_tokens) + n_past_p = ctypes.pointer(n_past) + with suppress_stdout_stderr(disable=self.verbose): + self._llava_cpp.llava_eval_image_embed( + llama.ctx, + embed, + llama.n_batch, + n_past_p, + ) + llama.n_tokens = n_past.value + # Get prompt tokens to avoid a cache miss prompt = llama.input_ids[: llama.n_tokens].tolist() if response_format is not None and response_format["type"] == "json_object": grammar = _grammar_for_response_format(response_format) - return _convert_completion_to_chat( - llama.create_completion( - prompt=prompt, - temperature=temperature, - top_p=top_p, - top_k=top_k, - min_p=min_p, - typical_p=typical_p, - stream=stream, - stop=stop, - max_tokens=max_tokens, - presence_penalty=presence_penalty, - frequency_penalty=frequency_penalty, - repeat_penalty=repeat_penalty, - tfs_z=tfs_z, - mirostat_mode=mirostat_mode, - mirostat_tau=mirostat_tau, - mirostat_eta=mirostat_eta, - model=model, - logits_processor=logits_processor, - grammar=grammar, - ), + # Convert legacy functions to tools + if functions is not None: + tools = [ + { + "type": "function", + "function": function, + } + for function in functions + ] + + # Convert legacy function_call to tool_choice + if function_call is not None: + if isinstance(function_call, str) and ( + function_call == "none" or function_call == "auto" + ): + tool_choice = function_call + if isinstance(function_call, dict) and "name" in function_call: + tool_choice = { + "type": "function", + "function": { + "name": function_call["name"], + }, + } + + tool = None + if tool_choice is not None and isinstance(tool_choice, dict) and tools is not None: + name = tool_choice["function"]["name"] + tool = next((t for t in tools if t["function"]["name"] == name), None) + if tool is None: + raise ValueError(f"Tool choice '{name}' not found in tools.") + schema = tool["function"]["parameters"] + try: + # create grammar from json schema + grammar = llama_grammar.LlamaGrammar.from_json_schema( + json.dumps(schema), verbose=llama.verbose + ) + except Exception as e: + grammar = llama_grammar.LlamaGrammar.from_string( + llama_grammar.JSON_GBNF, verbose=llama.verbose + ) + + completion_or_chunks = llama.create_completion( + prompt=prompt, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + typical_p=typical_p, + logprobs=top_logprobs if logprobs else None, stream=stream, + stop=stop, + seed=seed, + max_tokens=max_tokens, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + repeat_penalty=repeat_penalty, + tfs_z=tfs_z, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + model=model, + logits_processor=logits_processor, + grammar=grammar, + logit_bias=logit_bias, ) + if tool is not None: + tool_name = tool["function"]["name"] + return _convert_completion_to_chat_function( + tool_name, completion_or_chunks, stream + ) + return _convert_completion_to_chat(completion_or_chunks, stream=stream) + + @staticmethod + def _load_image(image_url: str) -> bytes: + # TODO: Add Pillow support for other image formats beyond (jpg, png) + if image_url.startswith("data:"): + import base64 + + image_bytes = base64.b64decode(image_url.split(",")[1]) + return image_bytes + else: + import urllib.request + + with urllib.request.urlopen(image_url) as f: + image_bytes = f.read() + return image_bytes + + @staticmethod + def get_image_urls(messages: List[llama_types.ChatCompletionRequestMessage]): + image_urls: List[str] = [] + for message in messages: + if message["role"] == "user": + if message["content"] is None: + continue + for content in message["content"]: + if isinstance(content, dict) and "type" in content: + if content["type"] == "image_url": + if ( + isinstance(content["image_url"], dict) + and "url" in content["image_url"] + ): + image_urls.append(content["image_url"]["url"]) + else: + image_urls.append(content["image_url"]) + return image_urls + + @staticmethod + def split_text_on_image_urls(text: str, image_urls: List[str]): + def find_first(s: str, substrs: List[str]): + for i, substr in enumerate(substrs): + pos = s.find(substr) + if pos != -1: + return pos, i + return None, None + + split_text: List[Tuple[Literal["text", "image_url"], str]] = [] + remaining = text + while remaining: + # Find first image_url + pos, i = find_first(remaining, image_urls) + if pos is not None and i is not None: + if pos > 0: + split_text.append(("text", remaining[:pos])) + split_text.append(("image_url", image_urls[i])) + remaining = remaining[pos + len(image_urls[i]) :] + else: + split_text.append(("text", remaining)) + remaining = "" + return split_text + + @classmethod + def from_pretrained( + cls, + repo_id: str, + filename: Optional[str], + local_dir: Optional[Union[str, os.PathLike[str]]] = None, + local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", + cache_dir: Optional[Union[str, os.PathLike[str]]] = None, + **kwargs: Any, + ) -> "Llava15ChatHandler": + import fnmatch + from pathlib import Path + try: + from huggingface_hub import hf_hub_download, HfFileSystem # type: ignore + from huggingface_hub.utils import validate_repo_id # type: ignore + except ImportError: + raise ImportError( + "Llama.from_pretrained requires the huggingface-hub package. " + "You can install it with `pip install huggingface-hub`." + ) + + validate_repo_id(repo_id) + + hffs = HfFileSystem() + + files = [ + file["name"] if isinstance(file, dict) else file + for file in hffs.ls(repo_id) # type: ignore + ] + + # split each file into repo_id, subfolder, filename + file_list: List[str] = [] + for file in files: + rel_path = Path(file).relative_to(repo_id) + file_list.append(str(rel_path)) + + matching_files = [file for file in file_list if fnmatch.fnmatch(file, filename)] # type: ignore + + if len(matching_files) == 0: + raise ValueError( + f"No file found in {repo_id} that match {filename}\n\n" + f"Available Files:\n{json.dumps(file_list)}" + ) + + if len(matching_files) > 1: + raise ValueError( + f"Multiple files found in {repo_id} matching {filename}\n\n" + f"Available Files:\n{json.dumps(files)}" + ) + + (matching_file,) = matching_files + + subfolder = str(Path(matching_file).parent) + filename = Path(matching_file).name + + # download the file + hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder, + local_dir=cast(Union[str, Path, None], local_dir), + local_dir_use_symlinks=local_dir_use_symlinks, + cache_dir=cast(Union[str, Path, None], cache_dir), + ) + + if local_dir is None: + model_path = hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder, + local_dir=local_dir, + local_dir_use_symlinks=local_dir_use_symlinks, + cache_dir=cast(Union[str, Path, None], cache_dir), + local_files_only=True, + ) + else: + model_path = os.path.join(local_dir, filename) + + return cls( + clip_model_path=model_path, + **kwargs, + ) + +class ObsidianChatHandler(Llava15ChatHandler): + # Prompt Format + # The model followed ChatML format. However, with ### as the seperator + + # <|im_start|>user + # What is this sign about?\n + # ### + # <|im_start|>assistant + # The sign is about bullying, and it is placed on a black background with a red background. + # ### + + CHAT_FORMAT = ( + "{% for message in messages %}" + # System message + "{% if message.role == 'system' %}" + "<|im_start|>system\n" + "{{ message.content }}\n" + "###\n" + "{% endif %}" + # User message + "{% if message.role == 'user' %}" + "<|im_start|>user\n" + "{% if message.content is string %}" + "{{ message.content }}" + "{% endif %}" + "{% if message.content is iterable %}" + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% if content.type == 'image_url' %}" + "{{ content.image_url }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "###\n" + "{% endif %}" + # Assistant message + "{% if message.role == 'assistant' %}" + "<|im_start|>assistant\n" + "{{ message.content }}" + "###\n" + "{% endif %}" + "{% endfor %}" + # Generation prompt + "{% if add_generation_prompt %}" + "<|im_start|>assistant\n" + "{% endif %}" + ) + +class MoondreamChatHandler(Llava15ChatHandler): + # Chat Format: + # f"\n\n{chat_history}Question: {question}\n\nAnswer:" + CHAT_FORMAT = ( + "{% for message in messages %}" + "{% if message.role == 'user' %}" + "{% if message.content is iterable %}" + "{% for content in message.content %}" + + # + "{% if content.type == 'image_url' %}" + "{% if content.image_url is string %}" + "{{ content.image_url }}\n\n" + "{% endif %}" + "{% if content.image_url is mapping %}" + "{{ content.image_url.url }}\n\n" + "{% endif %}" + "{% endif %}" + + # Question: + "{% if content.type == 'text' %}" + "Question: {{ content.text }}\n\n" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + + # Question: + "{% if message.content is string %}" + "Question: {{ message.content }}\n\n" + "{% endif %}" + + "{% endif %}" + + # Answer: + "{% if message.role == 'assistant' %}" + "Answer:{{ message.content }}\n\n" + "{% endif %}" + "{% endfor %}" + + # Generation prompt + "{% if add_generation_prompt %}" + "Answer:" + "{% endif %}" + ) + +class Llava16ChatHandler(Llava15ChatHandler): + DEFAULT_SYSTEM_MESSAGE = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. " + + # Example prompt + # "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. USER: \nWhat is shown in this image? ASSISTANT:" + + CHAT_FORMAT = ( + "{% for message in messages %}" + "{% if message.role == 'system' %}" + "{{ message.content }}" + "{% endif %}" + "{% if message.role == 'user' %}" + "{% if message.content is iterable %}" + "{% for content in message.content %}" + + # + "{% if content.type == 'image_url' %}" + "{% if content.image_url is string %}" + "{{ content.image_url }}\n" + "{% endif %}" + "{% if content.image_url is mapping %}" + "{{ content.image_url.url }}\n" + "{% endif %}" + "{% endif %}" + + # Question: + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + + # Question: + "{% if message.content is string %}" + "{{ message.content }}" + "{% endif %}" + + "{% endif %}" + + # Answer: + "{% if message.role == 'assistant' %}" + "{{ message.content }}" + "{% endif %}" + "{% endfor %}" + + # Generation prompt + "{% if add_generation_prompt %}" + "Answer:" + "{% endif %}" + ) + +class NanoLlavaChatHandler(Llava15ChatHandler): + # Prompt Format + # The model follow the ChatML standard, however, without \n at the end of <|im_end|>: + + # <|im_start|>system + # Answer the question<|im_end|><|im_start|>user + # + # What is the picture about?<|im_end|><|im_start|>assistant + + CHAT_FORMAT = ( + "{% for message in messages %}" + # System message + "{% if message.role == 'system' %}" + "<|im_start|>system\n" + "{{ message.content }}" + "<|im_end|>" + "{% endif %}" + # User message + "{% if message.role == 'user' %}" + "<|im_start|>user\n" + "{% if message.content is string %}" + "{{ message.content }}" + "{% endif %}" + "{% if message.content is iterable %}" + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% if content.type == 'image_url' %}" + "{{ content.image_url }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "<|im_end|>" + "{% endif %}" + # Assistant message + "{% if message.role == 'assistant' %}" + "<|im_start|>assistant\n" + "{{ message.content }}" + "<|im_end|>" + "{% endif %}" + "{% endfor %}" + # Generation prompt + "{% if add_generation_prompt %}" + "<|im_start|>assistant\n" + "{% endif %}" + ) @register_chat_completion_handler("chatml-function-calling") diff --git a/llama_cpp/llava_cpp.py b/llama_cpp/llava_cpp.py index 543c87d0b3..3ded1f25d3 100644 --- a/llama_cpp/llava_cpp.py +++ b/llama_cpp/llava_cpp.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import sys import os import ctypes @@ -14,10 +16,22 @@ Structure, ) import pathlib -from typing import List, Union, NewType, Optional, TypeVar, Callable, Any +from typing import ( + List, + Union, + NewType, + Optional, + TypeVar, + Callable, + Any, + TYPE_CHECKING, + Generic, +) +from typing_extensions import TypeAlias import llama_cpp.llama_cpp as llama_cpp + # Load the library def _load_shared_library(lib_base_name: str): # Construct the paths to the possible shared library names @@ -62,7 +76,7 @@ def _load_shared_library(lib_base_name: str): for _lib_path in _lib_paths: if _lib_path.exists(): try: - return ctypes.CDLL(str(_lib_path), **cdll_args) # type: ignore + return ctypes.CDLL(str(_lib_path), **cdll_args) # type: ignore except Exception as e: raise RuntimeError(f"Failed to load shared library '{_lib_path}': {e}") @@ -79,8 +93,27 @@ def _load_shared_library(lib_base_name: str): # ctypes helper +if TYPE_CHECKING: + CtypesCData = TypeVar("CtypesCData", bound=ctypes._CData) # type: ignore + + CtypesArray: TypeAlias = ctypes.Array[CtypesCData] # type: ignore + + CtypesPointer: TypeAlias = ctypes._Pointer[CtypesCData] # type: ignore + + CtypesVoidPointer: TypeAlias = ctypes.c_void_p + + class CtypesRef(Generic[CtypesCData]): + pass + + CtypesPointerOrRef: TypeAlias = Union[ + CtypesPointer[CtypesCData], CtypesRef[CtypesCData] + ] + + CtypesFuncPointer: TypeAlias = ctypes._FuncPointer # type: ignore + F = TypeVar("F", bound=Callable[..., Any]) + def ctypes_function_for_shared_library(lib: ctypes.CDLL): def ctypes_function( name: str, argtypes: List[Any], restype: Any, enabled: bool = True @@ -111,6 +144,7 @@ def decorator(f: F) -> F: clip_ctx_p = NewType("clip_ctx_p", int) clip_ctx_p_ctypes = c_void_p + # struct llava_image_embed { # float * embed; # int n_image_pos; @@ -121,36 +155,72 @@ class llava_image_embed(Structure): ("n_image_pos", c_int), ] + # /** sanity check for clip <-> llava embed size match */ # LLAVA_API bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip); -@ctypes_function("llava_validate_embed_size", [llama_cpp.llama_context_p_ctypes, clip_ctx_p_ctypes], c_bool) -def llava_validate_embed_size(ctx_llama: llama_cpp.llama_context_p, ctx_clip: clip_ctx_p, /) -> bool: - ... +@ctypes_function( + "llava_validate_embed_size", + [llama_cpp.llama_context_p_ctypes, clip_ctx_p_ctypes], + c_bool, +) +def llava_validate_embed_size( + ctx_llama: llama_cpp.llama_context_p, ctx_clip: clip_ctx_p, / +) -> bool: ... # /** build an image embed from image file bytes */ # LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length); -@ctypes_function("llava_image_embed_make_with_bytes", [clip_ctx_p_ctypes, c_int, POINTER(c_uint8), c_int], POINTER(llava_image_embed)) -def llava_image_embed_make_with_bytes(ctx_clip: clip_ctx_p, n_threads: Union[c_int, int], image_bytes: bytes, image_bytes_length: Union[c_int, int], /) -> "_Pointer[llava_image_embed]": - ... +@ctypes_function( + "llava_image_embed_make_with_bytes", + [clip_ctx_p_ctypes, c_int, POINTER(c_uint8), c_int], + POINTER(llava_image_embed), +) +def llava_image_embed_make_with_bytes( + ctx_clip: clip_ctx_p, + n_threads: Union[c_int, int], + image_bytes: CtypesArray[c_uint8], + image_bytes_length: Union[c_int, int], + /, +) -> "_Pointer[llava_image_embed]": ... + # /** build an image embed from a path to an image filename */ # LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path); -@ctypes_function("llava_image_embed_make_with_filename", [clip_ctx_p_ctypes, c_int, c_char_p], POINTER(llava_image_embed)) -def llava_image_embed_make_with_filename(ctx_clip: clip_ctx_p, n_threads: Union[c_int, int], image_path: bytes, /) -> "_Pointer[llava_image_embed]": - ... +@ctypes_function( + "llava_image_embed_make_with_filename", + [clip_ctx_p_ctypes, c_int, c_char_p], + POINTER(llava_image_embed), +) +def llava_image_embed_make_with_filename( + ctx_clip: clip_ctx_p, n_threads: Union[c_int, int], image_path: bytes, / +) -> "_Pointer[llava_image_embed]": ... + # LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed); # /** free an embedding made with llava_image_embed_make_* */ @ctypes_function("llava_image_embed_free", [POINTER(llava_image_embed)], None) -def llava_image_embed_free(embed: "_Pointer[llava_image_embed]", /): - ... +def llava_image_embed_free(embed: "_Pointer[llava_image_embed]", /): ... + # /** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */ # LLAVA_API bool llava_eval_image_embed(struct llama_context * ctx_llama, const struct llava_image_embed * embed, int n_batch, int * n_past); -@ctypes_function("llava_eval_image_embed", [llama_cpp.llama_context_p_ctypes, POINTER(llava_image_embed), c_int, POINTER(c_int)], c_bool) -def llava_eval_image_embed(ctx_llama: llama_cpp.llama_context_p, embed: "_Pointer[llava_image_embed]", n_batch: Union[c_int, int], n_past: "_Pointer[c_int]", /) -> bool: - ... +@ctypes_function( + "llava_eval_image_embed", + [ + llama_cpp.llama_context_p_ctypes, + POINTER(llava_image_embed), + c_int, + POINTER(c_int), + ], + c_bool, +) +def llava_eval_image_embed( + ctx_llama: llama_cpp.llama_context_p, + embed: "_Pointer[llava_image_embed]", + n_batch: Union[c_int, int], + n_past: "_Pointer[c_int]", + /, +) -> bool: ... ################################################ @@ -161,11 +231,12 @@ def llava_eval_image_embed(ctx_llama: llama_cpp.llama_context_p, embed: "_Pointe # /** load mmproj model */ # CLIP_API struct clip_ctx * clip_model_load (const char * fname, int verbosity); @ctypes_function("clip_model_load", [c_char_p, c_int], clip_ctx_p_ctypes) -def clip_model_load(fname: bytes, verbosity: Union[c_int, int], /) -> Optional[clip_ctx_p]: - ... +def clip_model_load( + fname: bytes, verbosity: Union[c_int, int], / +) -> Optional[clip_ctx_p]: ... + # /** free mmproj model */ # CLIP_API void clip_free(struct clip_ctx * ctx); @ctypes_function("clip_free", [clip_ctx_p_ctypes], None) -def clip_free(ctx: clip_ctx_p, /): - ... +def clip_free(ctx: clip_ctx_p, /): ... diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index c24fca6523..1ad592d211 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -72,9 +72,74 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler = None if settings.chat_format == "llava-1-5": assert settings.clip_model_path is not None, "clip model not found" - chat_handler = llama_cpp.llama_chat_format.Llava15ChatHandler( - clip_model_path=settings.clip_model_path, verbose=settings.verbose - ) + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.Llava15ChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.Llava15ChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) + elif settings.chat_format == "obsidian": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.ObsidianChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.ObsidianChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) + elif settings.chat_format == "llava-1-6": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.Llava16ChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.Llava16ChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) + elif settings.chat_format == "moondream": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.MoondreamChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.MoondreamChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) + elif settings.chat_format == "nanollava": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.NanoLlavaChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.NanoLlavaChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) elif settings.chat_format == "hf-autotokenizer": assert ( settings.hf_pretrained_model_name_or_path is not None From 26c7876ba0b31bd0e9f7bfc8f341b8b5b3d81fa7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 01:48:40 -0400 Subject: [PATCH 116/561] chore: Bump version --- CHANGELOG.md | 11 +++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 806876ce2f..26df77fb35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.66] + +- feat: Update llama.cpp to ggerganov/llama.cpp@8843a98c2ba97a25e93319a104f9ddfaf83ce4c4 +- feat: Generic Chat Formats, Tool Calling, and Huggingface Pull Support for Multimodal Models (Obsidian, LLaVA1.6, Moondream) by @abetlen in #1147 +- ci(fix): Workflow actions updates and fix arm64 wheels not included in release by @Smartappli in #1392 +- ci: Add support for pre-built cuda 12.4.1 wheels by @Smartappli in #1388 +- feat: Add support for str type kv_overrides by @abetlen in a411612b385cef100d76145da1fbd02a7b7cc894 +- fix: Functionary bug fixes by @jeffrey-fong in #1385 +- examples: fix quantize example by @iyubondyrev in #1387 +- ci: Update dependabot.yml by @Smartappli in #1391 + ## [0.2.65] - feat: Update llama.cpp to ggerganov/llama.cpp@46e12c4692a37bdd31a0432fc5153d7d22bc7f72 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 6db6333ba5..49657c973e 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.65" \ No newline at end of file +__version__ = "0.2.66" \ No newline at end of file From d03f15bb73a1d520970357b702a9e7d4cc2a7a62 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 02:58:55 -0400 Subject: [PATCH 117/561] fix(ci): Fix bug in use of upload-artifact failing to merge multiple artifacts into a single release. --- .github/workflows/build-and-release.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 78bcb87d4e..bb3c9f8d9b 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -39,9 +39,10 @@ jobs: - uses: actions/upload-artifact@v4 with: + name: wheels path: ./wheelhouse/*.whl - build_arm64_wheels: + build_wheels_arm64: name: Build arm64 wheels runs-on: ubuntu-latest steps: @@ -67,6 +68,7 @@ jobs: - name: Upload wheels as artifacts uses: actions/upload-artifact@v4 with: + name: wheels_arm64 path: ./wheelhouse/*.whl build_sdist: @@ -89,18 +91,25 @@ jobs: python -m build --sdist - uses: actions/upload-artifact@v4 with: + name: sdist path: ./dist/*.tar.gz release: name: Release - needs: [build_wheels, build_arm64_wheels, build_sdist] + needs: [build_wheels, build_wheels_arm64, build_sdist] runs-on: ubuntu-latest steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: release + - uses: actions/download-artifact@v3 with: - name: artifact + name: release path: dist + - uses: softprops/action-gh-release@v2 with: files: dist/* From 3489ef09d3775f4a87fb7114f619e8ba9cb6b656 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 03:08:46 -0400 Subject: [PATCH 118/561] fix: Ensure image renders before text in chat formats regardless of message content order. --- llama_cpp/llama_chat_format.py | 52 ++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index cbd8d203ce..63eaf8a03b 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2175,12 +2175,11 @@ class Llava15ChatHandler: "{% if message.role == 'user' %}" "{% if message.content is string %}" "\nUSER: {{ message.content }}" - "{% elif message.content is iterable %}" + "{% endif %}" + "{% if message.content is iterable %}" "\nUSER: " + "{% for content in message.content %}" - "{% if content.type == 'text' %}" - "{{ content.text }}" - "{% endif %}" "{% if content.type == 'image_url' and content.image_url is string %}" "{{ content.image_url }}" "{% endif %}" @@ -2188,6 +2187,13 @@ class Llava15ChatHandler: "{{ content.image_url.url }}" "{% endif %}" "{% endfor %}" + + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" "{% endif %}" "{% if message.role == 'assistant' and message.content is not none %}" @@ -2575,14 +2581,22 @@ class ObsidianChatHandler(Llava15ChatHandler): "{{ message.content }}" "{% endif %}" "{% if message.content is iterable %}" + + "{% for content in message.content %}" + "{% if content.type == 'image_url' and content.image_url is string %}" + "{{ content.image_url }}" + "{% endif %}" + "{% if content.type == 'image_url' and content.image_url is mapping %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endfor %}" + "{% for content in message.content %}" "{% if content.type == 'text' %}" "{{ content.text }}" "{% endif %}" - "{% if content.type == 'image_url' %}" - "{{ content.image_url }}" - "{% endif %}" "{% endfor %}" + "{% endif %}" "###\n" "{% endif %}" @@ -2606,9 +2620,9 @@ class MoondreamChatHandler(Llava15ChatHandler): "{% for message in messages %}" "{% if message.role == 'user' %}" "{% if message.content is iterable %}" - "{% for content in message.content %}" # + "{% for content in message.content %}" "{% if content.type == 'image_url' %}" "{% if content.image_url is string %}" "{{ content.image_url }}\n\n" @@ -2617,12 +2631,15 @@ class MoondreamChatHandler(Llava15ChatHandler): "{{ content.image_url.url }}\n\n" "{% endif %}" "{% endif %}" + "{% endfor %}" # Question: + "{% for content in message.content %}" "{% if content.type == 'text' %}" "Question: {{ content.text }}\n\n" "{% endif %}" "{% endfor %}" + "{% endif %}" # Question: @@ -2657,9 +2674,9 @@ class Llava16ChatHandler(Llava15ChatHandler): "{% endif %}" "{% if message.role == 'user' %}" "{% if message.content is iterable %}" - "{% for content in message.content %}" # + "{% for content in message.content %}" "{% if content.type == 'image_url' %}" "{% if content.image_url is string %}" "{{ content.image_url }}\n" @@ -2668,12 +2685,15 @@ class Llava16ChatHandler(Llava15ChatHandler): "{{ content.image_url.url }}\n" "{% endif %}" "{% endif %}" + "{% endfor %}" # Question: + "{% for content in message.content %}" "{% if content.type == 'text' %}" "{{ content.text }}" "{% endif %}" "{% endfor %}" + "{% endif %}" # Question: @@ -2719,14 +2739,22 @@ class NanoLlavaChatHandler(Llava15ChatHandler): "{{ message.content }}" "{% endif %}" "{% if message.content is iterable %}" + + "{% for content in message.content %}" + "{% if content.type == 'image_url' and content.image_url is string %}" + "{{ content.image_url }}" + "{% endif %}" + "{% if content.type == 'image_url' and content.image_url is mapping %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endfor %}" + "{% for content in message.content %}" "{% if content.type == 'text' %}" "{{ content.text }}" "{% endif %}" - "{% if content.type == 'image_url' %}" - "{{ content.image_url }}" - "{% endif %}" "{% endfor %}" + "{% endif %}" "<|im_end|>" "{% endif %}" From f417cce28a3f3386206dabe98e87c038d05b228d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 03:11:02 -0400 Subject: [PATCH 119/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26df77fb35..4b2c34b22e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.67] + +- fix: Ensure image renders before text in chat formats regardless of message content order by @abetlen in 3489ef09d3775f4a87fb7114f619e8ba9cb6b656 +- fix(ci): Fix bug in use of upload-artifact failing to merge multiple artifacts into a single release by @abetlen in d03f15bb73a1d520970357b702a9e7d4cc2a7a62 + ## [0.2.66] - feat: Update llama.cpp to ggerganov/llama.cpp@8843a98c2ba97a25e93319a104f9ddfaf83ce4c4 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 49657c973e..286baf445d 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.66" \ No newline at end of file +__version__ = "0.2.67" \ No newline at end of file From c8cd8c17c6f178285579ce1f06347edb7324088c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 03:12:46 -0400 Subject: [PATCH 120/561] docs: Update README to include CUDA 12.4 wheels --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d34caf409a..310873f542 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements: -- CUDA Version is 12.1, 12.2 or 12.3 +- CUDA Version is 12.1, 12.2, 12.3, or 12.4 - Python Version is 3.10, 3.11 or 3.12 ```bash @@ -133,6 +133,7 @@ Where `` is one of the following: - `cu121`: CUDA 12.1 - `cu122`: CUDA 12.2 - `cu123`: CUDA 12.3 +- `cu124`: CUDA 12.4 For example, to install the CUDA 12.1 wheel: From 6332527a698f5aa2f524773f42a0510ba5f650a0 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Tue, 30 Apr 2024 15:16:14 +0200 Subject: [PATCH 121/561] fix(ci): Fix build-and-release.yaml (#1413) * Update build-and-release.yaml * Update build-and-release.yaml --- .github/workflows/build-and-release.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index bb3c9f8d9b..8fbd68f4ad 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -39,7 +39,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-${{ matrix.os }} path: ./wheelhouse/*.whl build_wheels_arm64: @@ -100,14 +100,9 @@ jobs: runs-on: ubuntu-latest steps: - - name: Merge Artifacts - uses: actions/upload-artifact/merge@v4 + - uses: actions/download-artifact@v4 with: - name: release - - - uses: actions/download-artifact@v3 - with: - name: release + merge-multiple: true path: dist - uses: softprops/action-gh-release@v2 From 8c2b24d5aafffffadf37c3067ca12a45b59d95c4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 09:27:55 -0400 Subject: [PATCH 122/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 10 +++++++--- vendor/llama.cpp | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 9c8f77889e..46aa51662f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -242,8 +242,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 5 -LLAMA_SESSION_VERSION = 5 +# define LLAMA_SESSION_VERSION 6 +LLAMA_SESSION_VERSION = 6 # define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ @@ -730,6 +730,7 @@ class llama_model_params(ctypes.Structure): # bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) # bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU +# bool flash_attn; // whether to use flash attention # // Abort callback @@ -766,6 +767,7 @@ class llama_context_params(ctypes.Structure): logits_all (bool): the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU + flash_attn (bool): whether to use flash attention abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback """ @@ -795,6 +797,7 @@ class llama_context_params(ctypes.Structure): logits_all: bool embeddings: bool offload_kqv: bool + flash_attn: bool abort_callback: Callable[[ctypes.c_void_p], bool] abort_callback_data: ctypes.c_void_p @@ -823,6 +826,7 @@ class llama_context_params(ctypes.Structure): ("logits_all", ctypes.c_bool), ("embeddings", ctypes.c_bool), ("offload_kqv", ctypes.c_bool), + ("flash_attn", ctypes.c_bool), ("abort_callback", ggml_abort_callback), ("abort_callback_data", ctypes.c_void_p), ] @@ -1615,7 +1619,7 @@ def llama_get_kv_cache_used_cells(ctx: llama_context_p, /) -> int: ... -# // Clear the KV cache +# // Clear the KV cache - both cell info is erased and KV data is zeroed # LLAMA_API void llama_kv_cache_clear( # struct llama_context * ctx); @ctypes_function("llama_kv_cache_clear", [llama_context_p_ctypes], None) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8843a98c2b..77e15bec62 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8843a98c2ba97a25e93319a104f9ddfaf83ce4c4 +Subproject commit 77e15bec6217a39be59b9cc83d6b9afb6b0d8167 From 22d77eefd2edaf0148f53374d0cac74d0e25d06e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 09:29:16 -0400 Subject: [PATCH 123/561] feat: Add option to enable `flash_attn` to Lllama params and ModelSettings --- llama_cpp/llama.py | 4 ++++ llama_cpp/server/settings.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 96aac6668b..172f4c6460 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -92,6 +92,7 @@ def __init__( logits_all: bool = False, embedding: bool = False, offload_kqv: bool = True, + flash_attn: bool = False, # Sampling Params last_n_tokens_size: int = 64, # LoRA Params @@ -168,6 +169,7 @@ def __init__( logits_all: Return logits for all tokens, not just the last token. Must be True for completion to return logprobs. embedding: Embedding mode only. offload_kqv: Offload K, Q, V to GPU. + flash_attn: Use flash attention. last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque. lora_base: Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model. lora_path: Path to a LoRA file to apply to the model. @@ -310,6 +312,7 @@ def __init__( ) # Must be set to True for speculative decoding self.context_params.embeddings = embedding # TODO: Rename to embeddings self.context_params.offload_kqv = offload_kqv + self.context_params.flash_attn = flash_attn # KV cache quantization if type_k is not None: self.context_params.type_k = type_k @@ -1774,6 +1777,7 @@ def __getstate__(self): logits_all=self.context_params.logits_all, embedding=self.context_params.embeddings, offload_kqv=self.context_params.offload_kqv, + flash_offload=self.context_params.flash_offload, # Sampling Params last_n_tokens_size=self.last_n_tokens_size, # LoRA Params diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 0c858f9fe9..ed05a889f0 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -96,6 +96,9 @@ class ModelSettings(BaseSettings): offload_kqv: bool = Field( default=True, description="Whether to offload kqv to the GPU." ) + flash_attn: bool = Field( + default=False, description="Whether to use flash attention." + ) # Sampling Params last_n_tokens_size: int = Field( default=64, From 29b6e9a5c832b7d148044b45fb3cd6f3f923d96b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 09:32:47 -0400 Subject: [PATCH 124/561] fix: wrong parameter for flash attention in pickle __getstate__ --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 172f4c6460..f927f0ca26 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1777,7 +1777,7 @@ def __getstate__(self): logits_all=self.context_params.logits_all, embedding=self.context_params.embeddings, offload_kqv=self.context_params.offload_kqv, - flash_offload=self.context_params.flash_offload, + flash_attn=self.context_params.flash_attn, # Sampling Params last_n_tokens_size=self.last_n_tokens_size, # LoRA Params From b14dd98922c7f18468ae202eadbaf58fe17f5320 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 09:39:56 -0400 Subject: [PATCH 125/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b2c34b22e..c9681f355b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.68] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ +- feat: Add option to enable flash_attn to Lllama params and ModelSettings by @abetlen in 22d77eefd2edaf0148f53374d0cac74d0e25d06e +- fix(ci): Fix build-and-release.yaml by @Smartappli in #1413 + ## [0.2.67] - fix: Ensure image renders before text in chat formats regardless of message content order by @abetlen in 3489ef09d3775f4a87fb7114f619e8ba9cb6b656 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 286baf445d..63c622551c 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.67" \ No newline at end of file +__version__ = "0.2.68" \ No newline at end of file From 26478ab293bde6374e2c350fed7b07c3027939b6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 10:11:38 -0400 Subject: [PATCH 126/561] docs: Update README.md --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 310873f542..fce8eda57b 100644 --- a/README.md +++ b/README.md @@ -529,19 +529,22 @@ Then you'll need to use a custom chat handler to load the clip model and process You can also pull the model from the Hugging Face Hub using the `from_pretrained` method. ```python ->>> from llama_cpp import Llama ->>> from llama_cpp.llama_chat_format import MoondreamChatHandler ->>> chat_handler = MoondreamChatHandler.from_pretrained( +from llama_cpp import Llama +from llama_cpp.llama_chat_format import MoondreamChatHandler + +chat_handler = MoondreamChatHandler.from_pretrained( repo_id="vikhyatk/moondream2", filename="*mmproj*", ) ->>> llm = Llama.from_pretrained( - repo_id="vikhyatk/moondream2" + +llm = Llama.from_pretrained( + repo_id="vikhyatk/moondream2", filename="*text-model*", chat_handler=chat_handler, n_ctx=2048, # n_ctx should be increased to accomodate the image embedding ) ->>> llm.create_chat_completion( + +respoonse = llm.create_chat_completion( messages = [ { "role": "user", @@ -553,6 +556,7 @@ You can also pull the model from the Hugging Face Hub using the `from_pretrained } ] ) +print(response["choices"][0]["text"]) ``` **Note**: Multi-modal models also support tool calling and JSON mode. From 945c62c567a862e1c631a86bf88b86ebacefd40d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 10:15:04 -0400 Subject: [PATCH 127/561] docs: Change all examples from interpreter style to script style. --- README.md | 74 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index fce8eda57b..640671460c 100644 --- a/README.md +++ b/README.md @@ -277,20 +277,26 @@ The high-level API provides a simple managed interface through the [`Llama`](htt Below is a short example demonstrating how to use the high-level API to for basic text completion: ```python ->>> from llama_cpp import Llama ->>> llm = Llama( +from llama_cpp import Llama + +llm = Llama( model_path="./models/7B/llama-model.gguf", # n_gpu_layers=-1, # Uncomment to use GPU acceleration # seed=1337, # Uncomment to set a specific seed # n_ctx=2048, # Uncomment to increase the context window ) ->>> output = llm( +output = llm( "Q: Name the planets in the solar system? A: ", # Prompt max_tokens=32, # Generate up to 32 tokens, set to None to generate up to the end of the context window stop=["Q:", "\n"], # Stop generating just before the model would generate a new question echo=True # Echo the prompt back in the output ) # Generate a completion, can also call create_completion ->>> print(output) +print(output) +``` + +By default `llama-cpp-python` generates completions in an OpenAI compatible format: + +```python { "id": "cmpl-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "object": "text_completion", @@ -345,12 +351,12 @@ The model will will format the messages into a single prompt using the following Set `verbose=True` to see the selected chat format. ```python ->>> from llama_cpp import Llama ->>> llm = Llama( +from llama_cpp import Llama +llm = Llama( model_path="path/to/llama-2/llama-model.gguf", chat_format="llama-2" ) ->>> llm.create_chat_completion( +llm.create_chat_completion( messages = [ {"role": "system", "content": "You are an assistant who perfectly describes images."}, { @@ -375,9 +381,9 @@ To constrain chat responses to only valid JSON or a specific JSON Schema use the The following example will constrain the response to valid JSON strings only. ```python ->>> from llama_cpp import Llama ->>> llm = Llama(model_path="path/to/model.gguf", chat_format="chatml") ->>> llm.create_chat_completion( +from llama_cpp import Llama +llm = Llama(model_path="path/to/model.gguf", chat_format="chatml") +llm.create_chat_completion( messages=[ { "role": "system", @@ -397,9 +403,9 @@ The following example will constrain the response to valid JSON strings only. To constrain the response further to a specific JSON Schema add the schema to the `schema` property of the `response_format` argument. ```python ->>> from llama_cpp import Llama ->>> llm = Llama(model_path="path/to/model.gguf", chat_format="chatml") ->>> llm.create_chat_completion( +from llama_cpp import Llama +llm = Llama(model_path="path/to/model.gguf", chat_format="chatml") +llm.create_chat_completion( messages=[ { "role": "system", @@ -424,9 +430,9 @@ To constrain the response further to a specific JSON Schema add the schema to th The high-level API supports OpenAI compatible function and tool calling. This is possible through the `functionary` pre-trained models chat format or through the generic `chatml-function-calling` chat format. ```python ->>> from llama_cpp import Llama ->>> llm = Llama(model_path="path/to/chatml/llama-model.gguf", chat_format="chatml-function-calling") ->>> llm.create_chat_completion( +from llama_cpp import Llama +llm = Llama(model_path="path/to/chatml/llama-model.gguf", chat_format="chatml-function-calling") +llm.create_chat_completion( messages = [ { "role": "system", @@ -476,9 +482,9 @@ The various gguf-converted files for this set of models can be found [here](http Due to discrepancies between llama.cpp and HuggingFace's tokenizers, it is required to provide HF Tokenizer for functionary. The `LlamaHFTokenizer` class can be initialized and passed into the Llama class. This will override the default llama.cpp tokenizer used in Llama class. The tokenizer files are already included in the respective HF repositories hosting the gguf files. ```python ->>> from llama_cpp import Llama ->>> from llama_cpp.llama_tokenizer import LlamaHFTokenizer ->>> llm = Llama.from_pretrained( +from llama_cpp import Llama +from llama_cpp.llama_tokenizer import LlamaHFTokenizer +llm = Llama.from_pretrained( repo_id="meetkai/functionary-small-v2.2-GGUF", filename="functionary-small-v2.2.q4_0.gguf", chat_format="functionary-v2", @@ -504,15 +510,15 @@ You'll first need to download one of the available multi-modal models in GGUF fo Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. ```python ->>> from llama_cpp import Llama ->>> from llama_cpp.llama_chat_format import Llava15ChatHandler ->>> chat_handler = Llava15ChatHandler(clip_model_path="path/to/llava/mmproj.bin") ->>> llm = Llama( +from llama_cpp import Llama +from llama_cpp.llama_chat_format import Llava15ChatHandler +chat_handler = Llava15ChatHandler(clip_model_path="path/to/llava/mmproj.bin") +llm = Llama( model_path="./path/to/llava/llama-model.gguf", chat_handler=chat_handler, n_ctx=2048, # n_ctx should be increased to accomodate the image embedding ) ->>> llm.create_chat_completion( +llm.create_chat_completion( messages = [ {"role": "system", "content": "You are an assistant who perfectly describes images."}, { @@ -709,18 +715,18 @@ The entire low-level API can be found in [llama_cpp/llama_cpp.py](https://github Below is a short example demonstrating how to use the low-level API to tokenize a prompt: ```python ->>> import llama_cpp ->>> import ctypes ->>> llama_cpp.llama_backend_init(False) # Must be called once at the start of each program ->>> params = llama_cpp.llama_context_default_params() +import llama_cpp +import ctypes +llama_cpp.llama_backend_init(False) # Must be called once at the start of each program +params = llama_cpp.llama_context_default_params() # use bytes for char * params ->>> model = llama_cpp.llama_load_model_from_file(b"./models/7b/llama-model.gguf", params) ->>> ctx = llama_cpp.llama_new_context_with_model(model, params) ->>> max_tokens = params.n_ctx +model = llama_cpp.llama_load_model_from_file(b"./models/7b/llama-model.gguf", params) +ctx = llama_cpp.llama_new_context_with_model(model, params) +max_tokens = params.n_ctx # use ctypes arrays for array params ->>> tokens = (llama_cpp.llama_token * int(max_tokens))() ->>> n_tokens = llama_cpp.llama_tokenize(ctx, b"Q: Name the planets in the solar system? A: ", tokens, max_tokens, llama_cpp.c_bool(True)) ->>> llama_cpp.llama_free(ctx) +tokens = (llama_cpp.llama_token * int(max_tokens))() +n_tokens = llama_cpp.llama_tokenize(ctx, b"Q: Name the planets in the solar system? A: ", tokens, max_tokens, llama_cpp.c_bool(True)) +llama_cpp.llama_free(ctx) ``` Check out the [examples folder](examples/low_level_api) for more examples of using the low-level API. From 3226b3c5ef93998f5c412fb926ff136b50816bd7 Mon Sep 17 00:00:00 2001 From: Jonathan Soma Date: Tue, 30 Apr 2024 14:33:23 -0400 Subject: [PATCH 128/561] fix: UTF-8 handling with grammars (#1415) Use Python's built-in UTF-8 handling to get code points --- llama_cpp/llama_grammar.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 6c7b57a591..d9a3823b4f 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -556,17 +556,11 @@ def add_rule( # } def decode_utf8(src: const_char_p) -> Tuple[int, const_char_p]: """Decodes a UTF-8 character from the source string.""" - lookup = (1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4) - first_byte = ord(src[0]) # type: int - highbits = first_byte >> 4 # type: int - len = lookup[highbits] # type: int - mask = (1 << (8 - len)) - 1 # type: int - value = first_byte & mask # type: int - end = src + len # type: const_char_p # may overrun! - pos = src + 1 # type: const_char_p - while pos < end and pos[0]: - value = (value << 6) + (ord(pos[0]) & 0x3F) - pos += 1 + # Get the codepoint of the first character + value = ord(src[0]) + # Move the pointer ahead one character + pos = src + 1 + return value, pos From f116175a5a7c84569c88cad231855c1e6e59ff6e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 15:45:34 -0400 Subject: [PATCH 129/561] fix: Suppress all logs when verbose=False, use hardcoded fileno's to work in colab notebooks. Closes #796 Closes #729 --- llama_cpp/_internals.py | 8 +++++--- llama_cpp/_utils.py | 25 +++++++++++-------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index cc3d989287..d7409f63a0 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -15,6 +15,7 @@ from .llama_types import * from .llama_grammar import LlamaGrammar +from ._utils import suppress_stdout_stderr import llama_cpp.llama_cpp as llama_cpp @@ -47,9 +48,10 @@ def __init__( if not os.path.exists(path_model): raise ValueError(f"Model path does not exist: {path_model}") - self.model = llama_cpp.llama_load_model_from_file( - self.path_model.encode("utf-8"), self.params - ) + with suppress_stdout_stderr(disable=verbose): + self.model = llama_cpp.llama_load_model_from_file( + self.path_model.encode("utf-8"), self.params + ) if self.model is None: raise ValueError(f"Failed to load model from file: {path_model}") diff --git a/llama_cpp/_utils.py b/llama_cpp/_utils.py index 4a106470b0..781b265010 100644 --- a/llama_cpp/_utils.py +++ b/llama_cpp/_utils.py @@ -1,13 +1,15 @@ import os import sys -import sys from typing import Any, Dict # Avoid "LookupError: unknown encoding: ascii" when open() called in a destructor outnull_file = open(os.devnull, "w") errnull_file = open(os.devnull, "w") +STDOUT_FILENO = 1 +STDERR_FILENO = 2 + class suppress_stdout_stderr(object): # NOTE: these must be "saved" here to avoid exceptions when using # this context manager inside of a __del__ method @@ -22,12 +24,8 @@ def __enter__(self): if self.disable: return self - # Check if sys.stdout and sys.stderr have fileno method - if not hasattr(self.sys.stdout, 'fileno') or not hasattr(self.sys.stderr, 'fileno'): - return self # Return the instance without making changes - - self.old_stdout_fileno_undup = self.sys.stdout.fileno() - self.old_stderr_fileno_undup = self.sys.stderr.fileno() + self.old_stdout_fileno_undup = STDOUT_FILENO + self.old_stderr_fileno_undup = STDERR_FILENO self.old_stdout_fileno = self.os.dup(self.old_stdout_fileno_undup) self.old_stderr_fileno = self.os.dup(self.old_stderr_fileno_undup) @@ -47,15 +45,14 @@ def __exit__(self, *_): return # Check if sys.stdout and sys.stderr have fileno method - if hasattr(self.sys.stdout, 'fileno') and hasattr(self.sys.stderr, 'fileno'): - self.sys.stdout = self.old_stdout - self.sys.stderr = self.old_stderr + self.sys.stdout = self.old_stdout + self.sys.stderr = self.old_stderr - self.os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup) - self.os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup) + self.os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup) + self.os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup) - self.os.close(self.old_stdout_fileno) - self.os.close(self.old_stderr_fileno) + self.os.close(self.old_stdout_fileno) + self.os.close(self.old_stderr_fileno) class MetaSingleton(type): From 946156fb6c09389cc184cb5209f96aac54c08b85 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 15:46:45 -0400 Subject: [PATCH 130/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 77e15bec62..f364eb6fb5 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 77e15bec6217a39be59b9cc83d6b9afb6b0d8167 +Subproject commit f364eb6fb5d46118a76fa045f487318de4c24961 From 4f01c452b6c738dc56eacac3758119b12c57ea94 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Apr 2024 15:50:30 -0400 Subject: [PATCH 131/561] fix: Change default verbose value of verbose in image chat format handlers to True to match Llama --- llama_cpp/llama_chat_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 63eaf8a03b..0af410a972 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2205,7 +2205,7 @@ class Llava15ChatHandler: "{% endif %}" ) - def __init__(self, clip_model_path: str, verbose: bool = False): + def __init__(self, clip_model_path: str, verbose: bool = True): import llama_cpp.llava_cpp as llava_cpp self.clip_model_path = clip_model_path From 31b1d95a6c19f5b615a3286069f181a415f872e8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 2 May 2024 11:32:18 -0400 Subject: [PATCH 132/561] feat: Add llama-3-vision-alpha chat format --- llama_cpp/llama_chat_format.py | 64 ++++++++++++++++++++++++++++++++-- llama_cpp/server/model.py | 14 ++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 0af410a972..a17b86b30a 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2165,7 +2165,7 @@ def create_completion(stop): class Llava15ChatHandler: - DEFAULT_SYSTEM_MESSAGE = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." + DEFAULT_SYSTEM_MESSAGE: Optional[str] = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." CHAT_FORMAT = ( "{% for message in messages %}" @@ -2288,7 +2288,7 @@ def __call__( assert self.clip_ctx is not None system_prompt = _get_system_message(messages) - if system_prompt == "": + if system_prompt == "" and self.DEFAULT_SYSTEM_MESSAGE is not None: messages = [llama_types.ChatCompletionRequestSystemMessage(role="system", content=self.DEFAULT_SYSTEM_MESSAGE)] + messages image_urls = self.get_image_urls(messages) @@ -2771,6 +2771,66 @@ class NanoLlavaChatHandler(Llava15ChatHandler): "{% endif %}" ) +class Llama3VisionAlpha(Llava15ChatHandler): + # question = "" + q + + # prompt = f"<|start_header_id|>user<|end_header_id|>\n\n{question}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" + DEFAULT_SYSTEM_MESSAGE = None + + CHAT_FORMAT = ( + "{% for message in messages %}" + + "<|start_header_id|>" + + "{% if message.role == 'user' %}" + + "user<|end_header_id|>\n\n" + + "{% if message.content is iterable %}" + + # + "{% for content in message.content %}" + "{% if content.type == 'image_url' %}" + "{% if content.image_url is string %}" + "{{ content.image_url }}" + "{% endif %}" + "{% if content.image_url is mapping %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + + # Question: + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + + "{% endif %}" + + # Question: + "{% if message.content is string %}" + "{{ message.content }}" + "{% endif %}" + + "{% endif %}" + + # Answer: + "{% if message.role == 'assistant' %}" + "assistant<|end_header_id|>\n\n" + "{{ message.content }}" + "{% endif %}" + + "<|eot_id|>" + + "{% endfor %}" + + # Generation prompt + "{% if add_generation_prompt %}" + "<|start_header_id|>assistant<|end_header_id|>\n\n" + "{% endif %}" + ) @register_chat_completion_handler("chatml-function-calling") def chatml_function_calling( diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 1ad592d211..e102fadbd7 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -140,6 +140,20 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler = llama_cpp.llama_chat_format.NanoLlavaChatHandler( clip_model_path=settings.clip_model_path, verbose=settings.verbose ) + elif settings.chat_format == "llama-3-vision-alpha": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.Llama3VisionAlpha.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.Llama3VisionAlpha( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) elif settings.chat_format == "hf-autotokenizer": assert ( settings.hf_pretrained_model_name_or_path is not None From d75dea18db61becae72784b9d750897b7cdcda26 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 2 May 2024 12:00:44 -0400 Subject: [PATCH 133/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f364eb6fb5..6ecf3189e0 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f364eb6fb5d46118a76fa045f487318de4c24961 +Subproject commit 6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 From 21171223967896b9e941007841ebd1513b3cc4b9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 2 May 2024 12:07:09 -0400 Subject: [PATCH 134/561] chore: Bump version --- CHANGELOG.md | 10 +++++++++- llama_cpp/__init__.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9681f355b..9b995509eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.69] + +- feat: Update llama.cpp to ggerganov/llama.cpp@6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 +- feat: Add llama-3-vision-alpha chat format by @abetlen in 31b1d95a6c19f5b615a3286069f181a415f872e8 +- fix: Change default verbose value of verbose in image chat format handlers to True to match Llama by @abetlen in 4f01c452b6c738dc56eacac3758119b12c57ea94 +- fix: Suppress all logs when verbose=False, use hardcoded fileno's to work in colab notebooks by @abetlen in f116175a5a7c84569c88cad231855c1e6e59ff6e +- fix: UTF-8 handling with grammars by @jsoma in #1415 + ## [0.2.68] -- feat: Update llama.cpp to ggerganov/llama.cpp@ +- feat: Update llama.cpp to ggerganov/llama.cpp@77e15bec6217a39be59b9cc83d6b9afb6b0d8167 - feat: Add option to enable flash_attn to Lllama params and ModelSettings by @abetlen in 22d77eefd2edaf0148f53374d0cac74d0e25d06e - fix(ci): Fix build-and-release.yaml by @Smartappli in #1413 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 63c622551c..95c8819660 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.68" \ No newline at end of file +__version__ = "0.2.69" \ No newline at end of file From 2138561fab5e60672c63b6c446b62a8bc26e17c4 Mon Sep 17 00:00:00 2001 From: Daniel Thuerck Date: Fri, 3 May 2024 18:17:07 +0200 Subject: [PATCH 135/561] fix(server): Propagate `flash_attn` to model load. (#1424) --- llama_cpp/server/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index e102fadbd7..f002924109 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -242,6 +242,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: logits_all=settings.logits_all, embedding=settings.embedding, offload_kqv=settings.offload_kqv, + flash_attn=settings.flash_attn, # Sampling Params last_n_tokens_size=settings.last_n_tokens_size, # LoRA Params From 0a454bebe67d12a446981eb16028c168ca5faa81 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 3 May 2024 15:23:06 -0400 Subject: [PATCH 136/561] feat(server): Remove temperature bounds checks for server. Closes #1384 --- llama_cpp/server/types.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/llama_cpp/server/types.py b/llama_cpp/server/types.py index ce9c87a694..a20b3940f2 100644 --- a/llama_cpp/server/types.py +++ b/llama_cpp/server/types.py @@ -18,8 +18,6 @@ temperature_field = Field( default=0.8, - ge=0.0, - le=2.0, description="Adjust the randomness of the generated text.\n\n" + "Temperature is a hyperparameter that controls the randomness of the generated text. It affects the probability distribution of the model's output tokens. A higher temperature (e.g., 1.5) makes the output more random and creative, while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative. The default value is 0.8, which provides a balance between randomness and determinism. At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.", ) From 9f7a85571ae80d3b6ddbd3e1bae407b9f1e3448a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 3 May 2024 19:07:50 -0400 Subject: [PATCH 137/561] fix: Use memmove to copy str_value kv_override. Closes #1417 --- llama_cpp/llama.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index f927f0ca26..17576a69b8 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -262,7 +262,12 @@ def __init__( raise ValueError(f"Value for {k} is too long: {v}") v_bytes = v_bytes.ljust(128, b"\0") self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR - self._kv_overrides_array[i].value.str_value[:128] = v_bytes + # copy min(v_bytes, 128) to str_value + ctypes.memmove( + self._kv_overrides_array[i].value.str_value, + v_bytes, + min(len(v_bytes), 128), + ) else: raise ValueError(f"Unknown value type for {k}: {v}") From 1f56c648c3a4aed2b4fe3edb4dc745fae3e7d8ae Mon Sep 17 00:00:00 2001 From: Jeffrey Fong Date: Sat, 4 May 2024 22:11:20 +0800 Subject: [PATCH 138/561] feat: Implement streaming for Functionary v2 + Bug fixes (#1419) * set up streaming for v2 * assert v2 streaming, fix tool_call vs function_call * fix streaming with tool_choice/function_call * make functions return 1 function call only when 'auto' * fix --------- Co-authored-by: Andrei --- llama_cpp/llama_chat_format.py | 576 +++++++++++++++++++++++++-------- 1 file changed, 443 insertions(+), 133 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index a17b86b30a..3ab94e0d3c 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1894,6 +1894,8 @@ def prepare_messages_for_inference( function_call = ( tool_choice if isinstance(tool_choice, str) else tool_choice["function"] ) + elif function_call is not None: + pass else: function_call = "auto" @@ -1930,11 +1932,10 @@ def prepare_messages_for_inference( logits_processor=logits_processor, grammar=grammar, ) - completion_or_completion_chunks["choices"][0]["text"] = completion_or_completion_chunks["choices"][0]["text"].lstrip() + if stream is False: + completion_or_completion_chunks["choices"][0]["text"] = completion_or_completion_chunks["choices"][0]["text"].lstrip() return _convert_completion_to_chat(completion_or_completion_chunks, stream=stream) # type: ignore - assert stream is False # TODO: support stream mode - def get_grammar(function_call): function_body = None for function in functions or []: @@ -1968,7 +1969,7 @@ def get_grammar(function_call): return grammar - def create_completion(stop): + def create_completion(prompt, stop, grammar): completion = cast(llama_types.Completion, llama.create_completion( prompt=prompt, temperature=temperature, @@ -1976,7 +1977,7 @@ def create_completion(stop): top_k=top_k, min_p=min_p, typical_p=typical_p, - stream=False, + stream=stream, stop=stop, max_tokens=max_tokens, presence_penalty=presence_penalty, @@ -1996,172 +1997,481 @@ def create_completion(stop): content = "" function_calls, function_bodies = [], [] completion_tokens = 0 - - if version == "v1": - # If no or "auto" tool_choice/function_call - if isinstance(function_call, str) and function_call == "auto": - stops = ["\n", END_ASSISTANT_TOKEN] - # If tool_choice/function_call is provided - elif isinstance(function_call, dict): - prompt += f"{START_FUNCTION_CALL_TOKEN}{function_call['name']}:\n" - stops = END_FUNCTION_CALL_TOKEN - function_call = function_call["name"] - function_calls.append(function_call) - grammar = get_grammar(function_call) - else: - prompt = prompt - stops = ["\n", END_ASSISTANT_TOKEN] - - completion = create_completion(stop=stops) - completion_text = completion["choices"][0]["text"] - completion_tokens += completion["usage"]["completion_tokens"] + + def generate_streaming(tools, functions, function_call, prompt): + assert version == "v2", "Streaming for v1 is not supported" + + chunk_id, chunk_created = None, None - - # If the generation does not involve a function call - if ( - START_FUNCTION_CALL_TOKEN not in prompt - and START_FUNCTION_CALL_TOKEN not in completion_text - ): - completion["usage"]["completion_tokens"] = completion_tokens - return _convert_completion_to_chat(completion, stream=stream) # type: ignore - # If the generation involves a function call in completion, generate the parameters - elif ( - START_FUNCTION_CALL_TOKEN not in prompt - and START_FUNCTION_CALL_TOKEN in completion_text - ): - prompt += ( - completion_text.replace( - f"{START_FUNCTION_CALL_TOKEN} ", START_FUNCTION_CALL_TOKEN - ) - + "\n" - ) - function_calls.append( - completion_text.split(START_FUNCTION_CALL_TOKEN)[-1][:-1].strip() - ) - grammar = get_grammar(function_calls[-1]) - completion = create_completion(stop=END_FUNCTION_CALL_TOKEN) - completion_tokens += completion["usage"]["completion_tokens"] - function_bodies.append(completion["choices"][0]["text"].strip()) - # If the prompt involves a function call, just append generated parameters to function_bodies - else: - function_bodies.append(completion_text.strip()) - else: # If tool_choice/function_call is provided if isinstance(function_call, dict): prompt += f"{function_call['name']}\n{CONTENT_TOKEN}" - function_call = function_call["name"] - function_calls.append(function_call) - grammar = get_grammar(function_call) + grammar = get_grammar(function_call["name"]) stops = [STOP_TOKEN, FROM_TOKEN] - completion = create_completion(stop=stops) - completion_text = completion["choices"][0]["text"] - completion_tokens += completion["usage"]["completion_tokens"] - function_bodies.append(completion_text.strip()) + tool_id = "".join([random.choice(string.ascii_letters + string.digits) for _ in range(24)]) + completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) + completion_text = "" + first = True + for chunk in completion: + # Yield the tool/function name first + if first: + if tools is not None: + func_call_dict = { + "tool_calls": [ + { + "index": 0, + "id": "call_" + tool_id, + "type": "function", + "function": {"name": function_call["name"], "arguments": ""}, + } + ] + } + else: + func_call_dict = {"function_call": {"name": function_call["name"], "arguments": ""}} + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk["id"], + object="chat.completion.chunk", + created=chunk["created"], + model=chunk["model"], + choices=[ + {"index": 0, "logprobs": None, "delta": {"role": None, "content": None, **func_call_dict}} + ], + ) + first = False + if tools is not None: + func_call_dict = { + "tool_calls": [ + { + "index": 0, + "id": "call_" + tool_id, + "type": "function", + "function": { + "name": None, + "arguments": chunk["choices"][0]["text"].rstrip(), + }, + } + ] + } + else: + func_call_dict = {"function_call": {"name": None, "arguments": chunk["choices"][0]["text"].rstrip()}} + if len(chunk["choices"][0]["text"].rstrip()) > 0: + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk["id"], + object="chat.completion.chunk", + created=chunk["created"], + model=chunk["model"], + choices=[ + { + "index": 0, + "logprobs": chunk["choices"][0]["logprobs"], + "delta": { + "role": None, + "content": None, + **func_call_dict, + }, + } + ], + ) + # Yield tool_call/function_call stop message + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk["id"], + object="chat.completion.chunk", + created=chunk["created"], + model=chunk["model"], + choices=[ + { + "index": 0, + "finish_reason": "tool_calls" if tools is not None else "function_call", + "logprobs": None, + "delta": { + "role": None, "content": None, "function_call": None, "tool_calls": None + }, + } + ], + ) # If "auto" or no tool_choice/function_call elif isinstance(function_call, str) and function_call == "auto": + tool_index = 0 while True: # Generate function name first grammar = None stops = CONTENT_TOKEN - completion = create_completion(stop=stops) - completion_text = completion["choices"][0]["text"] - completion_tokens += completion["usage"]["completion_tokens"] + completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) + completion_text = "" + for chunk in completion: + completion_text += chunk["choices"][0]["text"] + if chunk_id is None: + chunk_id = chunk["id"] + if chunk_created is None: + chunk_created = chunk["created"] function_name = completion_text.strip() if function_name == "all": prompt += "all\n<|content|>" + # Yield the first empty message for content + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk_id, + model=chunk["model"], + created=chunk_created, + object="chat.completion.chunk", + choices=[ + { + "index": 0, + "delta": {"role": "assistant", "content": ""}, + "logprobs": None, + "finish_reason": None, + } + ], + ) else: - function_call = completion_text.strip() - prompt += f"{function_call}\n<|content|>" - function_calls.append(function_call) - grammar = get_grammar(function_call) + prompt += f"{function_name}\n<|content|>" + grammar = get_grammar(function_name) + tool_id = "".join([random.choice(string.ascii_letters + string.digits) for _ in range(24)]) + if tools is not None: + func_call_dict = { + "tool_calls": [ + { + "index": tool_index, + "id": "call_" + tool_id, + "type": "function", + "function": {"name": function_name, "arguments": ""}, + } + ] + } + else: + func_call_dict = {"function_call": {"name": function_name, "arguments": ""}} + # Stream function name + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk_id, + object="chat.completion.chunk", + created=chunk_created, + model=chunk["model"], + choices=[ + { + "index": 0, + "logprobs": chunk["choices"][0]["logprobs"], + "delta": { + "role": "assistant", + "content": None, + **func_call_dict, + }, + } + ], + ) # Generate content stops = [RECIPIENT_TOKEN, STOP_TOKEN] - completion = create_completion(stop=stops) - completion_text = completion["choices"][0]["text"] - completion_tokens += completion["usage"]["completion_tokens"] + completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) if function_name == "all": - if completion_text.endswith("\n<|from|>assistant\n"): - content += completion_text[:-len("\n<|from|>assistant\n")] - if completion_text.endswith("\n<|from|> assistant\n"): - content += completion_text[-len("\n<|from|> assistant\n")] - else: - content += completion_text - content = content.lstrip() + completion_text = "" + stop_sequence, buffer, is_end = "\n<|from|>assistant\n<|recipient|>", [], False + for i, chunk in enumerate(completion): + completion_text += chunk["choices"][0]["text"] + if is_end: + buffer.append(chunk["choices"][0]["text"].strip(" ")) + if stop_sequence.startswith("".join(buffer)): + continue + else: + buffer.pop() + while len(buffer) > 0: + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk_id, + object="chat.completion.chunk", + created=chunk_created, + model=chunk["model"], + choices=[ + { + "index": 0, + "logprobs": chunk["choices"][0]["logprobs"], + "delta": { + "role": "assistant", "content": buffer.pop(0) + }, + } + ], + ) + is_end = False + elif chunk["choices"][0]["text"] == "\n": + is_end = True + buffer.append(chunk["choices"][0]["text"].strip(" ")) + continue + + if len(buffer) == 0 and len(chunk["choices"][0]["text"]) > 0: + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk_id, + object="chat.completion.chunk", + created=chunk_created, + model=chunk["model"], + choices=[ + { + "index": 0, + "logprobs": chunk["choices"][0]["logprobs"], + "delta": { + "role": "assistant", + "content": chunk["choices"][0]["text"] if i > 0 else chunk["choices"][0]["text"].lstrip() + }, + } + ], + ) # Check whether the model wants to generate another turn if "<|from|> assistant" in completion_text or "<|from|>assistant" in completion_text: if completion_text.endswith("\n<|from|>assistant\n"): cleaned_completion_text = completion_text[:-len("\n<|from|>assistant\n")].strip() elif completion_text.endswith("\n<|from|> assistant\n"): - cleaned_completion_text = completion_text[-len("\n<|from|> assistant\n")].strip() + cleaned_completion_text = completion_text[:-len("\n<|from|> assistant\n")].strip() else: cleaned_completion_text = completion_text.strip() prompt += f"{cleaned_completion_text}\n<|from|>assistant\n<|recipient|>" else: + # Yield stop message + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk_id, + model=chunk["model"], + created=chunk_created, + object="chat.completion.chunk", + choices=[ + { + "index": 0, + "delta": {}, + "logprobs": None, + "finish_reason": "stop", + } + ], + ) break else: - function_bodies.append(completion_text.strip()) # Check whether the model wants to generate another turn + completion_text = "" + for chunk in completion: + completion_text += chunk["choices"][0]["text"] + if len(chunk["choices"][0]["text"].rstrip()) > 0: + if tools is not None: + func_call_dict = { + "tool_calls": [ + { + "index": tool_index, + "id": "call_" + tool_id, + "type": "function", + "function": { + "name": None, + "arguments": chunk["choices"][0]["text"].rstrip(), + }, + } + ] + } + else: + func_call_dict = {"function_call": {"name": None, "arguments": chunk["choices"][0]["text"].rstrip()}} + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk_id, + object="chat.completion.chunk", + created=chunk_created, + model=chunk["model"], + choices=[ + { + "index": 0, + "logprobs": chunk["choices"][0]["logprobs"], + "delta": { + "role": None, + "content": None, + **func_call_dict, + }, + } + ], + ) prompt += completion_text.strip() grammar = None - completion = create_completion(stop=stops) - completion_tokens += completion["usage"]["completion_tokens"] - if "<|from|> assistant" in completion["choices"][0]["text"] or "<|from|>assistant" in completion["choices"][0]["text"]: + completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) + completion_text += "".join([chunk["choices"][0]["text"] for chunk in completion]) + if ("<|from|> assistant" in completion_text or "<|from|>assistant" in completion_text) and tools is not None: prompt += "\n<|from|>assistant\n<|recipient|>" + tool_index += 1 else: + # Yield tool_call/function_call stop message + yield llama_types.CreateChatCompletionStreamResponse( + id="chat" + chunk_id, + object="chat.completion.chunk", + created=chunk_created, + model=chunk["model"], + choices=[ + { + "index": 0, + "finish_reason": "tool_calls" if tools is not None else "function_call", + "logprobs": None, + "delta": { + "role": None, "content": None, "function_call": None, "tool_calls": None + }, + } + ], + ) break - - assert "usage" in completion - assert len(function_calls) == len(function_bodies) - - tool_calls: List[llama_types.ChatCompletionMessageToolCall] = [] - for function_call, function_body in zip(function_calls, function_bodies): - tool_calls.append( - { - "id": "call_" - + "".join( - [ - random.choice(string.ascii_letters + string.digits) - for _ in range(24) - ] - ), - "type": "function", - "function": { - "name": function_call, - "arguments": function_body, - }, - } + + if stream is not False: + return generate_streaming( + tools=tools, functions=functions, function_call=function_call, prompt=prompt ) + else: + if version == "v1": + # If no or "auto" tool_choice/function_call + if isinstance(function_call, str) and function_call == "auto": + stops = ["\n", END_ASSISTANT_TOKEN] + # If tool_choice/function_call is provided + elif isinstance(function_call, dict): + prompt += f"{START_FUNCTION_CALL_TOKEN}{function_call['name']}:\n" + stops = END_FUNCTION_CALL_TOKEN + function_call = function_call["name"] + function_calls.append(function_call) + grammar = get_grammar(function_call) + else: + prompt = prompt + stops = ["\n", END_ASSISTANT_TOKEN] - # TODO: support stream mode - function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = {} - if len(tool_calls) > 0: - if tools is not None: - function_call_dict["tool_calls"] = tool_calls + completion = create_completion(stop=stops) + completion_text = completion["choices"][0]["text"] + completion_tokens += completion["usage"]["completion_tokens"] + + + # If the generation does not involve a function call + if ( + START_FUNCTION_CALL_TOKEN not in prompt + and START_FUNCTION_CALL_TOKEN not in completion_text + ): + completion["usage"]["completion_tokens"] = completion_tokens + return _convert_completion_to_chat(completion, stream=stream) # type: ignore + # If the generation involves a function call in completion, generate the parameters + elif ( + START_FUNCTION_CALL_TOKEN not in prompt + and START_FUNCTION_CALL_TOKEN in completion_text + ): + prompt += ( + completion_text.replace( + f"{START_FUNCTION_CALL_TOKEN} ", START_FUNCTION_CALL_TOKEN + ) + + "\n" + ) + function_calls.append( + completion_text.split(START_FUNCTION_CALL_TOKEN)[-1][:-1].strip() + ) + grammar = get_grammar(function_calls[-1]) + completion = create_completion(stop=END_FUNCTION_CALL_TOKEN) + completion_tokens += completion["usage"]["completion_tokens"] + function_bodies.append(completion["choices"][0]["text"].strip()) + # If the prompt involves a function call, just append generated parameters to function_bodies + else: + function_bodies.append(completion_text.strip()) else: - function_call_dict["function_call"] = { - "name": tool_calls[0]["function"]["name"], - "arguments": tool_calls[0]["function"]["arguments"], - } - completion["usage"]["completion_tokens"] = completion_tokens - return llama_types.CreateChatCompletionResponse( - id="chat" + completion["id"], - object="chat.completion", - created=completion["created"], - model=completion["model"], - choices=[ - { - "index": 0, - "logprobs": completion["choices"][0]["logprobs"], - "message": { - "role": "assistant", - "content": None if content == "" else content, - **function_call_dict, - }, - "finish_reason": "tool_calls" if len(tool_calls) > 0 else "stop", - } - ], - usage=completion["usage"], - ) + # If tool_choice/function_call is provided + if isinstance(function_call, dict): + prompt += f"{function_call['name']}\n{CONTENT_TOKEN}" + function_call = function_call["name"] + function_calls.append(function_call) + grammar = get_grammar(function_call) + stops = [STOP_TOKEN, FROM_TOKEN] + completion = create_completion(stop=stops) + completion_text = completion["choices"][0]["text"] + completion_tokens += completion["usage"]["completion_tokens"] + function_bodies.append(completion_text.strip()) + # If "auto" or no tool_choice/function_call + elif isinstance(function_call, str) and function_call == "auto": + while True: + # Generate function name first + grammar = None + stops = CONTENT_TOKEN + completion = create_completion(stop=stops) + completion_text = completion["choices"][0]["text"] + completion_tokens += completion["usage"]["completion_tokens"] + function_name = completion_text.strip() + if function_name == "all": + prompt += "all\n<|content|>" + else: + function_call = completion_text.strip() + prompt += f"{function_call}\n<|content|>" + function_calls.append(function_call) + grammar = get_grammar(function_call) + # Generate content + stops = [RECIPIENT_TOKEN, STOP_TOKEN] + completion = create_completion(stop=stops) + completion_text = completion["choices"][0]["text"] + completion_tokens += completion["usage"]["completion_tokens"] + if function_name == "all": + if completion_text.endswith("\n<|from|>assistant\n"): + content += completion_text[:-len("\n<|from|>assistant\n")] + if completion_text.endswith("\n<|from|> assistant\n"): + content += completion_text[-len("\n<|from|> assistant\n")] + else: + content += completion_text + content = content.lstrip() + # Check whether the model wants to generate another turn + if "<|from|> assistant" in completion_text or "<|from|>assistant" in completion_text: + if completion_text.endswith("\n<|from|>assistant\n"): + cleaned_completion_text = completion_text[:-len("\n<|from|>assistant\n")].strip() + elif completion_text.endswith("\n<|from|> assistant\n"): + cleaned_completion_text = completion_text[-len("\n<|from|> assistant\n")].strip() + else: + cleaned_completion_text = completion_text.strip() + prompt += f"{cleaned_completion_text}\n<|from|>assistant\n<|recipient|>" + else: + break + else: + function_bodies.append(completion_text.strip()) + # Check whether the model wants to generate another turn + prompt += completion_text.strip() + grammar = None + completion = create_completion(stop=stops) + completion_tokens += completion["usage"]["completion_tokens"] + if "<|from|> assistant" in completion["choices"][0]["text"] or "<|from|>assistant" in completion["choices"][0]["text"]: + prompt += "\n<|from|>assistant\n<|recipient|>" + else: + break + + assert "usage" in completion + assert len(function_calls) == len(function_bodies) + + tool_calls: List[llama_types.ChatCompletionMessageToolCall] = [] + for function_call, function_body in zip(function_calls, function_bodies): + tool_calls.append( + { + "id": "call_" + + "".join( + [ + random.choice(string.ascii_letters + string.digits) + for _ in range(24) + ] + ), + "type": "function", + "function": { + "name": function_call, + "arguments": function_body, + }, + } + ) + + # TODO: support stream mode + function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = {} + if len(tool_calls) > 0: + if tools is not None: + function_call_dict["tool_calls"] = tool_calls + else: + function_call_dict["function_call"] = { + "name": tool_calls[0]["function"]["name"], + "arguments": tool_calls[0]["function"]["arguments"], + } + completion["usage"]["completion_tokens"] = completion_tokens + return llama_types.CreateChatCompletionResponse( + id="chat" + completion["id"], + object="chat.completion", + created=completion["created"], + model=completion["model"], + choices=[ + { + "index": 0, + "logprobs": completion["choices"][0]["logprobs"], + "message": { + "role": "assistant", + "content": None if content == "" else content, + **function_call_dict, + }, + "finish_reason": "tool_calls" if len(tool_calls) > 0 else "stop", + } + ], + usage=completion["usage"], + ) class Llava15ChatHandler: From e0d7674e62bdc5b906d2461238993ea3a022f61f Mon Sep 17 00:00:00 2001 From: Noam Gat Date: Sat, 4 May 2024 17:14:59 +0300 Subject: [PATCH 139/561] fix: detokenization case where first token does not start with a leading space (#1375) * Fix tokenization edge case where llama output does not start with a space See this notebook: https://colab.research.google.com/drive/1Ooz11nFPk19zyJdMDx42CeesU8aWZMdI#scrollTo=oKpHw5PZ30uC * Update _internals.py Fixing to compare to b' ' instead of (str)' ' --------- Co-authored-by: Andrei --- llama_cpp/_internals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index d7409f63a0..b404601d30 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -203,7 +203,7 @@ def detokenize(self, tokens: List[int], special: bool = False) -> bytes: # NOTE: Llama1 models automatically added a space at the start of the prompt # this line removes a leading space if the first token is a beginning of sentence token return ( - output[1:] if len(tokens) > 0 and tokens[0] == self.token_bos() else output + output[1:] if len(tokens) > 0 and tokens[0] == self.token_bos() and output[0:1] == b' ' else output ) # Extra @@ -812,4 +812,4 @@ def sample( def accept(self, ctx_main: _LlamaContext, id: int, apply_grammar: bool): if apply_grammar and self.grammar is not None: ctx_main.grammar_accept_token(self.grammar, id) - self.prev.append(id) \ No newline at end of file + self.prev.append(id) From 3e2597eac888e6e12c7bc7021016ca5104db83ba Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 5 May 2024 12:12:27 -0400 Subject: [PATCH 140/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 6 +++++- vendor/llama.cpp | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 46aa51662f..9e934e0526 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -294,6 +294,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_MPT = 5, # LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, # LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, +# LLAMA_VOCAB_PRE_TYPE_REFACT = 8, +# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -303,6 +305,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_MPT = 5 LLAMA_VOCAB_PRE_TYPE_STARCODER = 6 LLAMA_VOCAB_PRE_TYPE_GPT2 = 7 +LLAMA_VOCAB_PRE_TYPE_REFACT = 8 +LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9 # // note: these values should be synchronized with ggml_rope @@ -494,7 +498,7 @@ class llama_token_data_array(ctypes.Structure): llama_token_data_array_p = ctypes.POINTER(llama_token_data_array) -# typedef bool (*llama_progress_callback)(float progress, void *ctx); +# typedef bool (*llama_progress_callback)(float progress, void * user_data); llama_progress_callback = ctypes.CFUNCTYPE( ctypes.c_bool, ctypes.c_float, ctypes.c_void_p ) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6ecf3189e0..628b299106 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 +Subproject commit 628b299106d1e9476fdecb3cbe546bf5c60f1b89 From 36668331074c79f40494b51e4d541cd03601b9f2 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sun, 5 May 2024 18:42:28 +0200 Subject: [PATCH 141/561] feat(ci): Add docker checks and check deps more frequently (#1426) * Update dependabot.yml Add github-actions update * Update dependabot.yml * Update dependabot.yml --- .github/dependabot.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c58c9ae570..6bf90273ac 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,8 +8,12 @@ updates: - package-ecosystem: "pip" # See documentation for possible values directory: "/" # Location of package manifests schedule: - interval: "weekly" + interval: "daily" - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "daily" + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "daily" From 0318702cdc860999ee70f277425edbbfe0e60419 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 5 May 2024 12:49:31 -0400 Subject: [PATCH 142/561] feat(server): Add support for setting root_path. Closes #1420 --- llama_cpp/server/app.py | 1 + llama_cpp/server/settings.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index b6ed9b1b6b..4cf10d1f66 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -132,6 +132,7 @@ def create_app( middleware=middleware, title="🦙 llama.cpp Python API", version=llama_cpp.__version__, + root_path=server_settings.root_path, ) app.add_middleware( CORSMiddleware, diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index ed05a889f0..a3e185007d 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -215,6 +215,10 @@ class ServerSettings(BaseSettings): default=False, description="Disable EventSource pings (may be needed for some clients).", ) + root_path: str = Field( + default="", + description="The root path for the server. Useful when running behind a reverse proxy.", + ) class Settings(ServerSettings, ModelSettings): From a50d24e3a7ce92d5a5256738afd3a4450d1d85e7 Mon Sep 17 00:00:00 2001 From: Bruno Alvisio Date: Tue, 7 May 2024 23:19:35 -0700 Subject: [PATCH 143/561] fix: chat_format log where auto-detected format prints `None` (#1434) --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 17576a69b8..a8c3f9a98b 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -451,7 +451,7 @@ def __init__( if self.chat_format is None and self.chat_handler is None: self.chat_format = "llama-2" if self.verbose: - print(f"Using fallback chat format: {chat_format}", file=sys.stderr) + print(f"Using fallback chat format: {self.chat_format}", file=sys.stderr) @property def ctx(self) -> llama_cpp.llama_context_p: From 07966b9ba7afa8a7c0a04a54fc07b8984b131385 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 8 May 2024 15:20:20 +0900 Subject: [PATCH 144/561] docs: update README.md (#1432) accomodate -> accommodate --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 640671460c..5f3ff68246 100644 --- a/README.md +++ b/README.md @@ -516,7 +516,7 @@ chat_handler = Llava15ChatHandler(clip_model_path="path/to/llava/mmproj.bin") llm = Llama( model_path="./path/to/llava/llama-model.gguf", chat_handler=chat_handler, - n_ctx=2048, # n_ctx should be increased to accomodate the image embedding + n_ctx=2048, # n_ctx should be increased to accommodate the image embedding ) llm.create_chat_completion( messages = [ @@ -547,7 +547,7 @@ llm = Llama.from_pretrained( repo_id="vikhyatk/moondream2", filename="*text-model*", chat_handler=chat_handler, - n_ctx=2048, # n_ctx should be increased to accomodate the image embedding + n_ctx=2048, # n_ctx should be increased to accommodate the image embedding ) respoonse = llm.create_chat_completion( From 903b28adf5846b7a76999993c218adef4946c0fe Mon Sep 17 00:00:00 2001 From: Sarunas Kalade Date: Wed, 8 May 2024 07:21:27 +0100 Subject: [PATCH 145/561] fix: adding missing args in create_completion for functionary chat handler (#1430) --- llama_cpp/llama_chat_format.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 3ab94e0d3c..9da6b98000 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2322,7 +2322,7 @@ def generate_streaming(tools, functions, function_call, prompt): prompt = prompt stops = ["\n", END_ASSISTANT_TOKEN] - completion = create_completion(stop=stops) + completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) completion_text = completion["choices"][0]["text"] completion_tokens += completion["usage"]["completion_tokens"] @@ -2349,7 +2349,7 @@ def generate_streaming(tools, functions, function_call, prompt): completion_text.split(START_FUNCTION_CALL_TOKEN)[-1][:-1].strip() ) grammar = get_grammar(function_calls[-1]) - completion = create_completion(stop=END_FUNCTION_CALL_TOKEN) + completion = create_completion(prompt=prompt, stop=END_FUNCTION_CALL_TOKEN, grammar=grammar) completion_tokens += completion["usage"]["completion_tokens"] function_bodies.append(completion["choices"][0]["text"].strip()) # If the prompt involves a function call, just append generated parameters to function_bodies @@ -2363,7 +2363,7 @@ def generate_streaming(tools, functions, function_call, prompt): function_calls.append(function_call) grammar = get_grammar(function_call) stops = [STOP_TOKEN, FROM_TOKEN] - completion = create_completion(stop=stops) + completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) completion_text = completion["choices"][0]["text"] completion_tokens += completion["usage"]["completion_tokens"] function_bodies.append(completion_text.strip()) @@ -2373,7 +2373,7 @@ def generate_streaming(tools, functions, function_call, prompt): # Generate function name first grammar = None stops = CONTENT_TOKEN - completion = create_completion(stop=stops) + completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) completion_text = completion["choices"][0]["text"] completion_tokens += completion["usage"]["completion_tokens"] function_name = completion_text.strip() @@ -2386,7 +2386,7 @@ def generate_streaming(tools, functions, function_call, prompt): grammar = get_grammar(function_call) # Generate content stops = [RECIPIENT_TOKEN, STOP_TOKEN] - completion = create_completion(stop=stops) + completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) completion_text = completion["choices"][0]["text"] completion_tokens += completion["usage"]["completion_tokens"] if function_name == "all": @@ -2413,7 +2413,7 @@ def generate_streaming(tools, functions, function_call, prompt): # Check whether the model wants to generate another turn prompt += completion_text.strip() grammar = None - completion = create_completion(stop=stops) + completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) completion_tokens += completion["usage"]["completion_tokens"] if "<|from|> assistant" in completion["choices"][0]["text"] or "<|from|>assistant" in completion["choices"][0]["text"]: prompt += "\n<|from|>assistant\n<|recipient|>" @@ -3564,4 +3564,4 @@ def chatml_function_calling( }, } - raise ValueError("Automatic streaming tool choice is not supported") \ No newline at end of file + raise ValueError("Automatic streaming tool choice is not supported") From 228949c1f7e1f88c2e24393f45fa4156df7dc0cc Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 8 May 2024 02:22:15 -0400 Subject: [PATCH 146/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 2 ++ vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 9e934e0526..d8075e457c 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -296,6 +296,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, # LLAMA_VOCAB_PRE_TYPE_REFACT = 8, # LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, +# LLAMA_VOCAB_PRE_TYPE_OLMO = 10, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -307,6 +308,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_GPT2 = 7 LLAMA_VOCAB_PRE_TYPE_REFACT = 8 LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9 +LLAMA_VOCAB_PRE_TYPE_OLMO = 10 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 628b299106..c0e6fbf8c3 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 628b299106d1e9476fdecb3cbe546bf5c60f1b89 +Subproject commit c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1 From 4a7122d22f7a49bc0d055c1dbb892d489812ba10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Wed, 8 May 2024 08:26:22 +0200 Subject: [PATCH 147/561] feat: fill-in-middle support (#1386) * Proper fill-in-middle support Use prefix/middle/suffix tokens when metadata is present in GGUF, like f.ex. in [this](https://huggingface.co/CISCai/CodeQwen1.5-7B-Chat-SOTA-GGUF) one. * fall back to internal prefix/middle/suffix id In some cases llama.cpp will make a guess at fim tokens, use them if there's no metadata. * typo-- * don't insert special tokens that are not there in suffix Note: add_bos is misnamed, it's actually add_special and can cause several special tokens to be added to the token list (the special parameter is actually parse_special). * don't add/parse any special tokens when using fim I've left original behavior when no fim tokens are found, but this should perhaps be re-evaluated. * don't append suffix to prompt_tokens unless fim tokens are detected * make sure we only do this for fim --------- Co-authored-by: Andrei --- llama_cpp/llama.py | 47 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index a8c3f9a98b..5acc112d11 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -955,18 +955,53 @@ def _create_completion( completion_id: str = f"cmpl-{str(uuid.uuid4())}" created: int = int(time.time()) + prefix_token_id: int = int(self.metadata.get("tokenizer.ggml.prefix_token_id", self._model.token_prefix())) + middle_token_id: int = int(self.metadata.get("tokenizer.ggml.middle_token_id", self._model.token_middle())) + suffix_token_id: int = int(self.metadata.get("tokenizer.ggml.suffix_token_id", self._model.token_suffix())) # If prompt is empty, initialize completion with BOS token to avoid # detokenization including a space at the beginning of the completion completion_tokens: List[int] = [] if len(prompt) > 0 else [self.token_bos()] # Add blank space to start of prompt to match OG llama tokenizer prompt_tokens: List[int] = ( ( - self.tokenize(prompt.encode("utf-8"), special=True) - if prompt != "" - else [self.token_bos()] + [prefix_token_id] + if prefix_token_id >= 0 and suffix is not None + else [] + ) + + + ( + ( + self.tokenize(prompt.encode("utf-8"), add_bos=(prefix_token_id < 0 or suffix is None), special=(prefix_token_id < 0 or suffix is None)) + if prompt != "" + else ( + [] + if prefix_token_id >= 0 and suffix is not None + else [self.token_bos()] + ) + ) + if isinstance(prompt, str) + else prompt + ) + + + ( + ( + [suffix_token_id] + + + ( + self.tokenize(suffix.encode("utf-8"), add_bos=False, special=False) + if suffix + else [] + ) + ) + if suffix_token_id >= 0 and suffix is not None + else [] + ) + + + ( + [middle_token_id] + if middle_token_id >= 0 and suffix is not None + else [] ) - if isinstance(prompt, str) - else prompt ) text: bytes = b"" returned_tokens: int = 0 @@ -1346,7 +1381,7 @@ def logit_bias_processor( if echo: text_str = prompt + text_str - if suffix is not None: + if suffix_token_id < 0 and suffix is not None: text_str = text_str + suffix logprobs_or_none: Optional[CompletionLogprobs] = None From 9ce5cb376a12a56028aec1fd3b0edc55949b996f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 8 May 2024 02:36:42 -0400 Subject: [PATCH 148/561] chore: Bump version --- CHANGELOG.md | 15 +++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b995509eb..911d7fbea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.70] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ +- feat: fill-in-middle support by @CISC in #1386 +- fix: adding missing args in create_completion for functionary chat handler by @skalade in #1430 +- docs: update README.md @eltociear in #1432 +- fix: chat_format log where auto-detected format prints None by @balvisio in #1434 +- feat(server): Add support for setting root_path by @abetlen in 0318702cdc860999ee70f277425edbbfe0e60419 +- feat(ci): Add docker checks and check deps more frequently by @Smartappli in #1426 +- fix: detokenization case where first token does not start with a leading space by @noamgat in #1375 +- feat: Implement streaming for Functionary v2 + Bug fixes by @jeffrey-fong in #1419 +- fix: Use memmove to copy str_value kv_override by @abetlen in 9f7a85571ae80d3b6ddbd3e1bae407b9f1e3448a +- feat(server): Remove temperature bounds checks for server by @abetlen in 0a454bebe67d12a446981eb16028c168ca5faa81 +- fix(server): Propagate flash_attn to model load by @dthuerck in #1424 + ## [0.2.69] - feat: Update llama.cpp to ggerganov/llama.cpp@6ecf3189e00a1e8e737a78b6d10e1d7006e050a2 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 95c8819660..6a36717fd7 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.69" \ No newline at end of file +__version__ = "0.2.70" \ No newline at end of file From 2a39b9957508c46b49caf83eba3925964545fc2f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 8 May 2024 08:42:22 -0400 Subject: [PATCH 149/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 11 +++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d8075e457c..461122a44f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -296,7 +296,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, # LLAMA_VOCAB_PRE_TYPE_REFACT = 8, # LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, -# LLAMA_VOCAB_PRE_TYPE_OLMO = 10, +# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 10, +# LLAMA_VOCAB_PRE_TYPE_OLMO = 11, +# LLAMA_VOCAB_PRE_TYPE_DBRX = 12, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -308,7 +310,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_GPT2 = 7 LLAMA_VOCAB_PRE_TYPE_REFACT = 8 LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9 -LLAMA_VOCAB_PRE_TYPE_OLMO = 10 +LLAMA_VOCAB_PRE_TYPE_QWEN2 = 10 +LLAMA_VOCAB_PRE_TYPE_OLMO = 11 +LLAMA_VOCAB_PRE_TYPE_DBRX = 12 # // note: these values should be synchronized with ggml_rope @@ -377,6 +381,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors # LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors # LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -409,6 +414,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_FTYPE_MOSTLY_IQ2_S = 28 LLAMA_FTYPE_MOSTLY_IQ2_M = 29 LLAMA_FTYPE_MOSTLY_IQ4_XS = 30 +LLAMA_FTYPE_MOSTLY_IQ1_M = 31 +LLAMA_FTYPE_MOSTLY_BF16 = 32 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c0e6fbf8c3..911b3900dd 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1 +Subproject commit 911b3900dded9a1cfe0f0e41b82c7a29baf3a217 From 77122638b4153e31d9f277b3d905c2900b536632 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 8 May 2024 13:12:31 -0400 Subject: [PATCH 150/561] fix: Make leading bos_token optional for image chat formats, fix nanollava system message --- llama_cpp/llama_chat_format.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 9da6b98000..beff6ca9cf 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2603,7 +2603,12 @@ def __call__( image_urls = self.get_image_urls(messages) template = jinja2.Template(self.CHAT_FORMAT) - text = template.render(messages=messages, add_generation_prompt=True) + text = template.render( + messages=messages, + add_generation_prompt=True, + eos_token=llama.detokenize([llama.token_eos()]), + bos_token=llama.detokenize([llama.token_bos()]), + ) split_text = self.split_text_on_image_urls(text, image_urls) def embed_image_bytes(image_bytes: bytes): @@ -2624,9 +2629,9 @@ def embed_image_bytes(image_bytes: bytes): # Evaluate prompt llama.reset() - for i, (type_, value) in enumerate(split_text): + for type_, value in split_text: if type_ == "text": - tokens = llama.tokenize(value.encode("utf8"), add_bos=i == 0) + tokens = llama.tokenize(value.encode("utf8"), add_bos=False, special=True) if llama.n_tokens + len(tokens) > llama.n_ctx(): raise ValueError("Prompt exceeds n_ctx") # TODO: Fix llama.eval(tokens) @@ -2644,6 +2649,8 @@ def embed_image_bytes(image_bytes: bytes): llama.n_batch, n_past_p, ) + # Required to avoid issues with hf tokenizer + llama.input_ids[llama.n_tokens : n_past.value] = -1 llama.n_tokens = n_past.value # Get prompt tokens to avoid a cache miss @@ -3033,6 +3040,7 @@ class NanoLlavaChatHandler(Llava15ChatHandler): # Answer the question<|im_end|><|im_start|>user # # What is the picture about?<|im_end|><|im_start|>assistant + DEFAULT_SYSTEM_MESSAGE = "Answer the question" CHAT_FORMAT = ( "{% for message in messages %}" From 3757328b703b2cd32dcbd5853271e3a8c8599fe7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 8 May 2024 22:16:18 -0400 Subject: [PATCH 151/561] fix: free last image embed in llava chat handler --- llama_cpp/llama_chat_format.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index beff6ca9cf..84de989a5b 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2615,6 +2615,11 @@ def embed_image_bytes(image_bytes: bytes): if self._last_image_embed is not None and self._last_image_hash is not None and hash(image_bytes) == self._last_image_hash: return self._last_image_embed with suppress_stdout_stderr(disable=self.verbose): + # Free the previous image embed + if self._last_image_embed is not None: + self._llava_cpp.llava_image_embed_free(self._last_image_embed) + self._last_image_embed = None + self._last_image_hash = None embed = ( self._llava_cpp.llava_image_embed_make_with_bytes( self.clip_ctx, From bf66a283e899e367c82bdf13b89179fb14ed176e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 9 May 2024 03:02:52 -0400 Subject: [PATCH 152/561] chore: Bump version --- CHANGELOG.md | 8 +++++++- llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 911d7fbea0..ec70b6d6d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.71] + +- feat: Update llama.cpp to ggerganov/llama.cpp@911b3900dded9a1cfe0f0e41b82c7a29baf3a217 +- fix: Make leading bos_token optional for image chat formats, fix nanollava system message by @abetlen in 77122638b4153e31d9f277b3d905c2900b536632 +- fix: free last image embed in llava chat handler by @abetlen in 3757328b703b2cd32dcbd5853271e3a8c8599fe7 + ## [0.2.70] -- feat: Update llama.cpp to ggerganov/llama.cpp@ +- feat: Update llama.cpp to ggerganov/llama.cpp@c0e6fbf8c380718102bd25fcb8d2e55f8f9480d1 - feat: fill-in-middle support by @CISC in #1386 - fix: adding missing args in create_completion for functionary chat handler by @skalade in #1430 - docs: update README.md @eltociear in #1432 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 6a36717fd7..c77d2a95a1 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.70" \ No newline at end of file +__version__ = "0.2.71" \ No newline at end of file From 5ab40e6167e64ecb06c2a4d8ae798391bf9a5893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Thu, 9 May 2024 15:49:09 +0200 Subject: [PATCH 153/561] feat: Support multiple chat templates - step 1 (#1396) * Support multiple chat templates - step 1 As a first step, allow user to to select template from metadata with chat_format parameter in the form of `chat_template.name`. * register chat templates to self.chat_formats instead of globally * Don't expose internal chat handlers yet --------- Co-authored-by: Andrei --- llama_cpp/llama.py | 50 ++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 5acc112d11..4212669ebe 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -378,6 +378,7 @@ def __init__( self.chat_format = chat_format self.chat_handler = chat_handler + self._chat_handlers: Dict[str, llama_chat_format.LlamaChatCompletionHandler] = {} self.draft_model = draft_model @@ -409,10 +410,33 @@ def __init__( if self.verbose: print(f"Model metadata: {self.metadata}", file=sys.stderr) + eos_token_id = int(self.metadata.get("tokenizer.ggml.eos_token_id", self.token_eos())) + bos_token_id = int(self.metadata.get("tokenizer.ggml.bos_token_id", self.token_bos())) + + eos_token = self._model.token_get_text(eos_token_id) + bos_token = self._model.token_get_text(bos_token_id) + + # Unfortunately the llama.cpp API does not return metadata arrays, so we can't get template names from tokenizer.chat_templates + template_choices = dict((name[10:], template) for name, template in self.metadata.items() if name.startswith("tokenizer.chat_template.")) + + if "tokenizer.chat_template" in self.metadata: + template_choices["chat_template.default"] = self.metadata["tokenizer.chat_template"] + + if self.verbose and template_choices: + print(f"Available chat formats from metadata: {', '.join(template_choices.keys())}", file=sys.stderr) + + for name, template in template_choices.items(): + self._chat_handlers[name] = llama_chat_format.Jinja2ChatFormatter( + template=template, + eos_token=eos_token, + bos_token=bos_token, + stop_token_ids=[eos_token_id], + ).to_chat_handler() + if ( self.chat_format is None and self.chat_handler is None - and "tokenizer.chat_template" in self.metadata + and "chat_template.default" in template_choices ): chat_format = llama_chat_format.guess_chat_format_from_gguf_metadata( self.metadata @@ -423,30 +447,12 @@ def __init__( if self.verbose: print(f"Guessed chat format: {chat_format}", file=sys.stderr) else: - template = self.metadata["tokenizer.chat_template"] - try: - eos_token_id = int(self.metadata["tokenizer.ggml.eos_token_id"]) - except: - eos_token_id = self.token_eos() - try: - bos_token_id = int(self.metadata["tokenizer.ggml.bos_token_id"]) - except: - bos_token_id = self.token_bos() - - eos_token = self._model.token_get_text(eos_token_id) - bos_token = self._model.token_get_text(bos_token_id) - if self.verbose: - print(f"Using gguf chat template: {template}", file=sys.stderr) + print(f"Using gguf chat template: {template_choices['chat_template.default']}", file=sys.stderr) print(f"Using chat eos_token: {eos_token}", file=sys.stderr) print(f"Using chat bos_token: {bos_token}", file=sys.stderr) - self.chat_handler = llama_chat_format.Jinja2ChatFormatter( - template=template, - eos_token=eos_token, - bos_token=bos_token, - stop_token_ids=[eos_token_id], - ).to_chat_handler() + self.chat_format = "chat_template.default" if self.chat_format is None and self.chat_handler is None: self.chat_format = "llama-2" @@ -1719,7 +1725,7 @@ def create_chat_completion( Returns: Generated chat completion or a stream of chat completion chunks. """ - handler = self.chat_handler or llama_chat_format.get_chat_completion_handler( + handler = self.chat_handler or self._chat_handlers.get(self.chat_format) or llama_chat_format.get_chat_completion_handler( self.chat_format ) return handler( From b454f40a9a1787b2b5659cd2cb00819d983185df Mon Sep 17 00:00:00 2001 From: Patrick Peng Date: Fri, 10 May 2024 12:47:56 +0800 Subject: [PATCH 154/561] Merge pull request from GHSA-56xg-wfcc-g829 Co-authored-by: Andrei --- llama_cpp/llama_chat_format.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 84de989a5b..e0c59c9f11 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -11,6 +11,7 @@ from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Union, Protocol, cast import jinja2 +from jinja2.sandbox import ImmutableSandboxedEnvironment import numpy as np import numpy.typing as npt @@ -191,7 +192,7 @@ def __init__( self.add_generation_prompt = add_generation_prompt self.stop_token_ids = set(stop_token_ids) if stop_token_ids is not None else None - self._environment = jinja2.Environment( + self._environment = ImmutableSandboxedEnvironment( loader=jinja2.BaseLoader(), trim_blocks=True, lstrip_blocks=True, From 561e880654aa4e1311b22577e9149962be0d8ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Fri, 10 May 2024 06:49:40 +0200 Subject: [PATCH 155/561] fix(security): Render all jinja templates in immutable sandbox (#1441) Chat templates are rendered with ImmutableSandboxedEnvironment in transformers so no need to do otherwise here. Co-authored-by: Andrei --- llama_cpp/llama_chat_format.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index e0c59c9f11..d795b30490 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -685,8 +685,7 @@ def hf_tokenizer_config_to_chat_formatter( assert isinstance(tokenizer_config["eos_token"], str) eos_token = tokenizer_config["eos_token"] - env = jinja2.Environment( - loader=jinja2.BaseLoader(), + env = ImmutableSandboxedEnvironment( trim_blocks=True, lstrip_blocks=True, ).from_string(chat_template) @@ -2603,7 +2602,10 @@ def __call__( messages = [llama_types.ChatCompletionRequestSystemMessage(role="system", content=self.DEFAULT_SYSTEM_MESSAGE)] + messages image_urls = self.get_image_urls(messages) - template = jinja2.Template(self.CHAT_FORMAT) + template = ImmutableSandboxedEnvironment( + trim_blocks=True, + lstrip_blocks=True, + ).from_string(self.CHAT_FORMAT) text = template.render( messages=messages, add_generation_prompt=True, @@ -3242,8 +3244,7 @@ def chatml_function_calling( "{% endfor %}" "{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" ) - template_renderer = jinja2.Environment( - loader=jinja2.BaseLoader(), + template_renderer = ImmutableSandboxedEnvironment( autoescape=jinja2.select_autoescape(["html", "xml"]), undefined=jinja2.StrictUndefined, ).from_string(function_calling_template) From 4badac3a60c6cef16d1249cae6b0c2af94cb9415 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 May 2024 00:56:19 -0400 Subject: [PATCH 156/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec70b6d6d8..578d08d699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.72] + +- fix(security): Remote Code Execution by Server-Side Template Injection in Model Metadata by @retr0reg in b454f40a9a1787b2b5659cd2cb00819d983185df +- fix(security): Update remaining jinja chat templates to use immutable sandbox by @CISC in #1441 + ## [0.2.71] - feat: Update llama.cpp to ggerganov/llama.cpp@911b3900dded9a1cfe0f0e41b82c7a29baf3a217 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index c77d2a95a1..0165379ae3 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.71" \ No newline at end of file +__version__ = "0.2.72" \ No newline at end of file From ac55d0a175115d1e719672ce1cb1bec776c738b1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 May 2024 02:38:10 -0400 Subject: [PATCH 157/561] fix: Clear kv cache to avoid kv bug when image is evaluated first --- llama_cpp/llama_chat_format.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index d795b30490..e2d7e27855 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2637,6 +2637,7 @@ def embed_image_bytes(image_bytes: bytes): # Evaluate prompt llama.reset() + llama._ctx.kv_cache_clear() for type_, value in split_text: if type_ == "text": tokens = llama.tokenize(value.encode("utf8"), add_bos=False, special=True) From eafb6ec5e8b4540e0b58e44b4deaaf592eb5014a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 May 2024 08:39:55 -0400 Subject: [PATCH 158/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 911b3900dd..25c6e82e7a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 911b3900dded9a1cfe0f0e41b82c7a29baf3a217 +Subproject commit 25c6e82e7a1ad25a42b0894e87d9b5c557409516 From 73165021bbe888d58ce1adc66a2bcd6a0ecf10dd Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 May 2024 09:44:18 -0400 Subject: [PATCH 159/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 578d08d699..9f36b44f14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.73] + +- feat: Update llama.cpp to ggerganov/llama.cpp@25c6e82e7a1ad25a42b0894e87d9b5c557409516 +- fix: Clear kv cache at beginning of image chat formats to avoid bug when image is evaluated first by @abetlen in ac55d0a175115d1e719672ce1cb1bec776c738b1 + ## [0.2.72] - fix(security): Remote Code Execution by Server-Side Template Injection in Model Metadata by @retr0reg in b454f40a9a1787b2b5659cd2cb00819d983185df diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 0165379ae3..d6e8d0bc9f 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.72" \ No newline at end of file +__version__ = "0.2.73" \ No newline at end of file From 7f59856fa6f3e23f07e12fc15aeb9359dc6c3bb4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 10 May 2024 10:18:47 -0400 Subject: [PATCH 160/561] fix: Enable CUDA backend for llava. Closes #1324 --- CMakeLists.txt | 3 ++- Makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 70f9b99936..dcaddb678e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,8 +51,9 @@ if (LLAMA_BUILD) ) if (LLAVA_BUILD) - if (LLAMA_CUBLAS) + if (LLAMA_CUBLAS OR LLAMA_CUDA) add_compile_definitions(GGML_USE_CUBLAS) + add_compile_definitions(GGML_USE_CUDA) endif() if (LLAMA_METAL) diff --git a/Makefile b/Makefile index 4ae0110749..83085d2706 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ build.debug: CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug" python3 -m pip install --verbose --config-settings=cmake.verbose=true --config-settings=logging.level=INFO --config-settings=install.strip=false --editable . build.cuda: - CMAKE_ARGS="-DLLAMA_CUBLAS=on" python3 -m pip install --verbose -e . + CMAKE_ARGS="-DLLAMA_CUDA=on" python3 -m pip install --verbose -e . build.opencl: CMAKE_ARGS="-DLLAMA_CLBLAST=on" python3 -m pip install --verbose -e . From 1547202b77c0932e36b6b3b8319c54d62ac2d127 Mon Sep 17 00:00:00 2001 From: Peng Yu Date: Fri, 10 May 2024 10:35:51 -0400 Subject: [PATCH 161/561] docs: Fix typo in README.md (#1444) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f3ff68246..d5ced86068 100644 --- a/README.md +++ b/README.md @@ -550,7 +550,7 @@ llm = Llama.from_pretrained( n_ctx=2048, # n_ctx should be increased to accommodate the image embedding ) -respoonse = llm.create_chat_completion( +response = llm.create_chat_completion( messages = [ { "role": "user", From 9dc5e20fb6b0992b4eab6608b36548346ac3eb25 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 12 May 2024 10:30:23 -0400 Subject: [PATCH 162/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 911b3900dd..b228aba91a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 911b3900dded9a1cfe0f0e41b82c7a29baf3a217 +Subproject commit b228aba91ac2cd9eb90e9d423ba1d0d20e0117e2 From 3c19faa0d4dc249e5dfdd9e3b79fe7d911ea0bfc Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 12 May 2024 10:32:52 -0400 Subject: [PATCH 163/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f36b44f14..2782be0234 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.74] + +- feat: Update llama.cpp to ggerganov/llama.cpp@b228aba91ac2cd9eb90e9d423ba1d0d20e0117e2 +- fix: Enable CUDA backend for llava by @abetlen in 7f59856fa6f3e23f07e12fc15aeb9359dc6c3bb4 +- docs: Fix typo in README.md by @yupbank in #1444 + ## [0.2.73] - feat: Update llama.cpp to ggerganov/llama.cpp@25c6e82e7a1ad25a42b0894e87d9b5c557409516 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index d6e8d0bc9f..b7b8e47c66 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.73" \ No newline at end of file +__version__ = "0.2.74" \ No newline at end of file From 3f8e17af63db75116bb5a93b2d12cf3407e8561e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 12 May 2024 11:45:55 -0400 Subject: [PATCH 164/561] fix(ci): Use version without extra platform tag in pep503 index --- scripts/releases-to-pep-503.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/releases-to-pep-503.sh b/scripts/releases-to-pep-503.sh index 00ae56721e..44fbbf3cf1 100755 --- a/scripts/releases-to-pep-503.sh +++ b/scripts/releases-to-pep-503.sh @@ -44,7 +44,9 @@ releases=$(echo $releases | tr ' ' '\n' | grep -E $pattern) # For each release, get all assets for release in $releases; do assets=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release | jq -r .assets) - echo "

$release

" >> index.html + # Get release version from release ie v0.1.0-cu121 -> v0.1.0 + release_version=$(echo $release | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+") + echo "

$release_version

" >> index.html for asset in $(echo $assets | jq -r .[].browser_download_url); do if [[ $asset == *".whl" ]]; then echo " $asset" >> index.html From 43ba1526c8e4a903ea060c00c4dcc1ad12227a91 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 13 May 2024 09:39:08 -0400 Subject: [PATCH 165/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b228aba91a..1c570d8bee 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b228aba91ac2cd9eb90e9d423ba1d0d20e0117e2 +Subproject commit 1c570d8beeebad95872dc738ea542a4a0022f78a From 50f5c74ecf0fe53e13187c2572487a7e7ca344fd Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 14 May 2024 09:30:04 -0400 Subject: [PATCH 166/561] Update llama.cpp --- llama_cpp/llama_cpp.py | 6 ++++++ vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 461122a44f..8da3f140cd 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -648,6 +648,9 @@ class llama_model_kv_override(ctypes.Structure): # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() # const float * tensor_split; +# // comma separated list of RPC servers to use for offloading +# const char * rpc_servers; + # // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. # // If the provided progress_callback returns true, model loading continues. # // If it returns false, model loading is immediately aborted. @@ -674,6 +677,7 @@ class llama_model_params(ctypes.Structure): split_mode (int): how to split the model across multiple GPUs main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() + rpc_servers (ctypes.c_char_p): comma separated list of RPC servers to use for offloading progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data @@ -687,6 +691,7 @@ class llama_model_params(ctypes.Structure): split_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] + rpc_servers: ctypes.c_char_p progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] @@ -700,6 +705,7 @@ class llama_model_params(ctypes.Structure): ("split_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), + ("rpc_servers", ctypes.c_char_p), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1c570d8bee..1265c670fd 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1c570d8beeebad95872dc738ea542a4a0022f78a +Subproject commit 1265c670fd8e41e1947352c96c5179adda97fb2c From 4b54f7933050cbe9107fd949008185f92cf7de8f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 May 2024 09:35:52 -0400 Subject: [PATCH 167/561] chore(deps): bump pypa/cibuildwheel from 2.17.0 to 2.18.0 (#1453) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.17.0 to 2.18.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.17.0...v2.18.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 8fbd68f4ad..b50da48458 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -29,7 +29,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.18.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -56,7 +56,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.18.0 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" From 389e09c2f55a9ac659fd64fada56cf750a7a378b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 14 May 2024 15:44:09 +0200 Subject: [PATCH 168/561] misc: Remove unnecessary metadata lookups (#1448) Special tokens are already mapped from metadata by llama.cpp --- llama_cpp/llama.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 4212669ebe..281ee79182 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -410,8 +410,8 @@ def __init__( if self.verbose: print(f"Model metadata: {self.metadata}", file=sys.stderr) - eos_token_id = int(self.metadata.get("tokenizer.ggml.eos_token_id", self.token_eos())) - bos_token_id = int(self.metadata.get("tokenizer.ggml.bos_token_id", self.token_bos())) + eos_token_id = self.token_eos() + bos_token_id = self.token_bos() eos_token = self._model.token_get_text(eos_token_id) bos_token = self._model.token_get_text(bos_token_id) @@ -961,9 +961,9 @@ def _create_completion( completion_id: str = f"cmpl-{str(uuid.uuid4())}" created: int = int(time.time()) - prefix_token_id: int = int(self.metadata.get("tokenizer.ggml.prefix_token_id", self._model.token_prefix())) - middle_token_id: int = int(self.metadata.get("tokenizer.ggml.middle_token_id", self._model.token_middle())) - suffix_token_id: int = int(self.metadata.get("tokenizer.ggml.suffix_token_id", self._model.token_suffix())) + prefix_token_id: int = self._model.token_prefix() + middle_token_id: int = self._model.token_middle() + suffix_token_id: int = self._model.token_suffix() # If prompt is empty, initialize completion with BOS token to avoid # detokenization including a space at the beginning of the completion completion_tokens: List[int] = [] if len(prompt) > 0 else [self.token_bos()] From 5212fb08ae69a721b2ced4e5e8b96ce642219e16 Mon Sep 17 00:00:00 2001 From: twaka Date: Tue, 14 May 2024 22:50:53 +0900 Subject: [PATCH 169/561] feat: add MinTokensLogitProcessor and min_tokens argument to server (#1333) * implement min_tokens * set default to 0 * pass min_tokens * fix * remove copy * implement MinTokensLogitsProcessor * format * fix condition --- llama_cpp/llama.py | 16 ++++++++++++++++ llama_cpp/server/app.py | 20 ++++++++++++++++++++ llama_cpp/server/types.py | 8 ++++++++ 3 files changed, 44 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 281ee79182..7145329450 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2084,3 +2084,19 @@ def __call__( self, input_ids: npt.NDArray[np.intc], logits: npt.NDArray[np.single] ) -> bool: return any([stopping_criteria(input_ids, logits) for stopping_criteria in self]) + + +class MinTokensLogitsProcessor(LogitsProcessor): + def __init__(self, min_tokens: int, token_eos: int): + self.min_tokens = min_tokens + self.token_eos = token_eos + self.prompt_tokens = None + + def __call__( + self, input_ids: npt.NDArray[np.intc], scores: npt.NDArray[np.single] + ) -> npt.NDArray[np.single]: + if self.prompt_tokens is None: + self.prompt_tokens = len(input_ids) + if len(input_ids) - self.prompt_tokens < self.min_tokens: + scores[self.token_eos] = -np.inf + return scores diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 4cf10d1f66..4cda4af7a7 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -275,6 +275,7 @@ async def create_completion( "best_of", "logit_bias_type", "user", + "min_tokens", } kwargs = body.model_dump(exclude=exclude) @@ -288,6 +289,15 @@ async def create_completion( if body.grammar is not None: kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) + if body.min_tokens > 0: + _min_tokens_logits_processor = llama_cpp.LogitsProcessorList( + [llama_cpp.MinTokensLogitsProcessor(body.min_tokens, llama.token_eos())] + ) + if "logits_processor" not in kwargs: + kwargs["logits_processor"] = _min_tokens_logits_processor + else: + kwargs["logits_processor"].extend(_min_tokens_logits_processor) + iterator_or_completion: Union[ llama_cpp.CreateCompletionResponse, Iterator[llama_cpp.CreateCompletionStreamResponse], @@ -445,6 +455,7 @@ async def create_chat_completion( "n", "logit_bias_type", "user", + "min_tokens", } kwargs = body.model_dump(exclude=exclude) llama = llama_proxy(body.model) @@ -458,6 +469,15 @@ async def create_chat_completion( if body.grammar is not None: kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) + if body.min_tokens > 0: + _min_tokens_logits_processor = llama_cpp.LogitsProcessorList( + [llama_cpp.MinTokensLogitsProcessor(body.min_tokens, llama.token_eos())] + ) + if "logits_processor" not in kwargs: + kwargs["logits_processor"] = _min_tokens_logits_processor + else: + kwargs["logits_processor"].extend(_min_tokens_logits_processor) + iterator_or_completion: Union[ llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) diff --git a/llama_cpp/server/types.py b/llama_cpp/server/types.py index a20b3940f2..a75f9e55b9 100644 --- a/llama_cpp/server/types.py +++ b/llama_cpp/server/types.py @@ -16,6 +16,12 @@ default=16, ge=1, description="The maximum number of tokens to generate." ) +min_tokens_field = Field( + default=0, + ge=0, + description="The minimum number of tokens to generate. It may return fewer tokens if another condition is met (e.g. max_tokens, stop).", +) + temperature_field = Field( default=0.8, description="Adjust the randomness of the generated text.\n\n" @@ -111,6 +117,7 @@ class CreateCompletionRequest(BaseModel): max_tokens: Optional[int] = Field( default=16, ge=0, description="The maximum number of tokens to generate." ) + min_tokens: int = min_tokens_field temperature: float = temperature_field top_p: float = top_p_field min_p: float = min_p_field @@ -206,6 +213,7 @@ class CreateChatCompletionRequest(BaseModel): default=None, description="The maximum number of tokens to generate. Defaults to inf", ) + min_tokens: int = min_tokens_field logprobs: Optional[bool] = Field( default=False, description="Whether to output the logprobs or not. Default is True" From ca8e3c967d4fc3b1b7763f7134e78eccad549919 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 15 May 2024 23:59:17 -0400 Subject: [PATCH 170/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1265c670fd..13ad16af12 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1265c670fd8e41e1947352c96c5179adda97fb2c +Subproject commit 13ad16af1231ab2d245d35df3295bcfa23de1305 From d99a6ba607a4885fb00e63e967964aa41bdbbbcb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 16 May 2024 00:37:27 -0400 Subject: [PATCH 171/561] fix: segfault for models without eos / bos tokens. Closes #1463 --- llama_cpp/llama.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 7145329450..043fb2a6e1 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -413,8 +413,8 @@ def __init__( eos_token_id = self.token_eos() bos_token_id = self.token_bos() - eos_token = self._model.token_get_text(eos_token_id) - bos_token = self._model.token_get_text(bos_token_id) + eos_token = self._model.token_get_text(eos_token_id) if eos_token_id != -1 else "" + bos_token = self._model.token_get_text(bos_token_id) if bos_token_id != -1 else "" # Unfortunately the llama.cpp API does not return metadata arrays, so we can't get template names from tokenizer.chat_templates template_choices = dict((name[10:], template) for name, template in self.metadata.items() if name.startswith("tokenizer.chat_template.")) From b564d05806381fd56f2a6d5488ac24be931e0e5b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 16 May 2024 00:41:21 -0400 Subject: [PATCH 172/561] chore: Bump version --- CHANGELOG.md | 7 +++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2782be0234..55db68e436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.75] + +- feat: Update llama.cpp to ggerganov/llama.cpp@13ad16af1231ab2d245d35df3295bcfa23de1305 +- fix: segfault for models without eos / bos tokens by @abetlen in d99a6ba607a4885fb00e63e967964aa41bdbbbcb +- feat: add MinTokensLogitProcessor and min_tokens argument to server by @twaka in #1333 +- misc: Remove unnecessary metadata lookups by @CISC in #1448 + ## [0.2.74] - feat: Update llama.cpp to ggerganov/llama.cpp@b228aba91ac2cd9eb90e9d423ba1d0d20e0117e2 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index b7b8e47c66..8f93055341 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.74" \ No newline at end of file +__version__ = "0.2.75" \ No newline at end of file From 03f171e81069932fda296c85ebafc6fdf11025d0 Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Fri, 17 May 2024 20:27:26 +0300 Subject: [PATCH 173/561] example: LLM inference with Ray Serve (#1465) --- examples/ray/README.md | 19 +++++++++++++++++++ examples/ray/llm.py | 20 ++++++++++++++++++++ examples/ray/requirements.txt | 3 +++ 3 files changed, 42 insertions(+) create mode 100644 examples/ray/README.md create mode 100755 examples/ray/llm.py create mode 100644 examples/ray/requirements.txt diff --git a/examples/ray/README.md b/examples/ray/README.md new file mode 100644 index 0000000000..6e338ba17e --- /dev/null +++ b/examples/ray/README.md @@ -0,0 +1,19 @@ +This is an example of doing LLM inference with [Ray](https://docs.ray.io/en/latest/index.html) and [Ray Serve](https://docs.ray.io/en/latest/serve/index.html). + +First, install the requirements: + +```bash +$ pip install -r requirements.txt +``` + +Deploy a GGUF model to Ray Serve with the following command: + +```bash +$ serve run llm:llm_builder model_path='../models/mistral-7b-instruct-v0.2.Q4_K_M.gguf' +``` + +This will start an API endpoint at `http://localhost:8000/`. You can query the model like this: + +```bash +$ curl -k -d '{"prompt": "tell me a joke", "max_tokens": 128}' -X POST http://localhost:8000 +``` diff --git a/examples/ray/llm.py b/examples/ray/llm.py new file mode 100755 index 0000000000..27b15d21d4 --- /dev/null +++ b/examples/ray/llm.py @@ -0,0 +1,20 @@ +from starlette.requests import Request +from typing import Dict +from ray import serve +from ray.serve import Application +from llama_cpp import Llama + +@serve.deployment +class LlamaDeployment: + def __init__(self, model_path: str): + self._llm = Llama(model_path=model_path) + + async def __call__(self, http_request: Request) -> Dict: + input_json = await http_request.json() + prompt = input_json["prompt"] + max_tokens = input_json.get("max_tokens", 64) + return self._llm(prompt, max_tokens=max_tokens) + + +def llm_builder(args: Dict[str, str]) -> Application: + return LlamaDeployment.bind(args["model_path"]) diff --git a/examples/ray/requirements.txt b/examples/ray/requirements.txt new file mode 100644 index 0000000000..a409fb882d --- /dev/null +++ b/examples/ray/requirements.txt @@ -0,0 +1,3 @@ +ray[serve] +--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu +llama-cpp-python From d8a3b013c3b5193656b527750557d17e61c09057 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 18 May 2024 01:19:19 -0400 Subject: [PATCH 174/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 13ad16af12..05834841dc 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 13ad16af1231ab2d245d35df3295bcfa23de1305 +Subproject commit 05834841dcb4f922983ea976539c70472272df9a From 5a595f035a094574a9bcf153e6696369f63fb585 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 22 May 2024 02:40:31 -0400 Subject: [PATCH 175/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 14 ++++++++------ vendor/llama.cpp | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 8da3f140cd..88fa071865 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -296,9 +296,10 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, # LLAMA_VOCAB_PRE_TYPE_REFACT = 8, # LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, -# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 10, -# LLAMA_VOCAB_PRE_TYPE_OLMO = 11, -# LLAMA_VOCAB_PRE_TYPE_DBRX = 12, +# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, +# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, +# LLAMA_VOCAB_PRE_TYPE_OLMO = 12, +# LLAMA_VOCAB_PRE_TYPE_DBRX = 13, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -310,9 +311,10 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_GPT2 = 7 LLAMA_VOCAB_PRE_TYPE_REFACT = 8 LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9 -LLAMA_VOCAB_PRE_TYPE_QWEN2 = 10 -LLAMA_VOCAB_PRE_TYPE_OLMO = 11 -LLAMA_VOCAB_PRE_TYPE_DBRX = 12 +LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10 +LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11 +LLAMA_VOCAB_PRE_TYPE_OLMO = 12 +LLAMA_VOCAB_PRE_TYPE_DBRX = 13 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 05834841dc..201cc11afa 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 05834841dcb4f922983ea976539c70472272df9a +Subproject commit 201cc11afa0a1950e1f632390b2ac6c937a0d8f0 From 087cc0b036e6a52fda0df6fb4575e908fa0e32fd Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 24 May 2024 01:43:36 -0400 Subject: [PATCH 176/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 16 ++++++++++++++++ vendor/llama.cpp | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 88fa071865..4284019cb9 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -2265,6 +2265,22 @@ def llama_set_n_threads( ... +# // Get the number of threads used for generation of a single token. +# LLAMA_API uint32_t llama_n_threads(struct llama_context * ctx); +@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_uint32) +def llama_n_threads(ctx: llama_context_p, /) -> int: + """Get the number of threads used for generation of a single token""" + ... + + +# // Get the number of threads used for prompt and batch processing (multiple token). +# LLAMA_API uint32_t llama_n_threads_batch(struct llama_context * ctx); +@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_uint32) +def llama_n_threads_batch(ctx: llama_context_p, /) -> int: + """Get the number of threads used for prompt and batch processing (multiple token)""" + ... + + # // Set whether to use causal attention or not # // If set to true, the model will only attend to the past tokens # LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 201cc11afa..0df0aa8e43 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 201cc11afa0a1950e1f632390b2ac6c937a0d8f0 +Subproject commit 0df0aa8e43c3378975269a51f9b876c8692e70da From 5cae1040e35326dc514552bb9423c3718a37bf44 Mon Sep 17 00:00:00 2001 From: Linghan Zhong Date: Fri, 24 May 2024 00:49:44 -0500 Subject: [PATCH 177/561] feat: Improve Llama.eval performance by avoiding list conversion (#1476) Co-authored-by: Andrei --- llama_cpp/llama.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 043fb2a6e1..6dad650c60 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -562,12 +562,12 @@ def eval(self, tokens: Sequence[int]): if self.context_params.logits_all: rows = n_tokens cols = self._n_vocab - logits = self._ctx.get_logits()[: rows * cols] + logits = np.ctypeslib.as_array(self._ctx.get_logits(), shape=(rows * cols, )) self.scores[n_past : n_past + n_tokens, :].reshape(-1)[: :] = logits else: rows = 1 cols = self._n_vocab - logits = self._ctx.get_logits()[: rows * cols] + logits = np.ctypeslib.as_array(self._ctx.get_logits(), shape=(rows * cols, )) self.scores[n_past + n_tokens - 1, :].reshape(-1)[: :] = logits # Update n_tokens self.n_tokens += n_tokens From a4c9ab885d3c5b95ca881e72666bef876df97844 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 24 May 2024 01:59:25 -0400 Subject: [PATCH 178/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55db68e436..3e11407c67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.76] + +- feat: Update llama.cpp to ggerganov/llama.cpp@0df0aa8e43c3378975269a51f9b876c8692e70da +- feat: Improve Llama.eval performance by avoiding list conversion by @thoughtp0lice in #1476 +- example: LLM inference with Ray Serve by @rgerganov in #1465 + ## [0.2.75] - feat: Update llama.cpp to ggerganov/llama.cpp@13ad16af1231ab2d245d35df3295bcfa23de1305 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 8f93055341..fcc50d75b8 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.75" \ No newline at end of file +__version__ = "0.2.76" \ No newline at end of file From ec43e8920f2231639bea33d3db6651841bcec45c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 24 May 2024 11:54:15 -0400 Subject: [PATCH 179/561] docs: Update multi-modal model section --- README.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d5ced86068..3255e69560 100644 --- a/README.md +++ b/README.md @@ -499,13 +499,16 @@ llm = Llama.from_pretrained( `llama-cpp-python` supports such as llava1.5 which allow the language model to read information from both text and images. -You'll first need to download one of the available multi-modal models in GGUF format: - -- [llava-v1.5-7b](https://huggingface.co/mys/ggml_llava-v1.5-7b) -- [llava-v1.5-13b](https://huggingface.co/mys/ggml_llava-v1.5-13b) -- [bakllava-1-7b](https://huggingface.co/mys/ggml_bakllava-1) -- [llava-v1.6-34b](https://huggingface.co/cjpais/llava-v1.6-34B-gguf) -- [moondream2](https://huggingface.co/vikhyatk/moondream2) +Below are the supported multi-modal models and their respective chat handlers (Python API) and chat formats (Server API). + +| Model | `LlamaChatHandler` | `chat_format` | +| --- | --- | --- | +| [llava-v1.5-7b](https://huggingface.co/mys/ggml_llava-v1.5-7b) | `Llava15ChatHandler` | `llava-1-5` | +| [llava-v1.5-13b](https://huggingface.co/mys/ggml_llava-v1.5-13b) | `Llava15ChatHandler` | `llava-1-5` | +| [llava-v1.6-34b](https://huggingface.co/cjpais/llava-v1.6-34B-gguf) | `Llava16ChatHandler` | `llava-1-6` | +| [moondream2](https://huggingface.co/vikhyatk/moondream2) | `MoondreamChatHandler` | `moondream2` | +| [nanollava](https://huggingface.co/abetlen/nanollava) | `NanollavaChatHandler` | `nanollava` | +| [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. From 9e8d7d55bd9873d10c23a78a954a3063a5e995c0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 24 May 2024 11:55:01 -0400 Subject: [PATCH 180/561] fix(docs): Fix link typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3255e69560..07445eb7ff 100644 --- a/README.md +++ b/README.md @@ -507,8 +507,8 @@ Below are the supported multi-modal models and their respective chat handlers (P | [llava-v1.5-13b](https://huggingface.co/mys/ggml_llava-v1.5-13b) | `Llava15ChatHandler` | `llava-1-5` | | [llava-v1.6-34b](https://huggingface.co/cjpais/llava-v1.6-34B-gguf) | `Llava16ChatHandler` | `llava-1-6` | | [moondream2](https://huggingface.co/vikhyatk/moondream2) | `MoondreamChatHandler` | `moondream2` | -| [nanollava](https://huggingface.co/abetlen/nanollava) | `NanollavaChatHandler` | `nanollava` | -| [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | +| [nanollava](https://huggingface.co/abetlen/nanollava-gguf) | `NanollavaChatHandler` | `nanollava` | +| [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha-gguf) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. From 2d8996414706c0bb69fae06ddf7f092f27790fe7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 24 May 2024 11:55:41 -0400 Subject: [PATCH 181/561] docs: Fix table formatting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 07445eb7ff..342e924a07 100644 --- a/README.md +++ b/README.md @@ -502,7 +502,7 @@ llm = Llama.from_pretrained( Below are the supported multi-modal models and their respective chat handlers (Python API) and chat formats (Server API). | Model | `LlamaChatHandler` | `chat_format` | -| --- | --- | --- | +|:--- |:--- |:--- | | [llava-v1.5-7b](https://huggingface.co/mys/ggml_llava-v1.5-7b) | `Llava15ChatHandler` | `llava-1-5` | | [llava-v1.5-13b](https://huggingface.co/mys/ggml_llava-v1.5-13b) | `Llava15ChatHandler` | `llava-1-5` | | [llava-v1.6-34b](https://huggingface.co/cjpais/llava-v1.6-34B-gguf) | `Llava16ChatHandler` | `llava-1-6` | From 454c9bb1cb3a58ecc37b58abeff6f245e3b8f316 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 27 May 2024 10:51:57 -0400 Subject: [PATCH 182/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 21 +++++++++++++++++---- vendor/llama.cpp | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 4284019cb9..d9b5087dae 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -300,6 +300,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, # LLAMA_VOCAB_PRE_TYPE_OLMO = 12, # LLAMA_VOCAB_PRE_TYPE_DBRX = 13, +# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -315,6 +316,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11 LLAMA_VOCAB_PRE_TYPE_OLMO = 12 LLAMA_VOCAB_PRE_TYPE_DBRX = 13 +LLAMA_VOCAB_PRE_TYPE_SMAUG = 14 # // note: these values should be synchronized with ggml_rope @@ -718,6 +720,8 @@ class llama_model_params(ctypes.Structure): ] +# // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations +# // https://github.com/ggerganov/llama.cpp/pull/7544 # struct llama_context_params { # uint32_t seed; // RNG seed, -1 for random # uint32_t n_ctx; // text context, 0 = from model @@ -744,15 +748,14 @@ class llama_model_params(ctypes.Structure): # ggml_backend_sched_eval_callback cb_eval; # void * cb_eval_user_data; -# enum ggml_type type_k; // data type for K cache -# enum ggml_type type_v; // data type for V cache +# enum ggml_type type_k; // data type for K cache [EXPERIMENTAL] +# enum ggml_type type_v; // data type for V cache [EXPERIMENTAL] # // Keep the booleans together to avoid misalignment during copy-by-value. # bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) # bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU -# bool flash_attn; // whether to use flash attention - +# bool flash_attn; // whether to use flash attention [EXPERIMENTAL] # // Abort callback # // if it returns true, execution of llama_decode() will be aborted @@ -2454,6 +2457,16 @@ def llama_token_is_eog(model: llama_model_p, token: Union[llama_token, int], /) ... +# // Identify if Token Id is a control token or a render-able token +# LLAMA_API bool llama_token_is_control(const struct llama_model * model, llama_token token); +@ctypes_function( + "llama_token_is_control", [llama_model_p_ctypes, llama_token], ctypes.c_bool +) +def llama_token_is_control(model: llama_model_p, token: Union[llama_token, int], /) -> bool: + """Identify if Token Id is a control token or a render-able token""" + ... + + # // Special tokens diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 0df0aa8e43..5487593bc7 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 0df0aa8e43c3378975269a51f9b876c8692e70da +Subproject commit 5487593bc7ee0b65b9d2e2985b4b61dc77043101 From c564007ff6c93d3408031db9562de13d50112707 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 10:57:17 -0400 Subject: [PATCH 183/561] chore(deps): bump pypa/cibuildwheel from 2.18.0 to 2.18.1 (#1472) updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei --- .github/workflows/build-and-release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index b50da48458..957912d09d 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -29,7 +29,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.18.0 + uses: pypa/cibuildwheel@v2.18.1 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -56,7 +56,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.18.0 + uses: pypa/cibuildwheel@v2.18.1 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" From c26004b1be20c485ea547a87b3871551f6a8774c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 28 May 2024 22:52:03 -0400 Subject: [PATCH 184/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 5487593bc7..504f0c340f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 5487593bc7ee0b65b9d2e2985b4b61dc77043101 +Subproject commit 504f0c340f6b5e04de682f6ddefdd3b81208df5d From 2907c26906272a333ab913054f7ae7cf2bbd94ed Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 28 May 2024 22:52:28 -0400 Subject: [PATCH 185/561] misc: Update debug build to keep all debug symbols for easier gdb debugging --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 83085d2706..3796d17e33 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,13 @@ build: python3 -m pip install --verbose -e . build.debug: - CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug" python3 -m pip install --verbose --config-settings=cmake.verbose=true --config-settings=logging.level=INFO --config-settings=install.strip=false --editable . + python3 -m pip install \ + --verbose \ + --config-settings=cmake.verbose=true \ + --config-settings=logging.level=INFO \ + --config-settings=install.strip=false \ + --config-settings=cmake.args="-DCMAKE_BUILD_TYPE=Debug;-DCMAKE_C_FLAGS='-ggdb -O0';-DCMAKE_CXX_FLAGS='-ggdb -O0'" \ + --editable . build.cuda: CMAKE_ARGS="-DLLAMA_CUDA=on" python3 -m pip install --verbose -e . From df45a4b3fe46e72664bda87301b318210c6d4782 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 29 May 2024 02:02:22 -0400 Subject: [PATCH 186/561] fix: fix string value kv_overrides. Closes #1487 --- llama_cpp/llama.py | 13 ++++++++----- llama_cpp/server/model.py | 4 +++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 6dad650c60..6d872e3e65 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -6,6 +6,7 @@ import time import json import ctypes +import typing import fnmatch import multiprocessing @@ -249,13 +250,13 @@ def __init__( self._kv_overrides_array[i].key = k.encode("utf-8") if isinstance(v, bool): self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL - self._kv_overrides_array[i].value.bool_value = v + self._kv_overrides_array[i].value.val_bool = v elif isinstance(v, int): self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT - self._kv_overrides_array[i].value.int_value = v + self._kv_overrides_array[i].value.val_i64 = v elif isinstance(v, float): self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT - self._kv_overrides_array[i].value.float_value = v + self._kv_overrides_array[i].value.val_f64 = v elif isinstance(v, str): # type: ignore v_bytes = v.encode("utf-8") if len(v_bytes) > 128: # TODO: Make this a constant @@ -263,10 +264,12 @@ def __init__( v_bytes = v_bytes.ljust(128, b"\0") self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR # copy min(v_bytes, 128) to str_value + address = typing.cast(int, ctypes.addressof(self._kv_overrides_array[i].value) + llama_cpp.llama_model_kv_override_value.val_str.offset) + buffer_start = ctypes.cast(address, ctypes.POINTER(ctypes.c_char)) ctypes.memmove( - self._kv_overrides_array[i].value.str_value, + buffer_start, v_bytes, - min(len(v_bytes), 128), + 128, ) else: raise ValueError(f"Unknown value type for {k}: {v}") diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index f002924109..4f83716687 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -183,7 +183,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: num_pred_tokens=settings.draft_model_num_pred_tokens ) - kv_overrides: Optional[Dict[str, Union[bool, int, float]]] = None + kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None if settings.kv_overrides is not None: assert isinstance(settings.kv_overrides, list) kv_overrides = {} @@ -197,6 +197,8 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: kv_overrides[key] = int(value) elif value_type == "float": kv_overrides[key] = float(value) + elif value_type == "str": + kv_overrides[key] = value else: raise ValueError(f"Unknown value type {value_type}") From 91d05aba469e9ce4632995b8f41e7c3b3c3518a5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 29 May 2024 02:28:58 -0400 Subject: [PATCH 187/561] fix: adjust kv_override member names to match llama.cpp --- llama_cpp/llama_cpp.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d9b5087dae..1668717a1c 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -613,17 +613,17 @@ class llama_batch(ctypes.Structure): # }; class llama_model_kv_override_value(ctypes.Union): _fields_ = [ - ("int_value", ctypes.c_int64), - ("float_value", ctypes.c_double), - ("bool_value", ctypes.c_bool), - ("str_value", ctypes.c_char * 128), + ("val_i64", ctypes.c_int64), + ("val_f64", ctypes.c_double), + ("val_bool", ctypes.c_bool), + ("val_str", ctypes.c_char * 128), ] if TYPE_CHECKING: - int_value: int - float_value: float - bool_value: bool - str_value: bytes + val_i64: int + val_f64: float + val_bool: bool + val_str: bytes class llama_model_kv_override(ctypes.Structure): From 165b4dc6c188f8fda2fc616154e111f710484eba Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 29 May 2024 02:29:44 -0400 Subject: [PATCH 188/561] fix: Fix typo in Llama3VisionAlphaChatHandler. Closes #1488 --- llama_cpp/llama_chat_format.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index e2d7e27855..8f3b1de118 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3098,7 +3098,7 @@ class NanoLlavaChatHandler(Llava15ChatHandler): "{% endif %}" ) -class Llama3VisionAlpha(Llava15ChatHandler): +class Llama3VisionAlphaChatHandler(Llava15ChatHandler): # question = "" + q # prompt = f"<|start_header_id|>user<|end_header_id|>\n\n{question}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" @@ -3159,6 +3159,10 @@ class Llama3VisionAlpha(Llava15ChatHandler): "{% endif %}" ) +# alias +Llama3VisionAlpha = Llama3VisionAlphaChatHandler + + @register_chat_completion_handler("chatml-function-calling") def chatml_function_calling( llama: llama.Llama, @@ -3193,7 +3197,6 @@ def chatml_function_calling( llama_types.CreateChatCompletionResponse, Iterator[llama_types.CreateChatCompletionStreamResponse], ]: - print(logprobs) function_calling_template = ( "{% for message in messages %}" "<|im_start|>{{ message.role }}\n" From af3ed503e9ce60fe6b5365031abad4176a3536b3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 1 Jun 2024 18:09:24 -0400 Subject: [PATCH 189/561] fix: Use numpy recarray for candidates data, fixes bug with temp < 0 --- llama_cpp/_internals.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index b404601d30..e3f539b11c 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -545,13 +545,12 @@ def add_sequence(self, batch: Sequence[int], seq_id: int, logits_all: bool): class _LlamaTokenDataArray: def __init__(self, *, n_vocab: int): self.n_vocab = n_vocab - self.candidates_data = np.array( - [], + self.candidates_data = np.recarray( + (self.n_vocab,), dtype=np.dtype( [("id", np.intc), ("logit", np.single), ("p", np.single)], align=True ), ) - self.candidates_data.resize(3, self.n_vocab, refcheck=False) self.candidates = llama_cpp.llama_token_data_array( data=self.candidates_data.ctypes.data_as(llama_cpp.llama_token_data_p), size=self.n_vocab, @@ -561,14 +560,11 @@ def __init__(self, *, n_vocab: int): self.default_candidates_data_p = np.zeros(self.n_vocab, dtype=np.single) def copy_logits(self, logits: npt.NDArray[np.single]): - self.candidates_data["id"][:] = self.default_candidates_data_id - self.candidates_data["logit"][:] = logits - self.candidates_data["p"][:] = self.default_candidates_data_p - self.candidates.data = self.candidates_data.ctypes.data_as( - llama_cpp.llama_token_data_p - ) - self.candidates.sorted = ctypes.c_bool(False) - self.candidates.size = ctypes.c_size_t(self.n_vocab) + self.candidates_data.id[:] = self.default_candidates_data_id + self.candidates_data.logit[:] = logits + self.candidates_data.p[:] = self.default_candidates_data_p + self.candidates.sorted = False + self.candidates.size = self.n_vocab # Python wrappers over common/common @@ -759,14 +755,14 @@ def sample( self.params.penalty_present, ) if not self.params.penalize_nl: - token_data_array.candidates_data["logit"][nl_token] = nl_logit + token_data_array.candidates_data.logit[nl_token] = nl_logit if self.grammar is not None: ctx_main.sample_grammar(token_data_array, self.grammar) if self.params.temp < 0: ctx_main.sample_softmax(token_data_array) - id = token_data_array.candidates_data["id"][0] + id = token_data_array.candidates_data.id[0] elif self.params.temp == 0: id = ctx_main.sample_token_greedy(token_data_array) else: From 6b018e00b15c18f0c72d85bdc64ee92f69b036ed Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 3 Jun 2024 11:19:10 -0400 Subject: [PATCH 190/561] misc: Improve llava error messages --- llama_cpp/llama_chat_format.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 8f3b1de118..b384749e07 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2642,13 +2642,13 @@ def embed_image_bytes(image_bytes: bytes): if type_ == "text": tokens = llama.tokenize(value.encode("utf8"), add_bos=False, special=True) if llama.n_tokens + len(tokens) > llama.n_ctx(): - raise ValueError("Prompt exceeds n_ctx") # TODO: Fix + raise ValueError(f"Prompt exceeds n_ctx: {llama.n_tokens + len(tokens)} > {llama.n_ctx()}") llama.eval(tokens) else: image_bytes = self.load_image(value) embed = embed_image_bytes(image_bytes) if llama.n_tokens + embed.contents.n_image_pos > llama.n_ctx(): - raise ValueError("Prompt exceeds n_ctx") # TODO: Fix + raise ValueError(f"Prompt exceeds n_ctx: {llama.n_tokens + embed.contents.n_image_pos} > {llama.n_ctx()}") n_past = ctypes.c_int(llama.n_tokens) n_past_p = ctypes.pointer(n_past) with suppress_stdout_stderr(disable=self.verbose): From cd3f1bb387fb351fab710d4eb10b13fd87e4aa36 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 4 Jun 2024 00:35:47 -0400 Subject: [PATCH 191/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 4 ++-- vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1668717a1c..706f50e1a6 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1233,12 +1233,12 @@ def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... def llama_pooling_type(ctx: llama_context_p, /) -> int: ... -# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); +# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) def llama_vocab_type(model: llama_model_p, /) -> int: ... -# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); +# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) def llama_rope_type(model: llama_model_p, /) -> int: ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 504f0c340f..bde7cd3cd9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 504f0c340f6b5e04de682f6ddefdd3b81208df5d +Subproject commit bde7cd3cd949c1a85d3a199498ac98e78039d46f From ae5682f500623383a3676fec17aec19330ead67e Mon Sep 17 00:00:00 2001 From: Engininja2 <139037756+Engininja2@users.noreply.github.com> Date: Mon, 3 Jun 2024 22:42:34 -0600 Subject: [PATCH 192/561] fix: Disable Windows+CUDA workaround when compiling for HIPBLAS (#1493) * Disable Windows+CUDA workaround when compiling for HIPBLAS * fix spacing * change condition to check for Windows & CUDA Co-authored-by: Andrei --------- Co-authored-by: Andrei --- CMakeLists.txt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dcaddb678e..4b9508d407 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,14 +41,16 @@ if (LLAMA_BUILD) RESOURCE DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp ) # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563 - install( - FILES $ - DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - ) - install( - FILES $ - DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp - ) + if (WIN32 AND (LLAMA_CUDA OR LLAMA_CUBLAS)) + install( + FILES $ + DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp + ) + install( + FILES $ + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp + ) + endif() if (LLAVA_BUILD) if (LLAMA_CUBLAS OR LLAMA_CUDA) From c3ef41ba06c6fab90592040e582985c13d185faa Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 4 Jun 2024 00:49:24 -0400 Subject: [PATCH 193/561] chore: Bump version --- CHANGELOG.md | 8 ++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e11407c67..1b9865da0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.77] + +- feat: Update llama.cpp to ggerganov/llama.cpp@bde7cd3cd949c1a85d3a199498ac98e78039d46f +- fix: string value kv_overrides by @abetlen in df45a4b3fe46e72664bda87301b318210c6d4782 +- fix: Fix typo in Llama3VisionAlphaChatHandler by @abetlen in 165b4dc6c188f8fda2fc616154e111f710484eba +- fix: Use numpy recarray for candidates data, fixes bug with temp < 0 by @abetlen in af3ed503e9ce60fe6b5365031abad4176a3536b3 +fix: Disable Windows+CUDA workaround when compiling for HIPBLAS by Engininja2 in #1493 + ## [0.2.76] - feat: Update llama.cpp to ggerganov/llama.cpp@0df0aa8e43c3378975269a51f9b876c8692e70da diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index fcc50d75b8..3df249f5eb 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.76" \ No newline at end of file +__version__ = "0.2.77" \ No newline at end of file From 027f7bc67890f1de801407fbbb608c182e2ad286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Tue, 4 Jun 2024 16:15:41 +0200 Subject: [PATCH 194/561] fix: Avoid duplicate special tokens in chat formats (#1439) * Templates sometimes have BOS in them, remove duplicate * tokenize chat format prompts before completion This is to ensure that we don't duplicate any special tokens. Hopefully I amended the existing formats correctly? * updated comment * corrected a few * add some missing internals * proper bos/eos detection * just let tokenizer do the job * typo-- * align test with new response * changed to a warning * move to another PR * Use python warnings module --------- Co-authored-by: Andrei Betlen --- llama_cpp/_internals.py | 8 ++++++++ llama_cpp/llama.py | 7 +++++++ llama_cpp/llama_chat_format.py | 17 ++++++++--------- tests/test_llama_chat_format.py | 3 ++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index e3f539b11c..fb437c3f1f 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -142,6 +142,14 @@ def token_eos(self) -> int: assert self.model is not None return llama_cpp.llama_token_eos(self.model) + def token_cls(self) -> int: + assert self.model is not None + return llama_cpp.llama_token_cls(self.model) + + def token_sep(self) -> int: + assert self.model is not None + return llama_cpp.llama_token_sep(self.model) + def token_nl(self) -> int: assert self.model is not None return llama_cpp.llama_token_nl(self.model) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 6d872e3e65..45e1526ef0 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -8,6 +8,7 @@ import ctypes import typing import fnmatch +import warnings import multiprocessing from typing import ( @@ -1019,6 +1020,12 @@ def _create_completion( ) model_name: str = model if model is not None else self.model_path + if prompt_tokens[:2] == [self.token_bos()] * 2: + warnings.warn( + f'Detected duplicate leading "{self._model.token_get_text(self.token_bos())}" in prompt, this will likely reduce response quality, consider removing it...', + RuntimeWarning, + ) + # NOTE: This likely doesn't work correctly for the first token in the prompt # because of the extra space added to the start of the prompt_tokens if logit_bias is not None: diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index b384749e07..975f74762d 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -160,6 +160,7 @@ class ChatFormatterResponse: prompt: str stop: Optional[Union[str, List[str]]] = None stopping_criteria: Optional[llama.StoppingCriteriaList] = None + added_special: bool = False class ChatFormatter(Protocol): @@ -232,7 +233,7 @@ def stop_on_last_token( return tokens[-1] in self.stop_token_ids stopping_criteria = llama.StoppingCriteriaList([stop_on_last_token]) - return ChatFormatterResponse(prompt=prompt, stop=[self.eos_token], stopping_criteria=stopping_criteria) + return ChatFormatterResponse(prompt=prompt, stop=[self.eos_token], stopping_criteria=stopping_criteria, added_special=True) def to_chat_handler(self) -> LlamaChatCompletionHandler: return chat_formatter_to_chat_completion_handler(self) @@ -548,7 +549,7 @@ def chat_completion_handler( tools=tools, tool_choice=tool_choice, ) - prompt = result.prompt + prompt = llama.tokenize(result.prompt.encode("utf-8"), add_bos=not result.added_special, special=True) if result.stop is not None: stop = [] if stop is None else [stop] if isinstance(stop, str) else stop rstop = result.stop if isinstance(result.stop, list) else [result.stop] @@ -655,7 +656,7 @@ def format_autotokenizer( prompt: str = tokenizer.apply_chat_template(messages, tokenize=False) # type: ignore assert isinstance(prompt, str) # Return formatted prompt and eos token by default - return ChatFormatterResponse(prompt=prompt, stop=tokenizer.eos_token) + return ChatFormatterResponse(prompt=prompt, stop=tokenizer.eos_token, added_special=True) return format_autotokenizer @@ -708,7 +709,7 @@ def format_tokenizer_config( bos_token=bos_token, eos_token=eos_token, ) - return ChatFormatterResponse(prompt=prompt, stop=[eos_token, bos_token]) + return ChatFormatterResponse(prompt=prompt, stop=[eos_token, bos_token], added_special=True) return format_tokenizer_config @@ -918,7 +919,7 @@ def format_llama2( messages: List[llama_types.ChatCompletionRequestMessage], **kwargs: Any, ) -> ChatFormatterResponse: - _system_template = "[INST] <>\n{system_message}\n<>" + _system_template = "[INST] <>\n{system_message}\n<>" _roles = dict(user="[INST]", assistant="[/INST]") _messages = _map_roles(messages, _roles) system_message = _get_system_message(messages) @@ -940,11 +941,10 @@ def format_llama3( user="<|start_header_id|>user<|end_header_id|>\n\n", assistant="<|start_header_id|>assistant<|end_header_id|>\n\n", ) - _begin_token = "<|begin_of_text|>" _sep = "<|eot_id|>" _messages = _map_roles(messages, _roles) _messages.append((_roles["assistant"], None)) - _prompt = _format_no_colon_single(_begin_token, _messages, _sep) + _prompt = _format_no_colon_single("", _messages, _sep) return ChatFormatterResponse(prompt=_prompt, stop=_sep) @@ -1229,10 +1229,9 @@ def format_mistral_instruct( messages: List[llama_types.ChatCompletionRequestMessage], **kwargs: Any, ) -> ChatFormatterResponse: - bos = "" eos = "" stop = eos - prompt = bos + prompt = "" for message in messages: if ( message["role"] == "user" diff --git a/tests/test_llama_chat_format.py b/tests/test_llama_chat_format.py index c10aee42e0..f031bf72b7 100644 --- a/tests/test_llama_chat_format.py +++ b/tests/test_llama_chat_format.py @@ -21,12 +21,13 @@ def test_mistral_instruct(): response = llama_chat_format.format_mistral_instruct( messages=messages, ) + prompt = ("" if response.added_special else "") + response.prompt reference = chat_formatter.render( messages=messages, bos_token="", eos_token="", ) - assert response.prompt == reference + assert prompt == reference mistral_7b_tokenizer_config = """{ From 6e0642ca195e2716e6e1bfa05d2b76f1e2dae0a3 Mon Sep 17 00:00:00 2001 From: Asghar Ghorbani Date: Tue, 4 Jun 2024 16:18:38 +0200 Subject: [PATCH 195/561] fix: fix logprobs when BOS is not present (#1471) * Fix lobprobs when BOS is not present * Fix logprobs when bos is not available --- llama_cpp/llama.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 45e1526ef0..fb26950b67 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1410,8 +1410,8 @@ def logit_bias_processor( top_logprobs: List[Optional[Dict[str, float]]] = [] if echo: - # Remove leading BOS token - all_tokens = prompt_tokens[1:] + completion_tokens + # Remove leading BOS token if exists + all_tokens = prompt_tokens[1 if prompt_tokens[0] == self.token_bos() else 0:] + completion_tokens else: all_tokens = completion_tokens From d634efcdd91a0f79ae0fac13cfdf46eb47d0602d Mon Sep 17 00:00:00 2001 From: nullname Date: Tue, 4 Jun 2024 22:38:21 +0800 Subject: [PATCH 196/561] feat: adding `rpc_servers` parameter to `Llama` class (#1477) * passthru rpc_servers params wip * enable llama rpc by default * convert string to byte * add rpc package * Revert "enable llama rpc by default" This reverts commit 832c6dd56c979514cec5df224bf2d2014dccd790. * update readme * Only set rpc_servers when provided * Add rpc servers to server options --------- Co-authored-by: Andrei Betlen --- Makefile | 3 +++ README.md | 11 +++++++++++ llama_cpp/llama.py | 7 +++++++ llama_cpp/server/model.py | 1 + llama_cpp/server/settings.py | 4 ++++ 5 files changed, 26 insertions(+) diff --git a/Makefile b/Makefile index 3796d17e33..d8fb0cc27b 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,9 @@ build.kompute: build.sycl: CMAKE_ARGS="-DLLAMA_SYCL=on" python3 -m pip install --verbose -e . +build.rpc: + CMAKE_ARGS="-DLLAMA_RPC=on" python3 -m pip install --verbose -e . + build.sdist: python3 -m build --sdist diff --git a/README.md b/README.md index 342e924a07..0f7abfb28d 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,17 @@ CMAKE_ARGS="-DLLAMA_SYCL=on -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx" pi ```
+
+RPC + +To install with RPC support, set the `LLAMA_RPC=on` environment variable before installing: + +```bash +source /opt/intel/oneapi/setvars.sh +CMAKE_ARGS="-DLLAMA_RPC=on" pip install llama-cpp-python +``` +
+ ### Windows Notes diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index fb26950b67..bf3bd656de 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -72,6 +72,7 @@ def __init__( split_mode: int = llama_cpp.LLAMA_SPLIT_MODE_LAYER, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, + rpc_servers: Optional[str] = None, vocab_only: bool = False, use_mmap: bool = True, use_mlock: bool = False, @@ -150,6 +151,7 @@ def __init__( split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options. main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. + rpc_servers: Comma separated list of RPC servers to use for offloading vocab_only: Only load the vocabulary no weights. use_mmap: Use mmap if possible. use_mlock: Force the system to keep the model in RAM. @@ -221,6 +223,11 @@ def __init__( ) # 0x7FFFFFFF is INT32 max, will be auto set to all layers self.model_params.split_mode = split_mode self.model_params.main_gpu = main_gpu + if rpc_servers is not None: + self.model_params.rpc_servers = rpc_servers.encode('utf-8') + self._rpc_servers = rpc_servers + else: + self._rpc_servers = None self.tensor_split = tensor_split self._c_tensor_split = None if self.tensor_split is not None: diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 4f83716687..d4d4acbe3f 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -226,6 +226,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: use_mmap=settings.use_mmap, use_mlock=settings.use_mlock, kv_overrides=kv_overrides, + rpc_servers=settings.rpc_servers, # Context Params seed=settings.seed, n_ctx=settings.n_ctx, diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index a3e185007d..4d924f3373 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -58,6 +58,10 @@ class ModelSettings(BaseSettings): default=None, description="List of model kv overrides in the format key=type:value where type is one of (bool, int, float). Valid true values are (true, TRUE, 1), otherwise false.", ) + rpc_servers: Optional[str] = Field( + default=None, + description="comma seperated list of rpc servers for offloading", + ) # Context Params seed: int = Field( default=llama_cpp.LLAMA_DEFAULT_SEED, description="Random seed. -1 for random." From 255e1b44955f1c81c6dccce312e162dbc5168def Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 7 Jun 2024 02:02:12 -0400 Subject: [PATCH 197/561] feat: Update llama.cpp --- llama_cpp/_internals.py | 4 +- llama_cpp/llama_cpp.py | 135 +++++++++++----------------------------- vendor/llama.cpp | 2 +- 3 files changed, 38 insertions(+), 103 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index fb437c3f1f..ba04291390 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -128,9 +128,9 @@ def token_get_score(self, token: int) -> float: assert self.model is not None return llama_cpp.llama_token_get_score(self.model, token) - def token_get_type(self, token: int) -> int: + def token_get_attr(self, token: int) -> int: assert self.model is not None - return llama_cpp.llama_token_get_type(self.model, token) + return llama_cpp.llama_token_get_attr(self.model, token) # Special tokens diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 706f50e1a6..6f37fcb69c 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -333,7 +333,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_ROPE_TYPE_GLM = 4 -# enum llama_token_type { +# enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file # LLAMA_TOKEN_TYPE_UNDEFINED = 0, # LLAMA_TOKEN_TYPE_NORMAL = 1, # LLAMA_TOKEN_TYPE_UNKNOWN = 2, @@ -351,6 +351,32 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_TOKEN_TYPE_BYTE = 6 +# enum llama_token_attr { +# LLAMA_TOKEN_ATTR_UNDEFINED = 0, +# LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0, +# LLAMA_TOKEN_ATTR_UNUSED = 1 << 1, +# LLAMA_TOKEN_ATTR_NORMAL = 1 << 2, +# LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL? +# LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4, +# LLAMA_TOKEN_ATTR_BYTE = 1 << 5, +# LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6, +# LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7, +# LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8, +# LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9, +# }; +LLAMA_TOKEN_ATTR_UNDEFINED = 0 +LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0 +LLAMA_TOKEN_ATTR_UNUSED = 1 << 1 +LLAMA_TOKEN_ATTR_NORMAL = 1 << 2 +LLAMA_TOKEN_ATTR_CONTROL = 1 << 3 +LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4 +LLAMA_TOKEN_ATTR_BYTE = 1 << 5 +LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6 +LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7 +LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8 +LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9 + + # // model file types # enum llama_ftype { # LLAMA_FTYPE_ALL_F32 = 0, @@ -959,6 +985,9 @@ class llama_model_quantize_params(ctypes.Structure): # // modifies a preceding LLAMA_GRETYPE_CHAR or # // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) # LLAMA_GRETYPE_CHAR_ALT = 6, + +# // any character (.) +# LLAMA_GRETYPE_CHAR_ANY = 7, # }; LLAMA_GRETYPE_END = 0 LLAMA_GRETYPE_ALT = 1 @@ -967,6 +996,7 @@ class llama_model_quantize_params(ctypes.Structure): LLAMA_GRETYPE_CHAR_NOT = 4 LLAMA_GRETYPE_CHAR_RNG_UPPER = 5 LLAMA_GRETYPE_CHAR_ALT = 6 +LLAMA_GRETYPE_CHAR_ANY = 7 # typedef struct llama_grammar_element { @@ -2438,11 +2468,11 @@ def llama_token_get_score( ) -> float: ... -# LLAMA_API enum llama_token_type llama_token_get_type(const struct llama_model * model, llama_token token); +# LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token); @ctypes_function( - "llama_token_get_type", [llama_model_p_ctypes, llama_token], ctypes.c_int + "llama_token_get_attr", [llama_model_p_ctypes, llama_token], ctypes.c_int ) -def llama_token_get_type( +def llama_token_get_attr( model: llama_model_p, token: Union[llama_token, int], / ) -> int: ... @@ -3200,104 +3230,9 @@ def llama_grammar_accept_token( # // -# // Beam search +# // Model split # // -# struct llama_beam_view { -# const llama_token * tokens; - - -# size_t n_tokens; -# float p; // Cumulative beam probability (renormalized relative to all beams) -# bool eob; // Callback should set this to true when a beam is at end-of-beam. -# }; -class llama_beam_view(ctypes.Structure): - if TYPE_CHECKING: - tokens: CtypesArray[llama_token] - n_tokens: int - p: float - eob: bool - - _fields_ = [ - ("tokens", llama_token_p), - ("n_tokens", ctypes.c_size_t), - ("p", ctypes.c_float), - ("eob", ctypes.c_bool), - ] - - -# // Passed to beam_search_callback function. -# // Whenever 0 < common_prefix_length, this number of tokens should be copied from any of the beams -# // (e.g. beams[0]) as they will be removed (shifted) from all beams in all subsequent callbacks. -# // These pointers are valid only during the synchronous callback, so should not be saved. -# struct llama_beams_state { -# struct llama_beam_view * beam_views; -# size_t n_beams; // Number of elements in beam_views[]. -# size_t common_prefix_length; // Current max length of prefix tokens shared by all beams. -# bool last_call; // True iff this is the last callback invocation. -# }; -class llama_beams_state(ctypes.Structure): - if TYPE_CHECKING: - beam_views: CtypesArray[llama_beam_view] - n_beams: int - common_prefix_length: int - last_call: bool - - _fields_ = [ - ("beam_views", ctypes.POINTER(llama_beam_view)), - ("n_beams", ctypes.c_size_t), - ("common_prefix_length", ctypes.c_size_t), - ("last_call", ctypes.c_bool), - ] - - -# // Type of pointer to the beam_search_callback function. -# // void* callback_data is any custom data passed to llama_beam_search, that is subsequently -# // passed back to beam_search_callback. This avoids having to use global variables in the callback. -# typedef void (*llama_beam_search_callback_fn_t)(void * callback_data, struct llama_beams_state); -llama_beam_search_callback_fn_t = ctypes.CFUNCTYPE( - None, ctypes.c_void_p, llama_beams_state -) - - -# /// @details Deterministically returns entire sentence constructed by a beam search. -# /// @param ctx Pointer to the llama_context. -# /// @param callback Invoked for each iteration of the beam_search loop, passing in beams_state. -# /// @param callback_data A pointer that is simply passed back to callback. -# /// @param n_beams Number of beams to use. -# /// @param n_past Number of tokens already evaluated. -# /// @param n_predict Maximum number of tokens to predict. EOS may occur earlier. -# /// @param n_threads Number of threads as passed to llama_eval(). -# LLAMA_API void llama_beam_search( -# struct llama_context * ctx, -# llama_beam_search_callback_fn_t callback, -# void * callback_data, -# size_t n_beams, -# int32_t n_past, -# int32_t n_predict); -@ctypes_function( - "llama_beam_search", - [ - llama_context_p_ctypes, - llama_beam_search_callback_fn_t, - ctypes.c_void_p, - ctypes.c_size_t, - ctypes.c_int32, - ctypes.c_int32, - ], - None, -) -def llama_beam_search( - ctx: llama_context_p, - callback: CtypesFuncPointer, - callback_data: ctypes.c_void_p, - n_beams: Union[ctypes.c_size_t, int], - n_past: Union[ctypes.c_int, int], - n_predict: Union[ctypes.c_int, int], - /, -): ... - - # /// @details Build a split GGUF final path for this chunk. # /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" # // Returns the split_path length. diff --git a/vendor/llama.cpp b/vendor/llama.cpp index bde7cd3cd9..ee459f40f6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit bde7cd3cd949c1a85d3a199498ac98e78039d46f +Subproject commit ee459f40f65810a810151b24eba5b8bd174ceffe From 83d6b26e6f45c31b5b55c4b5e5570397540d1729 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 8 Jun 2024 23:14:22 -0400 Subject: [PATCH 198/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ee459f40f6..5795b94182 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ee459f40f65810a810151b24eba5b8bd174ceffe +Subproject commit 5795b941827fdec6c1662986de962badff456718 From 1615eb9e5b74fe00eaa065520796b1af18c94557 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 10 Jun 2024 11:05:45 -0400 Subject: [PATCH 199/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 5795b94182..fd5ea0f897 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 5795b941827fdec6c1662986de962badff456718 +Subproject commit fd5ea0f897ecb3659d6c269ef6f3d833e865ead7 From 86a38ad4a02933fc7a1d86dac865b790dd7da1e6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 10 Jun 2024 11:14:33 -0400 Subject: [PATCH 200/561] chore: Bump version --- CHANGELOG.md | 7 +++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b9865da0a..7b1998502a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.78] + +- feat: Update llama.cpp to ggerganov/llama.cpp@fd5ea0f897ecb3659d6c269ef6f3d833e865ead7 +- fix: Avoid duplicate special tokens in chat formats by @CISC in #1439 +- fix: fix logprobs when BOS is not present by @ghorbani in #1471 +- feat: adding rpc_servers parameter to Llama class by @chraac in #1477 + ## [0.2.77] - feat: Update llama.cpp to ggerganov/llama.cpp@bde7cd3cd949c1a85d3a199498ac98e78039d46f diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 3df249f5eb..5dd2b43cd4 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.77" \ No newline at end of file +__version__ = "0.2.78" \ No newline at end of file From e342161371c85fa1bd68dca2727cd633480a6524 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 13 Jun 2024 03:38:11 -0400 Subject: [PATCH 201/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index fd5ea0f897..f578b86b21 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit fd5ea0f897ecb3659d6c269ef6f3d833e865ead7 +Subproject commit f578b86b2123d0f92afbaa98a031df4d4464e582 From dbcf64cf0796c27a241991ef6fd0e52283eeb462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Thu, 13 Jun 2024 09:45:24 +0200 Subject: [PATCH 202/561] feat: Support SPM infill (#1492) * Support SPM infill * typo-- * one less layer of parenthesis necessary * new required internals * manually add bos/eos if model requires it * add bos even when unknown This is identical behaviour to llama.cpp I guess any model that doesn't use BOS is recent enough to have the add_bos_token metadata. * don't add bos/eos on non-infill pre-tokenized prompt * add tokenizer hack to remove leading space in suffix * I keep forgetting metadata are strings * check if bos exists * add example * add cls/sep instead of bos/eos for WPM vocab * simplify * color-code filtered suffix --------- Co-authored-by: Andrei Betlen --- .../high_level_api/high_level_api_infill.py | 33 +++++++++ llama_cpp/_internals.py | 8 ++ llama_cpp/llama.py | 73 ++++++++++++------- 3 files changed, 87 insertions(+), 27 deletions(-) create mode 100644 examples/high_level_api/high_level_api_infill.py diff --git a/examples/high_level_api/high_level_api_infill.py b/examples/high_level_api/high_level_api_infill.py new file mode 100644 index 0000000000..27af6367e7 --- /dev/null +++ b/examples/high_level_api/high_level_api_infill.py @@ -0,0 +1,33 @@ +import argparse + +from llama_cpp import Llama + +parser = argparse.ArgumentParser() +parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin") +parser.add_argument("-p", "--prompt", type=str, default="def add(") +parser.add_argument("-s", "--suffix", type=str, default="\n return sum\n\n") +parser.add_argument("-i", "--spm-infill", action='store_true') +args = parser.parse_args() + +llm = Llama(model_path=args.model, n_gpu_layers=-1, spm_infill=args.spm_infill) + +output = llm.create_completion( + temperature = 0.0, + repeat_penalty = 1.0, + prompt = args.prompt, + suffix = args.suffix, +) + +# Models sometimes repeat suffix in response, attempt to filter that +response = output["choices"][0]["text"] +response_stripped = response.rstrip() +unwanted_response_suffix = args.suffix.rstrip() +unwanted_response_length = len(unwanted_response_suffix) + +filtered = False +if unwanted_response_suffix and response_stripped[-unwanted_response_length:] == unwanted_response_suffix: + response = response_stripped[:-unwanted_response_length] + filtered = True + +print(f"Fill-in-Middle completion{' (filtered)' if filtered else ''}:\n\n{args.prompt}\033[32m{response}\033[{'33' if filtered else '0'}m{args.suffix}\033[0m") + diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index ba04291390..27452c3241 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -170,6 +170,14 @@ def token_eot(self) -> int: assert self.model is not None return llama_cpp.llama_token_eot(self.model) + def add_bos_token(self) -> int: + assert self.model is not None + return llama_cpp.llama_add_bos_token(self.model) + + def add_eos_token(self) -> int: + assert self.model is not None + return llama_cpp.llama_add_eos_token(self.model) + # Tokenization def tokenize(self, text: bytes, add_bos: bool, special: bool): diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index bf3bd656de..f15e0ef57f 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -115,6 +115,7 @@ def __init__( type_k: Optional[int] = None, type_v: Optional[int] = None, # Misc + spm_infill: bool = False, verbose: bool = True, # Extra Params **kwargs, # type: ignore @@ -185,6 +186,7 @@ def __init__( verbose: Print verbose output to stderr. type_k: KV cache data type for K (default: f16) type_v: KV cache data type for V (default: f16) + spm_infill: Use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. Raises: ValueError: If the model path does not exist. @@ -343,6 +345,8 @@ def __init__( self.lora_scale = lora_scale self.lora_path = lora_path + self.spm_infill = spm_infill + if not os.path.exists(model_path): raise ValueError(f"Model path does not exist: {model_path}") @@ -972,14 +976,33 @@ def _create_completion( completion_id: str = f"cmpl-{str(uuid.uuid4())}" created: int = int(time.time()) + bos_token_id: int = self.token_bos() + cls_token_id: int = self._model.token_cls() + sep_token_id: int = self._model.token_sep() prefix_token_id: int = self._model.token_prefix() middle_token_id: int = self._model.token_middle() suffix_token_id: int = self._model.token_suffix() + add_space_prefix: bool = self.metadata.get("tokenizer.ggml.add_space_prefix", "true") == "true" + bos_tokens: List[int] = [cls_token_id if cls_token_id != -1 else bos_token_id] + eos_tokens: List[int] = [sep_token_id if sep_token_id != -1 else self.token_eos()] + + if (isinstance(prompt, list) and suffix is None) or self._model.add_bos_token() == 0 or bos_tokens[:1] == [-1]: + bos_tokens = [] + + if (isinstance(prompt, list) and suffix is None) or (self._model.add_eos_token() != 1 and sep_token_id == -1): + eos_tokens = [] + + suffix_space_prefix: int = 0 + # Tokenizer hack to remove leading space + if add_space_prefix and suffix_token_id >= 0 and suffix: + suffix = "☺" + suffix + suffix_space_prefix = 2 + # If prompt is empty, initialize completion with BOS token to avoid # detokenization including a space at the beginning of the completion - completion_tokens: List[int] = [] if len(prompt) > 0 else [self.token_bos()] + completion_tokens: List[int] = [] if len(prompt) > 0 else [bos_token_id] # Add blank space to start of prompt to match OG llama tokenizer - prompt_tokens: List[int] = ( + prefix_tokens: List[int] = ( ( [prefix_token_id] if prefix_token_id >= 0 and suffix is not None @@ -988,38 +1011,33 @@ def _create_completion( + ( ( - self.tokenize(prompt.encode("utf-8"), add_bos=(prefix_token_id < 0 or suffix is None), special=(prefix_token_id < 0 or suffix is None)) + self.tokenize(prompt.encode("utf-8"), add_bos=False, special=(prefix_token_id < 0 or suffix is None)) if prompt != "" - else ( - [] - if prefix_token_id >= 0 and suffix is not None - else [self.token_bos()] - ) + else [] ) if isinstance(prompt, str) else prompt ) - + + ) + suffix_tokens: List[int] = ( ( + [suffix_token_id] + + ( - [suffix_token_id] - + - ( - self.tokenize(suffix.encode("utf-8"), add_bos=False, special=False) - if suffix - else [] - ) + self.tokenize(suffix.encode("utf-8"), add_bos=False, special=False)[suffix_space_prefix:] + if suffix + else [] ) - if suffix_token_id >= 0 and suffix is not None - else [] - ) - + - ( - [middle_token_id] - if middle_token_id >= 0 and suffix is not None - else [] ) + if suffix_token_id >= 0 and suffix is not None + else [] + ) + middle_tokens: List[int] = ( + [middle_token_id] + if middle_token_id >= 0 and suffix is not None + else [] ) + prompt_tokens: List[int] = bos_tokens + ((suffix_tokens + prefix_tokens + middle_tokens) if self.spm_infill else (prefix_tokens + suffix_tokens + middle_tokens)) + eos_tokens text: bytes = b"" returned_tokens: int = 0 stop = ( @@ -1176,7 +1194,7 @@ def logit_bias_processor( # not sure how to handle this branch when dealing # with CJK output, so keep it unchanged for token in remaining_tokens: - if token == self.token_bos(): + if token == bos_token_id: continue token_end_position += len(self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens])) # Check if stop sequence is in the token @@ -1303,7 +1321,7 @@ def logit_bias_processor( logprobs_or_none: Optional[CompletionLogprobs] = None if logprobs is not None: - if token == self.token_bos(): + if token == bos_token_id: continue token_str = self.detokenize([token]).decode( "utf-8", errors="ignore" @@ -1431,7 +1449,7 @@ def logit_bias_processor( for idx, (token, token_str, logprobs_token) in enumerate( zip(all_tokens, all_token_strs, all_logprobs) ): - if token == self.token_bos(): + if token == bos_token_id: continue text_offsets.append( text_offset @@ -1858,6 +1876,7 @@ def __getstate__(self): type_k=self.context_params.type_k, type_v=self.context_params.type_v, # Misc + spm_infill=self.spm_infill, verbose=self.verbose, ) From 320a5d7ea5ebcc5a8dc114d9bdf7c584537b04b0 Mon Sep 17 00:00:00 2001 From: Junpei Kawamoto Date: Thu, 13 Jun 2024 02:16:14 -0600 Subject: [PATCH 203/561] feat: Add `.close()` method to `Llama` class to explicitly free model from memory (#1513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add explicit methods to free model This commit introduces a `close` method to both `Llama` and `_LlamaModel`, allowing users to explicitly free the model from RAM/VRAM. The previous implementation relied on the destructor of `_LlamaModel` to free the model. However, in Python, the timing of destructor calls is unclear—for instance, the `del` statement does not guarantee immediate invocation of the destructor. This commit provides an explicit method to release the model, which works immediately and allows the user to load another model without memory issues. Additionally, this commit implements a context manager in the `Llama` class, enabling the automatic closure of the `Llama` object when used with the `with` statement. * feat: Implement ContextManager in _LlamaModel, _LlamaContext, and _LlamaBatch This commit enables automatic resource management by implementing the `ContextManager` protocol in `_LlamaModel`, `_LlamaContext`, and `_LlamaBatch`. This ensures that resources are properly managed and released within a `with` statement, enhancing robustness and safety in resource handling. * feat: add ExitStack for Llama's internal class closure This update implements ExitStack to manage and close internal classes in Llama, enhancing efficient and safe resource management. * Use contextlib ExitStack and closing * Explicitly free model when closing resources on server --------- Co-authored-by: Andrei Betlen --- llama_cpp/_internals.py | 56 ++++++++++++++++++++++----------------- llama_cpp/llama.py | 21 ++++++++++----- llama_cpp/server/model.py | 3 +++ 3 files changed, 50 insertions(+), 30 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 27452c3241..ee990d4743 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -9,6 +9,7 @@ Sequence, ) from dataclasses import dataclass, field +from contextlib import ExitStack import numpy as np import numpy.typing as npt @@ -27,9 +28,6 @@ class _LlamaModel: """Intermediate Python wrapper for a llama.cpp llama_model. NOTE: For stability it's recommended you use the Llama class instead.""" - _llama_free_model = None - # NOTE: this must be "saved" here to avoid exceptions when calling __del__ - def __init__( self, *, @@ -40,8 +38,7 @@ def __init__( self.path_model = path_model self.params = params self.verbose = verbose - - self._llama_free_model = llama_cpp._lib.llama_free_model # type: ignore + self._exit_stack = ExitStack() self.model = None @@ -56,11 +53,17 @@ def __init__( if self.model is None: raise ValueError(f"Failed to load model from file: {path_model}") - def __del__(self): - if self.model is not None and self._llama_free_model is not None: - self._llama_free_model(self.model) + def free_model(): + if self.model is None: + return + llama_cpp.llama_free_model(self.model) self.model = None + self._exit_stack.callback(free_model) + + def close(self): + self._exit_stack.close() + def vocab_type(self) -> int: assert self.model is not None return llama_cpp.llama_vocab_type(self.model) @@ -257,8 +260,6 @@ class _LlamaContext: """Intermediate Python wrapper for a llama.cpp llama_context. NOTE: For stability it's recommended you use the Llama class instead.""" - _llama_free = None - def __init__( self, *, @@ -269,24 +270,28 @@ def __init__( self.model = model self.params = params self.verbose = verbose + self._exit_stack = ExitStack() - self._llama_free = llama_cpp._lib.llama_free # type: ignore self.ctx = None assert self.model.model is not None - self.ctx = llama_cpp.llama_new_context_with_model( - self.model.model, self.params - ) + self.ctx = llama_cpp.llama_new_context_with_model(self.model.model, self.params) if self.ctx is None: raise ValueError("Failed to create llama_context") - def __del__(self): - if self.ctx is not None and self._llama_free is not None: - self._llama_free(self.ctx) + def free_ctx(): + if self.ctx is None: + return + llama_cpp.llama_free(self.ctx) self.ctx = None + self._exit_stack.callback(free_ctx) + + def close(self): + self._exit_stack.close() + def n_ctx(self) -> int: assert self.ctx is not None return llama_cpp.llama_n_ctx(self.ctx) @@ -501,8 +506,6 @@ def default_params(): class _LlamaBatch: - _llama_batch_free = None - def __init__( self, *, n_tokens: int, embd: int, n_seq_max: int, verbose: bool = True ): @@ -510,19 +513,24 @@ def __init__( self.embd = embd self.n_seq_max = n_seq_max self.verbose = verbose - - self._llama_batch_free = llama_cpp._lib.llama_batch_free # type: ignore + self._exit_stack = ExitStack() self.batch = None self.batch = llama_cpp.llama_batch_init( self._n_tokens, self.embd, self.n_seq_max ) - def __del__(self): - if self.batch is not None and self._llama_batch_free is not None: - self._llama_batch_free(self.batch) + def free_batch(): + if self.batch is None: + return + llama_cpp.llama_batch_free(self.batch) self.batch = None + self._exit_stack.callback(free_batch) + + def close(self): + self._exit_stack.close() + def n_tokens(self) -> int: assert self.batch is not None return self.batch.n_tokens diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index f15e0ef57f..459b29f920 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -9,7 +9,9 @@ import typing import fnmatch import warnings +import contextlib import multiprocessing +from types import TracebackType from typing import ( List, @@ -21,6 +23,7 @@ Deque, Callable, Dict, + Type, ) from collections import deque from pathlib import Path @@ -350,9 +353,11 @@ def __init__( if not os.path.exists(model_path): raise ValueError(f"Model path does not exist: {model_path}") - self._model = _LlamaModel( + self._stack = contextlib.ExitStack() + + self._model = self._stack.enter_context(contextlib.closing(_LlamaModel( path_model=self.model_path, params=self.model_params, verbose=self.verbose - ) + ))) # Override tokenizer self.tokenizer_ = tokenizer or LlamaTokenizer(self) @@ -364,18 +369,18 @@ def __init__( self.context_params.n_ctx = self._model.n_ctx_train() self.context_params.n_batch = self.n_batch - self._ctx = _LlamaContext( + self._ctx = self._stack.enter_context(contextlib.closing(_LlamaContext( model=self._model, params=self.context_params, verbose=self.verbose, - ) + ))) - self._batch = _LlamaBatch( + self._batch = self._stack.enter_context(contextlib.closing(_LlamaBatch( n_tokens=self.n_batch, embd=0, n_seq_max=self.context_params.n_ctx, verbose=self.verbose, - ) + ))) if self.lora_path: if self._model.apply_lora_from_file( @@ -1959,6 +1964,10 @@ def pooling_type(self) -> str: """Return the pooling type.""" return self._ctx.pooling_type() + def close(self) -> None: + """Explicitly free the model from memory.""" + self._stack.close() + @staticmethod def logits_to_logprobs( logits: Union[npt.NDArray[np.single], List], axis: int = -1 diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index d4d4acbe3f..ad39c1004b 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -44,6 +44,8 @@ def __call__(self, model: Optional[str] = None) -> llama_cpp.Llama: if self._current_model is not None: return self._current_model + if self._current_model: + self._current_model.close() self._current_model = None settings = self._model_settings_dict[model] @@ -65,6 +67,7 @@ def __iter__(self): def free(self): if self._current_model: + self._current_model.close() del self._current_model @staticmethod From 5af81634cb0c47943f5653275b04f7747b89792d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 10:12:02 -0400 Subject: [PATCH 204/561] chore(deps): bump pypa/cibuildwheel from 2.18.1 to 2.19.0 (#1522) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.18.1 to 2.19.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.18.1...v2.19.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 957912d09d..ebf52b3370 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -29,7 +29,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.18.1 + uses: pypa/cibuildwheel@v2.19.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -56,7 +56,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.18.1 + uses: pypa/cibuildwheel@v2.19.0 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" From 9e396b3ebd997e3155b5bb8ab3ab162dc26da5c5 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 13 Jun 2024 16:19:57 +0200 Subject: [PATCH 205/561] feat: Update workflows and pre-built wheels (#1416) * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * revert * Bump pyhton from 3.8 to 3.9 * Remove python 3.8 * Remove Python 3.7 and 3.8 deprecated * Bump python from 3.8 to 3.9 * Add python 3.9 * Add python 3.9, remove macos-11 deprecated, add macos-14 * Bump python 3.8 to 3.9 * Add python 3.13 * Add python 3.13 * python 3.13 remove * remove python 3.13 * remove python 3.8 * Bump macos-13 to macos-14 * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update generate-index-from-release.yaml Add avx, avx2 and avx512 * Update test.yaml * Update test-pypi.yaml * Update publish.yaml * Update publish-to-test.yaml * Update build-wheels-cuda.yaml Cuda with AVX2 by default * Update build-wheels-cuda.yaml * remove DEPRECATED 32 bits * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml Upgrade matrix os to latest version * Update build-wheels-metal.yaml * Update build-wheels-cuda.yaml * Update test.yaml * Update test-pypi.yaml * Update test.yaml Add cache: 'pip' * Update publish-to-test.yaml * Update build-wheels-metal.yaml Add cache: 'pip' * Update build-wheels-cuda.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml remove x86_64 * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * revert * Remove cpu variants --------- Co-authored-by: Andrei Betlen --- .github/workflows/build-wheels-cuda.yaml | 23 +++--- .github/workflows/build-wheels-metal.yaml | 91 +++++++++-------------- .github/workflows/publish-to-test.yaml | 9 ++- .github/workflows/publish.yaml | 8 +- .github/workflows/test-pypi.yaml | 27 ++++--- .github/workflows/test.yaml | 47 ++++++------ 6 files changed, 96 insertions(+), 109 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index ae9e8632c0..b8496d8508 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -20,8 +20,8 @@ jobs: id: set-matrix run: | $matrix = @{ - 'os' = @('ubuntu-20.04', 'windows-latest') - 'pyver' = @("3.10", "3.11", "3.12") + 'os' = @('ubuntu-latest', 'windows-latest') + 'pyver' = @("3.9", "3.10", "3.11", "3.12") 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") 'releasetag' = @("basic") } @@ -50,6 +50,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.pyver }} + cache: 'pip' - name: Setup Mamba uses: conda-incubator/setup-miniconda@v3.0.4 @@ -109,15 +110,15 @@ jobs: $env:VERBOSE = '1' $env:CMAKE_ARGS = '-DLLAMA_CUBLAS=on -DCMAKE_CUDA_ARCHITECTURES=all' $env:CMAKE_ARGS = "-DLLAMA_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS" - if ($env:AVXVER -eq 'AVX') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off' - } - if ($env:AVXVER -eq 'AVX512') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX512=on' - } - if ($env:AVXVER -eq 'basic') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off' - } + # if ($env:AVXVER -eq 'AVX') { + $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off' + # } + # if ($env:AVXVER -eq 'AVX512') { + # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX512=on' + # } + # if ($env:AVXVER -eq 'basic') { + # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off' + # } python -m build --wheel # write the build tag to the output Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index fc798c84e7..f007eb3981 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -6,81 +6,60 @@ permissions: contents: write jobs: - define_matrix: - name: Define Build Matrix - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - defaults: - run: - shell: pwsh - - steps: - - name: Define Job Output - id: set-matrix - run: | - $matrix = @{ - 'os' = @('macos-11', 'macos-12', 'macos-13') - 'pyver' = @('3.10', '3.11', '3.12') - } - - $matrixOut = ConvertTo-Json $matrix -Compress - Write-Output ('matrix=' + $matrixOut) >> $env:GITHUB_OUTPUT - build_wheels: - name: ${{ matrix.os }} Python ${{ matrix.pyver }} - needs: define_matrix + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: - matrix: ${{ fromJSON(needs.define_matrix.outputs.matrix) }} - env: - OSVER: ${{ matrix.os }} + matrix: + os: [macos-12, macos-13, macos-14] steps: - uses: actions/checkout@v4 with: submodules: "recursive" + # Used to host cibuildwheel - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.pyver }} - - - name: Install Dependencies - run: | - python -m pip install build wheel cmake + python-version: "3.12" + cache: 'pip' - - name: Build Wheel + - name: Install dependencies run: | - XCODE15PATH="/Applications/Xcode_15.0.app/Contents/Developer" - XCODE15BINPATH="${XCODE15PATH}/Toolchains/XcodeDefault.xctoolchain/usr/bin" - export CMAKE_ARGS="-DLLAMA_NATIVE=off -DLLAMA_METAL=on" - [[ "$OSVER" == "macos-13" ]] && export CC="${XCODE15BINPATH}/cc" && export CXX="${XCODE15BINPATH}/c++" && export MACOSX_DEPLOYMENT_TARGET="13.0" - [[ "$OSVER" == "macos-12" ]] && export MACOSX_DEPLOYMENT_TARGET="12.0" - [[ "$OSVER" == "macos-11" ]] && export MACOSX_DEPLOYMENT_TARGET="11.0" + python -m pip install --upgrade pip + python -m pip install -e .[all] - export CMAKE_OSX_ARCHITECTURES="arm64" && export ARCHFLAGS="-arch arm64" - VERBOSE=1 python -m build --wheel - - if [[ "$OSVER" == "macos-13" ]]; then - export SDKROOT="${XCODE15PATH}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.0.sdk" - export MACOSX_DEPLOYMENT_TARGET="14.0" - VERBOSE=1 python -m build --wheel - fi - - for file in ./dist/*.whl; do cp "$file" "${file/arm64.whl/aarch64.whl}"; done + - name: Build wheels + uses: pypa/cibuildwheel@v2.18.1 + env: + # disable repair + CIBW_REPAIR_WHEEL_COMMAND: "" + CIBW_ARCHS: "arm64" + CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DLLAMA_METAL=on" + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" + with: + package-dir: . + output-dir: wheelhouse2 - export CMAKE_OSX_ARCHITECTURES="x86_64" && export CMAKE_ARGS="-DLLAMA_NATIVE=off -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off -DLLAMA_METAL=on" && export ARCHFLAGS="-arch x86_64" - VERBOSE=1 python -m build --wheel + - uses: actions/upload-artifact@v4 + with: + name: wheels-mac_${{ matrix.os }} + path: ./wheelhouse2/*.whl - if [[ "$OSVER" == "macos-13" ]]; then - export SDKROOT="${XCODE15PATH}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX14.0.sdk" - export MACOSX_DEPLOYMENT_TARGET="14.0" - VERBOSE=1 python -m build --wheel - fi + release: + name: Release + needs: [build_wheels] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: dist2 + - uses: softprops/action-gh-release@v2 with: - files: dist/* + files: dist2/* # set release name to -metal tag_name: ${{ github.ref_name }}-metal env: diff --git a/.github/workflows/publish-to-test.yaml b/.github/workflows/publish-to-test.yaml index 2bf0ea9ba4..19613233be 100644 --- a/.github/workflows/publish-to-test.yaml +++ b/.github/workflows/publish-to-test.yaml @@ -22,7 +22,8 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.11" + cache: 'pip' - name: Append Dev Version to __version__ run: | DEV_VERSION=${{ github.event.inputs.dev_version }} @@ -31,11 +32,11 @@ jobs: sed -i 's/__version__ = \".*\"/__version__ = \"'"${NEW_VERSION}"'\"/' llama_cpp/__init__.py - name: Install dependencies run: | - python3 -m pip install --upgrade pip build - python3 -m pip install -e .[all] + python -m pip install --upgrade pip build + python -m pip install -e .[all] - name: Build source distribution run: | - python3 -m build --sdist + python -m build --sdist - name: Publish to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index bc4ec9063b..c6abb43b34 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -16,14 +16,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.9" - name: Install dependencies run: | - python3 -m pip install --upgrade pip build - python3 -m pip install -e .[all] + python -m pip install --upgrade pip build + python -m pip install -e .[all] - name: Build source distribution run: | - python3 -m build --sdist + python -m build --sdist - name: Publish distribution to PyPI # TODO: move to tag based releases # if: startsWith(github.ref, 'refs/tags') diff --git a/.github/workflows/test-pypi.yaml b/.github/workflows/test-pypi.yaml index aa8e8fa0b0..d7131956d1 100644 --- a/.github/workflows/test-pypi.yaml +++ b/.github/workflows/test-pypi.yaml @@ -8,57 +8,60 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | - python3 -m pip install --upgrade pip - python3 -m pip install --verbose llama-cpp-python[all] + python -m pip install --upgrade pip + python -m pip install --verbose llama-cpp-python[all] - name: Test with pytest run: | - python3 -c "import llama_cpp" + python -c "import llama_cpp" build-windows: runs-on: windows-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | - python3 -m pip install --upgrade pip - python3 -m pip install --verbose llama-cpp-python[all] + python -m pip install --upgrade pip + python -m pip install --verbose llama-cpp-python[all] - name: Test with pytest run: | - python3 -c "import llama_cpp" + python -c "import llama_cpp" build-macos: runs-on: macos-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | - python3 -m pip install --upgrade pip - python3 -m pip install --verbose llama-cpp-python[all] + python -m pip install --upgrade pip + python -m pip install --verbose llama-cpp-python[all] - name: Test with pytest run: | - python3 -c "import llama_cpp" + python -c "import llama_cpp" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 292343c2bf..6d8231d186 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -24,20 +24,21 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | - python3 -m pip install --upgrade pip - python3 -m pip install .[all] -v + python -m pip install --upgrade pip + python -m pip install .[all] -v - name: Test with pytest run: | - python3 -m pytest + python -m pytest build-windows: runs-on: windows-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -47,20 +48,21 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | - python3 -m pip install --upgrade pip - python3 -m pip install .[all] -v + python -m pip install --upgrade pip + python -m pip install .[all] -v - name: Test with pytest run: | - python3 -m pytest + python -m pytest build-macos: - runs-on: macos-13 + runs-on: macos-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -70,13 +72,14 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' - name: Install dependencies run: | - python3 -m pip install --upgrade pip - python3 -m pip install .[all] --verbose + python -m pip install --upgrade pip + python -m pip install .[all] --verbose - name: Test with pytest run: | - python3 -m pytest + python -m pytest # build-linux-opencl: @@ -98,29 +101,29 @@ jobs: # sudo apt-get install -y --no-install-recommends llvm intel-oneapi-runtime-opencl intel-oneapi-runtime-compilers libclblast-dev # - name: Install dependencies # run: | - # python3 -m pip install --upgrade pip - # CMAKE_ARGS="-DLLAMA_CLBLAST=on" python3 -m pip install .[all] --verbose + # python -m pip install --upgrade pip + # CMAKE_ARGS="-DLLAMA_CLBLAST=on" python -m pip install .[all] --verbose # - name: Test with pytest # run: | - # python3 -m pytest + # python -m pytest build-macos-metal: - runs-on: macos-13 + runs-on: macos-latest steps: - uses: actions/checkout@v4 with: submodules: "recursive" - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.9" - name: Install dependencies run: | - python3 -m pip install --upgrade pip - CMAKE_ARGS="-DLLAMA_METAL=on" python3 -m pip install .[all] --verbose + python -m pip install --upgrade pip + CMAKE_ARGS="-DLLAMA_METAL=on" python -m pip install .[all] --verbose - name: Test with pytest run: | - python3 -m pytest + python -m pytest From 8401c6f2d133543350e14de6076fe7ec9efe6653 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 13 Jun 2024 11:31:31 -0400 Subject: [PATCH 206/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f578b86b21..172c825684 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f578b86b2123d0f92afbaa98a031df4d4464e582 +Subproject commit 172c8256840ffd882ab9992ecedbb587d9b21f15 From f4491c4903f791b74de9e98e9dc03b87fcc9dfbb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 17 Jun 2024 11:56:03 -0400 Subject: [PATCH 207/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 2 ++ vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 6f37fcb69c..6d24324f01 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -301,6 +301,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_OLMO = 12, # LLAMA_VOCAB_PRE_TYPE_DBRX = 13, # LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, +# LLAMA_VOCAB_PRE_TYPE_PORO = 15, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -317,6 +318,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_OLMO = 12 LLAMA_VOCAB_PRE_TYPE_DBRX = 13 LLAMA_VOCAB_PRE_TYPE_SMAUG = 14 +LLAMA_VOCAB_PRE_TYPE_PORO = 15 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 172c825684..b473e95084 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 172c8256840ffd882ab9992ecedbb587d9b21f15 +Subproject commit b473e95084c286780165568cf0f385f21141d68d From 4c1d74c0aecb5fe263674cc761573654b16d4c20 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 19 Jun 2024 10:07:20 -0400 Subject: [PATCH 208/561] fix: Make destructor to automatically call .close() method on Llama class. --- llama_cpp/_internals.py | 9 +++++++++ llama_cpp/llama.py | 3 +++ 2 files changed, 12 insertions(+) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index ee990d4743..423c1b5bcd 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -64,6 +64,9 @@ def free_model(): def close(self): self._exit_stack.close() + def __del__(self): + self.close() + def vocab_type(self) -> int: assert self.model is not None return llama_cpp.llama_vocab_type(self.model) @@ -292,6 +295,9 @@ def free_ctx(): def close(self): self._exit_stack.close() + def __del__(self): + self.close() + def n_ctx(self) -> int: assert self.ctx is not None return llama_cpp.llama_n_ctx(self.ctx) @@ -531,6 +537,9 @@ def free_batch(): def close(self): self._exit_stack.close() + def __del__(self): + self.close() + def n_tokens(self) -> int: assert self.batch is not None return self.batch.n_tokens diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 459b29f920..5f5396683c 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1968,6 +1968,9 @@ def close(self) -> None: """Explicitly free the model from memory.""" self._stack.close() + def __del__(self) -> None: + self.close() + @staticmethod def logits_to_logprobs( logits: Union[npt.NDArray[np.single], List], axis: int = -1 From 554fd08e7d90f1c49cc513febdc79cc723ceb512 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 19 Jun 2024 10:07:28 -0400 Subject: [PATCH 209/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b473e95084..9c77ec1d74 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b473e95084c286780165568cf0f385f21141d68d +Subproject commit 9c77ec1d74874ee22bdef8f110e8e8d41389abf2 From 6c331909ca365cb5aa490288a02853f1bb1d6979 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 19 Jun 2024 10:10:01 -0400 Subject: [PATCH 210/561] chore: Bump version --- CHANGELOG.md | 7 +++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b1998502a..42b14c862f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.79] + +- feat: Update llama.cpp to ggerganov/llama.cpp@9c77ec1d74874ee22bdef8f110e8e8d41389abf2 +- feat(ci): Update workflows and pre-built wheels by @Smartappli in #1416 +- feat: Add .close() method to Llama class to explicitly free model from memory by @jkawamoto in #1513 +- feat: Support SPM infill by @CISC in #1492 + ## [0.2.78] - feat: Update llama.cpp to ggerganov/llama.cpp@fd5ea0f897ecb3659d6c269ef6f3d833e865ead7 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 5dd2b43cd4..71371b3b07 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.78" \ No newline at end of file +__version__ = "0.2.79" \ No newline at end of file From d98a24a25b49b02007071dc3e4bc2cc9a0212935 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 20 Jun 2024 10:50:40 -0400 Subject: [PATCH 211/561] docs: Remove references to deprecated opencl backend. Closes #1512 --- Makefile | 3 --- README.md | 11 ----------- 2 files changed, 14 deletions(-) diff --git a/Makefile b/Makefile index d8fb0cc27b..90f562fa19 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,6 @@ build.debug: build.cuda: CMAKE_ARGS="-DLLAMA_CUDA=on" python3 -m pip install --verbose -e . -build.opencl: - CMAKE_ARGS="-DLLAMA_CLBLAST=on" python3 -m pip install --verbose -e . - build.openblas: CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" python3 -m pip install --verbose -e . diff --git a/README.md b/README.md index 0f7abfb28d..4c37ba3e4a 100644 --- a/README.md +++ b/README.md @@ -165,17 +165,6 @@ pip install llama-cpp-python \ --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/metal ``` - -
- -CLBlast (OpenCL) - -To install with CLBlast, set the `LLAMA_CLBLAST=on` environment variable before installing: - -```bash -CMAKE_ARGS="-DLLAMA_CLBLAST=on" pip install llama-cpp-python -``` -
From 5beec1a1fd28acf6e32d45a6b990ef6389d288ed Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 21 Jun 2024 12:09:14 -0400 Subject: [PATCH 212/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 9c77ec1d74..557b653dc9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 9c77ec1d74874ee22bdef8f110e8e8d41389abf2 +Subproject commit 557b653dc9ed91e8c313e87500e0050c775f81b6 From 27d53589ff884559919b4bd2c40690223feaaffa Mon Sep 17 00:00:00 2001 From: Jon Craton Date: Fri, 21 Jun 2024 12:10:15 -0400 Subject: [PATCH 213/561] docs: Update readme examples to use newer Qwen2 model (#1544) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c37ba3e4a..4a710d4e56 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,7 @@ You'll need to install the `huggingface-hub` package to use this feature (`pip i ```python llm = Llama.from_pretrained( - repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF", + repo_id="Qwen/Qwen2-0.5B-Instruct-GGUF", filename="*q8_0.gguf", verbose=False ) @@ -688,7 +688,7 @@ For possible options, see [llama_cpp/llama_chat_format.py](llama_cpp/llama_chat_ If you have `huggingface-hub` installed, you can also use the `--hf_model_repo_id` flag to load a model from the Hugging Face Hub. ```bash -python3 -m llama_cpp.server --hf_model_repo_id Qwen/Qwen1.5-0.5B-Chat-GGUF --model '*q8_0.gguf' +python3 -m llama_cpp.server --hf_model_repo_id Qwen/Qwen2-0.5B-Instruct-GGUF --model '*q8_0.gguf' ``` ### Web Server Features From 398fe81547d2b36a57730d19e66fb69184054958 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jun 2024 12:10:34 -0400 Subject: [PATCH 214/561] chore(deps): bump docker/build-push-action from 5 to 6 (#1539) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5 to 6. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5...v6) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 4ebe3bb6d7..b5c7346db4 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -31,7 +31,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: "docker/simple/Dockerfile" From 35c980eb2ece665a8d3b636968358bd82f3f27bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jun 2024 12:10:43 -0400 Subject: [PATCH 215/561] chore(deps): bump pypa/cibuildwheel from 2.18.1 to 2.19.1 (#1527) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.18.1 to 2.19.1. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.18.1...v2.19.1) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 4 ++-- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index ebf52b3370..34e9dc139f 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -29,7 +29,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.19.0 + uses: pypa/cibuildwheel@v2.19.1 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -56,7 +56,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.19.0 + uses: pypa/cibuildwheel@v2.19.1 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index f007eb3981..11ab795c8e 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -30,7 +30,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.18.1 + uses: pypa/cibuildwheel@v2.19.1 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" From 04959f1884c8ef93bd5a4aa40ff0accb8438c0c1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 21 Jun 2024 16:56:15 -0400 Subject: [PATCH 216/561] feat: Update llama_cpp.py bindings --- llama_cpp/llama_cpp.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 6d24324f01..1116b1f51f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -468,11 +468,13 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_POOLING_TYPE_NONE = 0, # LLAMA_POOLING_TYPE_MEAN = 1, # LLAMA_POOLING_TYPE_CLS = 2, +# LLAMA_POOLING_TYPE_LAST = 3, # }; LLAMA_POOLING_TYPE_UNSPECIFIED = -1 LLAMA_POOLING_TYPE_NONE = 0 LLAMA_POOLING_TYPE_MEAN = 1 LLAMA_POOLING_TYPE_CLS = 2 +LLAMA_POOLING_TYPE_LAST = 3 # enum llama_split_mode { # LLAMA_SPLIT_MODE_NONE = 0, // single GPU @@ -761,7 +763,6 @@ class llama_model_params(ctypes.Structure): # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` # enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id -# // (ignored if no pooling layer) # // ref: https://github.com/ggerganov/llama.cpp/pull/2054 # float rope_freq_base; // RoPE base frequency, 0 = from model @@ -2316,6 +2317,16 @@ def llama_n_threads_batch(ctx: llama_context_p, /) -> int: ... +# // Set whether the model is in embeddings model or not +# // If true, embeddings will be returned but logits will not +# LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); +@ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None) +def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /): + """Set whether the model is in embeddings model or not + If true, embeddings will be returned but logits will not""" + ... + + # // Set whether to use causal attention or not # // If set to true, the model will only attend to the past tokens # LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); From 117cbb2f531eee1db284440097866be7d7b153c4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 Jul 2024 21:28:11 -0400 Subject: [PATCH 217/561] feat: Update llama.cpp --- CMakeLists.txt | 69 ++++++++++++++++++++++-------------------- llama_cpp/llama_cpp.py | 7 ++++- vendor/llama.cpp | 2 +- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b9508d407..353a8d9960 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,9 +5,28 @@ project(llama_cpp) option(LLAMA_BUILD "Build llama.cpp shared library and install alongside python package" ON) option(LLAVA_BUILD "Build llava shared library and install alongside python package" ON) +if(SKBUILD_STATE STREQUAL "editable") + # Install into the source directory + # Temporary fix for https://github.com/scikit-build/scikit-build-core/issues/374 + set(LLAMA_CPP_PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp) +else() + set(LLAMA_CPP_PYTHON_INSTALL_DIR ${SKBUILD_PLATLIB_DIR}/llama_cpp) +endif() + if (LLAMA_BUILD) set(BUILD_SHARED_LIBS "On") + set(CMAKE_SKIP_BUILD_RPATH FALSE) + + # When building, don't use the install RPATH already + # (but later on when installing) + set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) + + # Add the automatically determined parts of the RPATH + # which point to directories outside the build tree to the install RPATH + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + set(CMAKE_INSTALL_RPATH "${LLAMA_CPP_PYTHON_INSTALL_DIR}") + # Building llama if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") # Need to disable these llama.cpp flags on Apple x86_64, @@ -24,31 +43,26 @@ if (LLAMA_BUILD) add_subdirectory(vendor/llama.cpp) install( - TARGETS llama - LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - RUNTIME DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - ARCHIVE DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - FRAMEWORK DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - RESOURCE DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp + TARGETS llama + LIBRARY DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + RUNTIME DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + ARCHIVE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + FRAMEWORK DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + RESOURCE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} ) - # Temporary fix for https://github.com/scikit-build/scikit-build-core/issues/374 install( - TARGETS llama - LIBRARY DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp - RUNTIME DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp - ARCHIVE DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp - FRAMEWORK DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp - RESOURCE DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp + TARGETS ggml + LIBRARY DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + RUNTIME DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + ARCHIVE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + FRAMEWORK DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + RESOURCE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} ) # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563 if (WIN32 AND (LLAMA_CUDA OR LLAMA_CUBLAS)) install( FILES $ - DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - ) - install( - FILES $ - DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp + DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} ) endif() @@ -71,20 +85,11 @@ if (LLAMA_BUILD) endif() install( TARGETS llava_shared - LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - RUNTIME DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - ARCHIVE DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - FRAMEWORK DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - RESOURCE DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp - ) - # Temporary fix for https://github.com/scikit-build/scikit-build-core/issues/374 - install( - TARGETS llava_shared - LIBRARY DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp - RUNTIME DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp - ARCHIVE DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp - FRAMEWORK DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp - RESOURCE DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp + LIBRARY DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + RUNTIME DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + ARCHIVE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + FRAMEWORK DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + RESOURCE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} ) endif() endif() diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1116b1f51f..9014e0268f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -273,6 +273,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback # LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE # LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece +# LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram # }; LLAMA_VOCAB_TYPE_NONE = 0 """For models without vocab""" @@ -282,6 +283,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa """GPT-2 tokenizer based on byte-level BPE""" LLAMA_VOCAB_TYPE_WPM = 3 """BERT tokenizer based on WordPiece""" +LLAMA_VOCAB_TYPE_UGM = 4 +"""T5 tokenizer based on Unigram""" # // pre-tokenization types @@ -302,6 +305,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_DBRX = 13, # LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, # LLAMA_VOCAB_PRE_TYPE_PORO = 15, +# LLAMA_VOCAB_PRE_TYPE_VIKING = 16, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -319,6 +323,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_DBRX = 13 LLAMA_VOCAB_PRE_TYPE_SMAUG = 14 LLAMA_VOCAB_PRE_TYPE_PORO = 15 +LLAMA_VOCAB_PRE_TYPE_VIKING = 16 # // note: these values should be synchronized with ggml_rope @@ -2317,7 +2322,7 @@ def llama_n_threads_batch(ctx: llama_context_p, /) -> int: ... -# // Set whether the model is in embeddings model or not +# // Set whether the model is in embeddings mode or not # // If true, embeddings will be returned but logits will not # LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); @ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 557b653dc9..5fac350b9c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 557b653dc9ed91e8c313e87500e0050c775f81b6 +Subproject commit 5fac350b9cc49d0446fc291b9c4ad53666c77591 From bf5e0bb4b151f4ca2f5a21af68eb832a96a79d75 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 Jul 2024 21:29:13 -0400 Subject: [PATCH 218/561] fix(server): Update `embeddings=False` by default. Embeddings should be enabled by default for embedding models. --- llama_cpp/server/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 4d924f3373..9d9b533d52 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -96,7 +96,7 @@ class ModelSettings(BaseSettings): default=True, description="if true, use experimental mul_mat_q kernels" ) logits_all: bool = Field(default=True, description="Whether to return logits.") - embedding: bool = Field(default=True, description="Whether to use embeddings.") + embedding: bool = Field(default=False, description="Whether to use embeddings.") offload_kqv: bool = Field( default=True, description="Whether to offload kqv to the GPU." ) From 73ddf297bea8769ae0d745bb8b88e1b059edb92e Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Mon, 1 Jul 2024 22:31:25 -0300 Subject: [PATCH 219/561] fix(ci): Fix the CUDA workflow (#1551) --- .github/workflows/build-wheels-cuda.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index b8496d8508..31c8619051 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -20,7 +20,7 @@ jobs: id: set-matrix run: | $matrix = @{ - 'os' = @('ubuntu-latest', 'windows-latest') + 'os' = @('ubuntu-latest', 'windows-2019') 'pyver' = @("3.9", "3.10", "3.11", "3.12") 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") 'releasetag' = @("basic") @@ -43,6 +43,12 @@ jobs: AVXVER: ${{ matrix.releasetag }} steps: + - name: Add MSBuild to PATH + if: runner.os == 'Windows' + uses: microsoft/setup-msbuild@v1.1 + with: + vs-version: '[16.11,16.12)' + - uses: actions/checkout@v4 with: submodules: "recursive" @@ -85,7 +91,7 @@ jobs: if: runner.os == 'Windows' run: | $y = (gi '.\MSBuildExtensions').fullname + '\*' - (gi 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Microsoft\VC\*\BuildCustomizations').fullname.foreach({cp $y $_}) + (gi 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\*\BuildCustomizations').fullname.foreach({cp $y $_}) $cupath = 'CUDA_PATH_V' + $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','_') echo "$cupath=$env:CONDA_PREFIX" >> $env:GITHUB_ENV From c546c94b48061dd22ecab8fe5309ded750ae9fb9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 Jul 2024 22:03:19 -0400 Subject: [PATCH 220/561] misc: Install shared libraries to lib subdirectory --- CMakeLists.txt | 6 ++++-- llama_cpp/llama_cpp.py | 2 +- llama_cpp/llava_cpp.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 353a8d9960..02adc445f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,11 +8,13 @@ option(LLAVA_BUILD "Build llava shared library and install alongside python pack if(SKBUILD_STATE STREQUAL "editable") # Install into the source directory # Temporary fix for https://github.com/scikit-build/scikit-build-core/issues/374 - set(LLAMA_CPP_PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp) + set(LLAMA_CPP_PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib) else() - set(LLAMA_CPP_PYTHON_INSTALL_DIR ${SKBUILD_PLATLIB_DIR}/llama_cpp) + set(LLAMA_CPP_PYTHON_INSTALL_DIR ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib) endif() +message(STATUS "LLAMA_CPP_PYTHON_INSTALL_DIR: ${LLAMA_CPP_PYTHON_INSTALL_DIR}") + if (LLAMA_BUILD) set(BUILD_SHARED_LIBS "On") diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 9014e0268f..1999b9a403 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -23,7 +23,7 @@ # Load the library def _load_shared_library(lib_base_name: str): # Construct the paths to the possible shared library names - _base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) + _base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" # Searching for the library in the current directory under the name "libllama" (default name # for llamacpp) and "llama" (default name for this repo) _lib_paths: List[pathlib.Path] = [] diff --git a/llama_cpp/llava_cpp.py b/llama_cpp/llava_cpp.py index 3ded1f25d3..b80d859133 100644 --- a/llama_cpp/llava_cpp.py +++ b/llama_cpp/llava_cpp.py @@ -35,7 +35,7 @@ # Load the library def _load_shared_library(lib_base_name: str): # Construct the paths to the possible shared library names - _base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) + _base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" # Searching for the library in the current directory under the name "libllama" (default name # for llamacpp) and "llama" (default name for this repo) _lib_paths: List[pathlib.Path] = [] From 139774b8b0a0d15101c60fbe5dab5fa3e4428b95 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 Jul 2024 22:21:34 -0400 Subject: [PATCH 221/561] fix: Update shared library rpath --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 02adc445f9..6ad49f3d20 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ if (LLAMA_BUILD) # Add the automatically determined parts of the RPATH # which point to directories outside the build tree to the install RPATH set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - set(CMAKE_INSTALL_RPATH "${LLAMA_CPP_PYTHON_INSTALL_DIR}") + set(CMAKE_INSTALL_RPATH "$ORIGIN") # Building llama if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") From d5f6a15a9b390807b7d7e023b1d7363294dc02f3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 Jul 2024 23:03:26 -0400 Subject: [PATCH 222/561] fix: force $ORIGIN rpath for shared library files --- CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ad49f3d20..b0553cd246 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ if (LLAMA_BUILD) # which point to directories outside the build tree to the install RPATH set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) set(CMAKE_INSTALL_RPATH "$ORIGIN") + set(CMAKE_SKIP_RPATH FALSE) # Building llama if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") @@ -52,6 +53,10 @@ if (LLAMA_BUILD) FRAMEWORK DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} RESOURCE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} ) + set_target_properties(llama PROPERTIES + INSTALL_RPATH "$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE + ) install( TARGETS ggml LIBRARY DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} @@ -60,6 +65,10 @@ if (LLAMA_BUILD) FRAMEWORK DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} RESOURCE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} ) + set_target_properties(ggml PROPERTIES + INSTALL_RPATH "$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE + ) # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563 if (WIN32 AND (LLAMA_CUDA OR LLAMA_CUBLAS)) install( @@ -93,5 +102,9 @@ if (LLAMA_BUILD) FRAMEWORK DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} RESOURCE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} ) + set_target_properties(llava PROPERTIES + INSTALL_RPATH "$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE + ) endif() endif() From e51f200f2c2faea42286828a90ad4d838830e7a2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 Jul 2024 23:11:49 -0400 Subject: [PATCH 223/561] fix: Fix installation location for shared libraries --- CMakeLists.txt | 70 ++++++++++++++++++-------------------------------- 1 file changed, 25 insertions(+), 45 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b0553cd246..fc58cb884f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,15 +5,28 @@ project(llama_cpp) option(LLAMA_BUILD "Build llama.cpp shared library and install alongside python package" ON) option(LLAVA_BUILD "Build llava shared library and install alongside python package" ON) -if(SKBUILD_STATE STREQUAL "editable") - # Install into the source directory - # Temporary fix for https://github.com/scikit-build/scikit-build-core/issues/374 - set(LLAMA_CPP_PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib) -else() - set(LLAMA_CPP_PYTHON_INSTALL_DIR ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib) -endif() - -message(STATUS "LLAMA_CPP_PYTHON_INSTALL_DIR: ${LLAMA_CPP_PYTHON_INSTALL_DIR}") +function(llama_cpp_python_install_target target) + install( + TARGETS ${target} + LIBRARY DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + RUNTIME DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + ARCHIVE DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + FRAMEWORK DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + RESOURCE DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + ) + install( + TARGETS ${target} + LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib + RUNTIME DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib + ARCHIVE DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib + FRAMEWORK DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib + RESOURCE DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib + ) + set_target_properties(${target} PROPERTIES + INSTALL_RPATH "$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE + ) +endfunction() if (LLAMA_BUILD) set(BUILD_SHARED_LIBS "On") @@ -45,30 +58,8 @@ if (LLAMA_BUILD) endif() add_subdirectory(vendor/llama.cpp) - install( - TARGETS llama - LIBRARY DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - RUNTIME DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - ARCHIVE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - FRAMEWORK DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - RESOURCE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - ) - set_target_properties(llama PROPERTIES - INSTALL_RPATH "$ORIGIN" - BUILD_WITH_INSTALL_RPATH TRUE - ) - install( - TARGETS ggml - LIBRARY DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - RUNTIME DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - ARCHIVE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - FRAMEWORK DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - RESOURCE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - ) - set_target_properties(ggml PROPERTIES - INSTALL_RPATH "$ORIGIN" - BUILD_WITH_INSTALL_RPATH TRUE - ) + llama_cpp_python_install_target(llama) + llama_cpp_python_install_target(ggml) # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563 if (WIN32 AND (LLAMA_CUDA OR LLAMA_CUBLAS)) install( @@ -94,17 +85,6 @@ if (LLAMA_BUILD) if (WIN32) set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) endif() - install( - TARGETS llava_shared - LIBRARY DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - RUNTIME DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - ARCHIVE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - FRAMEWORK DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - RESOURCE DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} - ) - set_target_properties(llava PROPERTIES - INSTALL_RPATH "$ORIGIN" - BUILD_WITH_INSTALL_RPATH TRUE - ) + llama_cpp_python_install_target(llava_shared) endif() endif() From 73fe013a4857962c60c6cb16c330cece639fb8ac Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 Jul 2024 23:17:02 -0400 Subject: [PATCH 224/561] fix: Fix RPATH so it works on macos --- CMakeLists.txt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc58cb884f..e0281b0590 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,19 @@ function(llama_cpp_python_install_target target) INSTALL_RPATH "$ORIGIN" BUILD_WITH_INSTALL_RPATH TRUE ) + if(UNIX) + if(APPLE) + set_target_properties(${target} PROPERTIES + INSTALL_RPATH "@loader_path" + BUILD_WITH_INSTALL_RPATH TRUE + ) + else() + set_target_properties(${target} PROPERTIES + INSTALL_RPATH "$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE + ) + endif() + endif() endfunction() if (LLAMA_BUILD) @@ -40,7 +53,6 @@ if (LLAMA_BUILD) # Add the automatically determined parts of the RPATH # which point to directories outside the build tree to the install RPATH set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - set(CMAKE_INSTALL_RPATH "$ORIGIN") set(CMAKE_SKIP_RPATH FALSE) # Building llama From dc20e8c342264b950c0cb3f8d7ba816fba15cfc8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 1 Jul 2024 23:28:19 -0400 Subject: [PATCH 225/561] fix: Copy dependencies for windows fix: :( fix fix fix: Copy runtime dlls on windows fix: Add explicit copy command for windows dlls fix fix fix: >:( fix fix fix fix Update path on windows check dll dependancies fix: Update PATH on win32 ci: Update test.yaml --- .github/workflows/test.yaml | 26 -------------------------- CMakeLists.txt | 29 ++++++++++++++++++++++++++--- llama_cpp/llama_cpp.py | 5 +++++ 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6d8231d186..9a938ae114 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -81,32 +81,6 @@ jobs: run: | python -m pytest - # build-linux-opencl: - - # runs-on: ubuntu-latest - - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: "recursive" - # - name: Set up Python 3.8 - # uses: actions/setup-python@v5 - # with: - # python-version: "3.8" - # - name: Set up OpenCL & CLBlast - # run: | - # wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null - # echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list - # sudo apt-get update - # sudo apt-get install -y --no-install-recommends llvm intel-oneapi-runtime-opencl intel-oneapi-runtime-compilers libclblast-dev - # - name: Install dependencies - # run: | - # python -m pip install --upgrade pip - # CMAKE_ARGS="-DLLAMA_CLBLAST=on" python -m pip install .[all] --verbose - # - name: Test with pytest - # run: | - # python -m pytest - build-macos-metal: diff --git a/CMakeLists.txt b/CMakeLists.txt index e0281b0590..0c4fa026ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,11 +72,24 @@ if (LLAMA_BUILD) add_subdirectory(vendor/llama.cpp) llama_cpp_python_install_target(llama) llama_cpp_python_install_target(ggml) + # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563 - if (WIN32 AND (LLAMA_CUDA OR LLAMA_CUBLAS)) + if (WIN32) install( FILES $ - DESTINATION ${LLAMA_CPP_PYTHON_INSTALL_DIR} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + ) + install( + FILES $ + DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib + ) + install( + FILES $ + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + ) + install( + FILES $ + DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib ) endif() @@ -98,5 +111,15 @@ if (LLAMA_BUILD) set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) endif() llama_cpp_python_install_target(llava_shared) + if (WIN32) + install( + FILES $ + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + ) + install( + FILES $ + DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib + ) + endif() endif() -endif() +endif() \ No newline at end of file diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1999b9a403..8da50af924 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -52,7 +52,12 @@ def _load_shared_library(lib_base_name: str): _lib_paths = [_lib.resolve()] cdll_args = dict() # type: ignore + # Add the library directory to the DLL search path on Windows (if needed) + if sys.platform == "win32": + os.add_dll_directory(str(_base_path)) + os.environ['PATH'] = str(_base_path) + os.pathsep + os.environ['PATH'] + if sys.platform == "win32" and sys.version_info >= (3, 8): os.add_dll_directory(str(_base_path)) if "CUDA_PATH" in os.environ: From 296304b60bb83689659883c9cc24f4c074dd88ff Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 Jul 2024 02:49:20 -0400 Subject: [PATCH 226/561] fix(server): Fix bug in FastAPI streaming response where dependency was released before request completes causing SEGFAULT --- llama_cpp/server/app.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 4cda4af7a7..f4d356b230 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -2,6 +2,7 @@ import os import json +import contextlib from threading import Lock from functools import partial @@ -156,6 +157,7 @@ async def get_event_publisher( request: Request, inner_send_chan: MemoryObjectSendStream, iterator: Iterator, + on_complete=None, ): async with inner_send_chan: try: @@ -175,6 +177,9 @@ async def get_event_publisher( with anyio.move_on_after(1, shield=True): print(f"Disconnected from client (via refresh/close) {request.client}") raise e + finally: + if on_complete: + on_complete() def _logit_bias_tokens_to_input_ids( @@ -258,8 +263,11 @@ async def authenticate( async def create_completion( request: Request, body: CreateCompletionRequest, - llama_proxy: LlamaProxy = Depends(get_llama_proxy), ) -> llama_cpp.Completion: + exit_stack = contextlib.ExitStack() + llama_proxy = await run_in_threadpool( + lambda: exit_stack.enter_context(contextlib.contextmanager(get_llama_proxy)()) + ) if isinstance(body.prompt, list): assert len(body.prompt) <= 1 body.prompt = body.prompt[0] if len(body.prompt) > 0 else "" @@ -312,6 +320,7 @@ async def create_completion( def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: yield first_response yield from iterator_or_completion + exit_stack.close() send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( @@ -321,6 +330,7 @@ def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: request=request, inner_send_chan=send_chan, iterator=iterator(), + on_complete=exit_stack.close, ), sep="\n", ping_message_factory=_ping_message_factory, @@ -449,8 +459,15 @@ async def create_chat_completion( }, } ), - llama_proxy: LlamaProxy = Depends(get_llama_proxy), ) -> llama_cpp.ChatCompletion: + # This is a workaround for an issue in FastAPI dependencies + # where the dependency is cleaned up before a StreamingResponse + # is complete. + # https://github.com/tiangolo/fastapi/issues/11143 + exit_stack = contextlib.ExitStack() + llama_proxy = await run_in_threadpool( + lambda: exit_stack.enter_context(contextlib.contextmanager(get_llama_proxy)()) + ) exclude = { "n", "logit_bias_type", @@ -491,6 +508,7 @@ async def create_chat_completion( def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: yield first_response yield from iterator_or_completion + exit_stack.close() send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( @@ -500,11 +518,13 @@ def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: request=request, inner_send_chan=send_chan, iterator=iterator(), + on_complete=exit_stack.close, ), sep="\n", ping_message_factory=_ping_message_factory, ) else: + exit_stack.close() return iterator_or_completion From bd5d17bcf0c61fd2cd43ef48c2f616266f1e29c8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 Jul 2024 02:55:19 -0400 Subject: [PATCH 227/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 5fac350b9c..023b8807e1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 5fac350b9cc49d0446fc291b9c4ad53666c77591 +Subproject commit 023b8807e10bc3ade24a255f01c1ad2a01bb4228 From b4cc9234748cb4bb96db1175a143139d6b7d8557 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 Jul 2024 02:55:57 -0400 Subject: [PATCH 228/561] chore: Bump version --- CHANGELOG.md | 8 ++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42b14c862f..a897256764 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.80] + +- feat: Update llama.cpp to ggerganov/llama.cpp@023b8807e10bc3ade24a255f01c1ad2a01bb4228 +- fix(server): Fix bug in FastAPI streaming response where dependency was released before request completes causing SEGFAULT by @abetlen in 296304b60bb83689659883c9cc24f4c074dd88ff +- fix(server): Update default config value for embeddings to False to fix error in text generation where logits were not allocated by llama.cpp by @abetlen in bf5e0bb4b151f4ca2f5a21af68eb832a96a79d75 +- fix(ci): Fix the CUDA workflow by @oobabooga in #1551 +- docs: Update readme examples to use newer Qwen2 model by @jncraton in #1544 + ## [0.2.79] - feat: Update llama.cpp to ggerganov/llama.cpp@9c77ec1d74874ee22bdef8f110e8e8d41389abf2 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 71371b3b07..a83d8ad74c 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.79" \ No newline at end of file +__version__ = "0.2.80" \ No newline at end of file From 4fb6fc12a02a68884c25dd9f6a421cacec7604c6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 Jul 2024 03:09:19 -0400 Subject: [PATCH 229/561] fix(ci): Use LLAMA_CUDA for cuda wheels --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 31c8619051..78d0cba81b 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -114,7 +114,7 @@ jobs: $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH } $env:VERBOSE = '1' - $env:CMAKE_ARGS = '-DLLAMA_CUBLAS=on -DCMAKE_CUDA_ARCHITECTURES=all' + $env:CMAKE_ARGS = '-DLLAMA_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=all' $env:CMAKE_ARGS = "-DLLAMA_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS" # if ($env:AVXVER -eq 'AVX') { $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off' From 387d01d94581dbcb1288304094bb1acee5ca24ae Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 Jul 2024 11:06:57 -0400 Subject: [PATCH 230/561] fix(misc): Fix type errors --- llama_cpp/server/app.py | 43 ++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index f4d356b230..8fd41a3fc7 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -2,6 +2,7 @@ import os import json +import typing import contextlib from threading import Lock @@ -88,11 +89,12 @@ def get_llama_proxy(): llama_outer_lock.release() -_ping_message_factory = None +_ping_message_factory: typing.Optional[typing.Callable[[], bytes]] = None -def set_ping_message_factory(factory): - global _ping_message_factory - _ping_message_factory = factory + +def set_ping_message_factory(factory: typing.Callable[[], bytes]): + global _ping_message_factory + _ping_message_factory = factory def create_app( @@ -155,20 +157,19 @@ def create_app( async def get_event_publisher( request: Request, - inner_send_chan: MemoryObjectSendStream, - iterator: Iterator, - on_complete=None, + inner_send_chan: MemoryObjectSendStream[typing.Any], + iterator: Iterator[typing.Any], + on_complete: typing.Optional[typing.Callable[[], None]] = None, ): + server_settings = next(get_server_settings()) + interrupt_requests = server_settings.interrupt_requests if server_settings else False async with inner_send_chan: try: async for chunk in iterate_in_threadpool(iterator): await inner_send_chan.send(dict(data=json.dumps(chunk))) if await request.is_disconnected(): raise anyio.get_cancelled_exc_class()() - if ( - next(get_server_settings()).interrupt_requests - and llama_outer_lock.locked() - ): + if interrupt_requests and llama_outer_lock.locked(): await inner_send_chan.send(dict(data="[DONE]")) raise anyio.get_cancelled_exc_class()() await inner_send_chan.send(dict(data="[DONE]")) @@ -268,6 +269,11 @@ async def create_completion( llama_proxy = await run_in_threadpool( lambda: exit_stack.enter_context(contextlib.contextmanager(get_llama_proxy)()) ) + if llama_proxy is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Service is not available", + ) if isinstance(body.prompt, list): assert len(body.prompt) <= 1 body.prompt = body.prompt[0] if len(body.prompt) > 0 else "" @@ -409,7 +415,7 @@ async def create_chat_completion( {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Who won the world series in 2020"}, ], - "response_format": { "type": "json_object" } + "response_format": {"type": "json_object"}, }, }, "tool_calling": { @@ -434,15 +440,15 @@ async def create_chat_completion( }, "required": ["name", "age"], }, - } + }, } ], "tool_choice": { "type": "function", "function": { "name": "User", - } - } + }, + }, }, }, "logprobs": { @@ -454,7 +460,7 @@ async def create_chat_completion( {"role": "user", "content": "What is the capital of France?"}, ], "logprobs": True, - "top_logprobs": 10 + "top_logprobs": 10, }, }, } @@ -468,6 +474,11 @@ async def create_chat_completion( llama_proxy = await run_in_threadpool( lambda: exit_stack.enter_context(contextlib.contextmanager(get_llama_proxy)()) ) + if llama_proxy is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Service is not available", + ) exclude = { "n", "logit_bias_type", From 8992a1a6467322bc8be28b74efa9e6a38d4c33b1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 Jul 2024 11:09:31 -0400 Subject: [PATCH 231/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 2 ++ vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 8da50af924..ee63de7cf7 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -311,6 +311,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, # LLAMA_VOCAB_PRE_TYPE_PORO = 15, # LLAMA_VOCAB_PRE_TYPE_VIKING = 16, +# LLAMA_VOCAB_PRE_TYPE_JAIS = 17, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -329,6 +330,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_SMAUG = 14 LLAMA_VOCAB_PRE_TYPE_PORO = 15 LLAMA_VOCAB_PRE_TYPE_VIKING = 16 +LLAMA_VOCAB_PRE_TYPE_JAIS = 17 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 023b8807e1..968967376d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 023b8807e10bc3ade24a255f01c1ad2a01bb4228 +Subproject commit 968967376dc2c018d29f897c4883d335bbf384fb From 3a551eb5263fdbd24b36d7770856374c04e92788 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 Jul 2024 11:58:26 -0400 Subject: [PATCH 232/561] fix(ci): Update macos image (macos-11 is removed) --- .github/workflows/build-and-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 34e9dc139f..e736cf9648 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-11] + os: [ubuntu-20.04, windows-2019, macos-12] steps: - uses: actions/checkout@v4 From 01bddd669ca1208f1844ce8d0ba9872532641c9d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 2 Jul 2024 12:02:45 -0400 Subject: [PATCH 233/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a897256764..a953af7e70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.81] + +- feat: Update llama.cpp to ggerganov/llama.cpp@968967376dc2c018d29f897c4883d335bbf384fb +- fix(ci): Fix CUDA wheels, use LLAMA_CUDA instead of removed LLAMA_CUBLAS by @abetlen in 4fb6fc12a02a68884c25dd9f6a421cacec7604c6 +- fix(ci): Fix MacOS release, use macos-12 image instead of removed macos-11 by @abetlen in 3a551eb5263fdbd24b36d7770856374c04e92788 + ## [0.2.80] - feat: Update llama.cpp to ggerganov/llama.cpp@023b8807e10bc3ade24a255f01c1ad2a01bb4228 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index a83d8ad74c..f87e2a3884 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.80" \ No newline at end of file +__version__ = "0.2.81" \ No newline at end of file From 7e20e346bd49cc8f0031eb053fe879a38c777b6f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 4 Jul 2024 12:23:47 -0400 Subject: [PATCH 234/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 34 ++++++++++++++++++++++++++++++++++ vendor/llama.cpp | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index ee63de7cf7..1fc1968eef 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1444,6 +1444,24 @@ def llama_get_model_tensor( ... +# // Returns true if the model contains an encoder that requires llama_encode() call +# LLAMA_API bool llama_model_has_encoder(const struct llama_model * model); +@ctypes_function("llama_model_has_encoder", [llama_model_p_ctypes], ctypes.c_bool) +def llama_model_has_encoder(model: llama_model_p, /) -> bool: + """Returns true if the model contains an encoder that requires llama_encode() call""" + ... + + +# // For encoder-decoder models, this function returns id of the token that must be provided +# // to the decoder to start generating output sequence. For other models, it returns -1. +# LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model); +@ctypes_function("llama_model_decoder_start_token", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_decoder_start_token(model: llama_model_p, /) -> int: + """For encoder-decoder models, this function returns id of the token that must be provided + to the decoder to start generating output sequence. For other models, it returns -1.""" + ... + + # // Returns 0 on success # LLAMA_API uint32_t llama_model_quantize( # const char * fname_inp, @@ -2271,6 +2289,22 @@ def llama_batch_free(batch: llama_batch, /): ... +# // Processes a batch of tokens with the ecoder part of the encoder-decoder model. +# // Stores the encoder output internally for later use by the decoder cross-attention layers. +# // 0 - success +# // < 0 - error +# LLAMA_API int32_t llama_encode( +# struct llama_context * ctx, +# struct llama_batch batch); +@ctypes_function("llama_encode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) +def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: + """Processes a batch of tokens with the ecoder part of the encoder-decoder model. + Stores the encoder output internally for later use by the decoder cross-attention layers. + 0 - success + < 0 - error""" + ... + + # // Positive return values does not mean a fatal error, but rather a warning. # // 0 - success # // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 968967376d..51d2ebadbb 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 968967376dc2c018d29f897c4883d335bbf384fb +Subproject commit 51d2ebadbbf365b894f3888361df42dbacb12b7a From 62804eeb12322ab167c22bc1d2dea10175e90029 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 5 Jul 2024 20:44:46 -0400 Subject: [PATCH 235/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 72 ++++++++++++++++++++++++++++++++++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1fc1968eef..5ee3476c92 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -488,6 +488,15 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_POOLING_TYPE_CLS = 2 LLAMA_POOLING_TYPE_LAST = 3 +# enum llama_attention_type { +# LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1, +# LLAMA_ATTENTION_TYPE_CAUSAL = 0, +# LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1, +# }; +LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1 +LLAMA_ATTENTION_TYPE_CAUSAL = 0 +LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1 + # enum llama_split_mode { # LLAMA_SPLIT_MODE_NONE = 0, // single GPU # LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs @@ -775,6 +784,7 @@ class llama_model_params(ctypes.Structure): # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` # enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id +# enum llama_attention_type attention_type; // attention type to use for embeddings # // ref: https://github.com/ggerganov/llama.cpp/pull/2054 # float rope_freq_base; // RoPE base frequency, 0 = from model @@ -817,6 +827,7 @@ class llama_context_params(ctypes.Structure): n_threads_batch (int): number of threads to use for batch processing rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type` pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) + attention_type (int): attention type to use for embeddings rope_freq_base (float): RoPE base frequency, 0 = from model rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model @@ -847,6 +858,7 @@ class llama_context_params(ctypes.Structure): n_threads_batch: int rope_scaling_type: int pooling_type: int + attention_type: int rope_freq_base: float rope_freq_scale: float yarn_ext_factor: float @@ -876,6 +888,7 @@ class llama_context_params(ctypes.Structure): ("n_threads_batch", ctypes.c_uint32), ("rope_scaling_type", ctypes.c_int), ("pooling_type", ctypes.c_int), + ("attention_type", ctypes.c_int), ("rope_freq_base", ctypes.c_float), ("rope_freq_scale", ctypes.c_float), ("yarn_ext_factor", ctypes.c_float), @@ -2642,6 +2655,7 @@ def llama_token_eot(model: llama_model_p, /) -> int: ... # /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. # /// @return Returns the number of tokens on success, no more than n_tokens_max # /// @return Returns a negative number on failure - the number of tokens that would have been returned +# /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so. # /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated # /// as plaintext. Does not insert a leading space. # LLAMA_API int32_t llama_tokenize( @@ -2683,7 +2697,7 @@ def llama_tokenize( text_len: The length of the text. tokens: The tokens pointer must be large enough to hold the resulting tokens. n_max_tokens: The maximum number of tokens to return. - add_special: Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext. Does not insert a leading space. + add_special: Allow adding special tokenns if the model is configured to do so. parse_special: Allow parsing special tokens. Returns: @@ -2696,13 +2710,14 @@ def llama_tokenize( # // Token Id -> Piece. # // Uses the vocabulary in the provided context. # // Does not write null terminator to the buffer. -# // User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens. +# // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix') # // @param special If true, special tokens are rendered in the output. # LLAMA_API int32_t llama_token_to_piece( # const struct llama_model * model, # llama_token token, # char * buf, # int32_t length, +# int32_t lstrip, # bool special); @ctypes_function( "llama_token_to_piece", @@ -2711,6 +2726,7 @@ def llama_tokenize( llama_token, ctypes.c_char_p, ctypes.c_int32, + ctypes.c_int32, ctypes.c_bool, ], ctypes.c_int32, @@ -2720,6 +2736,7 @@ def llama_token_to_piece( token: Union[llama_token, int], buf: Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]], length: Union[ctypes.c_int, int], + lstrip: Union[ctypes.c_int, int], special: Union[ctypes.c_bool, bool], /, ) -> int: @@ -2733,10 +2750,61 @@ def llama_token_to_piece( token: The token to convert. buf: The buffer to write the token to. length: The length of the buffer. + lstrip: The number of leading spaces to skip. special: If true, special tokens are rendered in the output.""" ... +# /// @details Convert the provided tokens into text (inverse of llama_tokenize()). +# /// @param text The char pointer must be large enough to hold the resulting text. +# /// @return Returns the number of chars/bytes on success, no more than text_len_max. +# /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned. +# /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. +# /// @param unparse_special If true, special tokens are rendered in the output. +# LLAMA_API int32_t llama_detokenize( +# const struct llama_model * model, +# const llama_token * tokens, +# int32_t n_tokens, +# char * text, +# int32_t text_len_max, +# bool remove_special, +# bool unparse_special); +@ctypes_function( + "llama_detokenize", + [ + llama_model_p_ctypes, + ctypes.POINTER(llama_token), + ctypes.c_int32, + ctypes.c_char_p, + ctypes.c_int32, + ctypes.c_bool, + ctypes.c_bool, + ], + ctypes.c_int32, +) +def llama_detokenize( + model: llama_model_p, + tokens: CtypesArray[llama_token], + n_tokens: Union[ctypes.c_int, int], + text: bytes, + text_len_max: Union[ctypes.c_int, int], + remove_special: Union[ctypes.c_bool, bool], + unparse_special: Union[ctypes.c_bool, bool], + /, +) -> int: + """Convert the provided tokens into text (inverse of llama_tokenize()). + + Args: + model: The model to use for tokenization. + tokens: The tokens to convert. + n_tokens: The number of tokens. + text: The buffer to write the text to. + text_len_max: The length of the buffer. + remove_special: Allow to remove BOS and EOS tokens if model is configured to do so. + unparse_special: If true, special tokens are rendered in the output.""" + ... + + # /// Apply chat template. Inspired by hf apply_chat_template() on python. # /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model" # /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 51d2ebadbb..213701b51a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 51d2ebadbbf365b894f3888361df42dbacb12b7a +Subproject commit 213701b51a17175d0d326b566efc03f30ec7fbe6 From 157d91359d5d51cee5a2f44fd9f54734603d74bc Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 5 Jul 2024 21:42:02 -0400 Subject: [PATCH 236/561] fix: update token_to_piece --- llama_cpp/_internals.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 423c1b5bcd..125d72ded3 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -208,7 +208,7 @@ def tokenize(self, text: bytes, add_bos: bool, special: bool): def token_to_piece(self, token: int, special: bool = False) -> bytes: assert self.model is not None buf = ctypes.create_string_buffer(32) - llama_cpp.llama_token_to_piece(self.model, token, buf, 32, special) + llama_cpp.llama_token_to_piece(self.model, token, buf, 32, 0, special) return bytes(buf) def detokenize(self, tokens: List[int], special: bool = False) -> bytes: @@ -218,7 +218,7 @@ def detokenize(self, tokens: List[int], special: bool = False) -> bytes: buffer = (ctypes.c_char * size)() for token in tokens: n = llama_cpp.llama_token_to_piece( - self.model, llama_cpp.llama_token(token), buffer, size, special + self.model, llama_cpp.llama_token(token), buffer, size, 0, special ) assert n <= size output += bytes(buffer[:n]) @@ -635,10 +635,10 @@ def _tokenize(model: _LlamaModel, text: str, add_bos: bool, special: bool) -> li def _token_to_piece(model: _LlamaModel, token: int, special: bool = False) -> str: assert model.model is not None result = (ctypes.c_char * 8)(0) - n_tokens = llama_cpp.llama_token_to_piece(model.model, token, result, len(result), special) + n_tokens = llama_cpp.llama_token_to_piece(model.model, token, result, 0, len(result), special) if n_tokens < 0: result = (ctypes.c_char * -n_tokens)(0) - check = llama_cpp.llama_token_to_piece(model.model, token, result, len(result), special) + check = llama_cpp.llama_token_to_piece(model.model, token, result, 0, len(result), special) if check != -n_tokens: raise RuntimeError(f"Failed to get piece: token={token}") else: From 218d3610ae6e8f4849eb7b7661a0239ef3351aea Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Jul 2024 00:15:22 -0400 Subject: [PATCH 237/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 12 ++++++++---- vendor/llama.cpp | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 5ee3476c92..a0f5832c62 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -310,8 +310,10 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_DBRX = 13, # LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, # LLAMA_VOCAB_PRE_TYPE_PORO = 15, -# LLAMA_VOCAB_PRE_TYPE_VIKING = 16, -# LLAMA_VOCAB_PRE_TYPE_JAIS = 17, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, +# LLAMA_VOCAB_PRE_TYPE_VIKING = 18, +# LLAMA_VOCAB_PRE_TYPE_JAIS = 19, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -329,8 +331,10 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_DBRX = 13 LLAMA_VOCAB_PRE_TYPE_SMAUG = 14 LLAMA_VOCAB_PRE_TYPE_PORO = 15 -LLAMA_VOCAB_PRE_TYPE_VIKING = 16 -LLAMA_VOCAB_PRE_TYPE_JAIS = 17 +LLAMA_VOCAV_PRE_TYPE_CHATGLM3 = 16 +LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17 +LLAMA_VOCAB_PRE_TYPE_VIKING = 18 +LLAMA_VOCAB_PRE_TYPE_JAIS = 19 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 213701b51a..7fdb6f73e3 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 213701b51a17175d0d326b566efc03f30ec7fbe6 +Subproject commit 7fdb6f73e35605c8dbc39e9f19cd9ed84dbc87f2 From 1a55417455660cd626e8e8a37a19659f97c28ef1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Jul 2024 00:29:29 -0400 Subject: [PATCH 238/561] fix: Update LLAMA_ flags to GGML_ flags --- CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c4fa026ff..c6b35ed6c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,14 +59,14 @@ if (LLAMA_BUILD) if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") # Need to disable these llama.cpp flags on Apple x86_64, # otherwise users may encounter invalid instruction errors - set(LLAMA_AVX "Off" CACHE BOOL "llama: enable AVX" FORCE) - set(LLAMA_AVX2 "Off" CACHE BOOL "llama: enable AVX2" FORCE) - set(LLAMA_FMA "Off" CACHE BOOL "llama: enable FMA" FORCE) - set(LLAMA_F16C "Off" CACHE BOOL "llama: enable F16C" FORCE) + set(GGML_AVX "Off" CACHE BOOL "ggml: enable AVX" FORCE) + set(GGML_AVX2 "Off" CACHE BOOL "ggml: enable AVX2" FORCE) + set(GGML_FMA "Off" CACHE BOOL "gml: enable FMA" FORCE) + set(GGML_F16C "Off" CACHE BOOL "gml: enable F16C" FORCE) endif() if (APPLE) - set(LLAMA_METAL_EMBED_LIBRARY "On" CACHE BOOL "llama: embed metal library" FORCE) + set(GGML_METAL_EMBED_LIBRARY "On" CACHE BOOL "llama: embed metal library" FORCE) endif() add_subdirectory(vendor/llama.cpp) @@ -122,4 +122,4 @@ if (LLAMA_BUILD) ) endif() endif() -endif() \ No newline at end of file +endif() From 09a4f78e05f43fcd93efc82cfddca250af1bad4d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Jul 2024 00:37:03 -0400 Subject: [PATCH 239/561] fix(ci): Update LLAMA_ flags to GGML_ --- .github/workflows/build-wheels-cuda.yaml | 10 +++++----- .github/workflows/build-wheels-metal.yaml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 78d0cba81b..5d445b5de8 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -114,16 +114,16 @@ jobs: $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH } $env:VERBOSE = '1' - $env:CMAKE_ARGS = '-DLLAMA_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=all' - $env:CMAKE_ARGS = "-DLLAMA_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS" + $env:CMAKE_ARGS = '-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=all' + $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS" # if ($env:AVXVER -eq 'AVX') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off' + $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' # } # if ($env:AVXVER -eq 'AVX512') { - # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX512=on' + # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX512=on' # } # if ($env:AVXVER -eq 'basic') { - # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off' + # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' # } python -m build --wheel # write the build tag to the output diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 11ab795c8e..b56fb48aa9 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -35,7 +35,7 @@ jobs: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" CIBW_ARCHS: "arm64" - CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DLLAMA_METAL=on" + CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DGGML_METAL=on" CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" with: package-dir: . diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9a938ae114..78f0b49837 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -97,7 +97,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - CMAKE_ARGS="-DLLAMA_METAL=on" python -m pip install .[all] --verbose + CMAKE_ARGS="-DGGML_METAL=on" python -m pip install .[all] --verbose - name: Test with pytest run: | python -m pytest From 0481a3a8e124b9c7eb5413930cb0597215889f3a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Jul 2024 00:39:32 -0400 Subject: [PATCH 240/561] fix(docs): Update LLAMA_ flags to GGML_ flags --- README.md | 46 +++++++++++++++++++++---------------------- docs/install/macos.md | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4a710d4e56..8363ef6857 100644 --- a/README.md +++ b/README.md @@ -64,13 +64,13 @@ All `llama.cpp` cmake build options can be set via the `CMAKE_ARGS` environment ```bash # Linux and Mac -CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" \ +CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" \ pip install llama-cpp-python ``` ```powershell # Windows -$env:CMAKE_ARGS = "-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" +$env:CMAKE_ARGS = "-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-python ```
@@ -83,13 +83,13 @@ They can also be set via `pip install -C / --config-settings` command and saved ```bash pip install --upgrade pip # ensure pip is up to date pip install llama-cpp-python \ - -C cmake.args="-DLLAMA_BLAS=ON;-DLLAMA_BLAS_VENDOR=OpenBLAS" + -C cmake.args="-DGGML_BLAS=ON;-DGGML_BLAS_VENDOR=OpenBLAS" ``` ```txt # requirements.txt -llama-cpp-python -C cmake.args="-DLLAMA_BLAS=ON;-DLLAMA_BLAS_VENDOR=OpenBLAS" +llama-cpp-python -C cmake.args="-DGGML_BLAS=ON;-DGGML_BLAS_VENDOR=OpenBLAS" ``` @@ -101,20 +101,20 @@ Below are some common backends, their build commands and any additional environm
OpenBLAS (CPU) -To install with OpenBLAS, set the `LLAMA_BLAS` and `LLAMA_BLAS_VENDOR` environment variables before installing: +To install with OpenBLAS, set the `GGML_BLAS` and `GGML_BLAS_VENDOR` environment variables before installing: ```bash -CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-python +CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-python ```
CUDA -To install with CUDA support, set the `LLAMA_CUDA=on` environment variable before installing: +To install with CUDA support, set the `GGML_CUDA=on` environment variable before installing: ```bash -CMAKE_ARGS="-DLLAMA_CUDA=on" pip install llama-cpp-python +CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python ``` **Pre-built Wheel (New)** @@ -147,10 +147,10 @@ pip install llama-cpp-python \
Metal -To install with Metal (MPS), set the `LLAMA_METAL=on` environment variable before installing: +To install with Metal (MPS), set the `GGML_METAL=on` environment variable before installing: ```bash -CMAKE_ARGS="-DLLAMA_METAL=on" pip install llama-cpp-python +CMAKE_ARGS="-DGGML_METAL=on" pip install llama-cpp-python ``` **Pre-built Wheel (New)** @@ -170,10 +170,10 @@ pip install llama-cpp-python \
hipBLAS (ROCm) -To install with hipBLAS / ROCm support for AMD cards, set the `LLAMA_HIPBLAS=on` environment variable before installing: +To install with hipBLAS / ROCm support for AMD cards, set the `GGML_HIPBLAS=on` environment variable before installing: ```bash -CMAKE_ARGS="-DLLAMA_HIPBLAS=on" pip install llama-cpp-python +CMAKE_ARGS="-DGGML_HIPBLAS=on" pip install llama-cpp-python ```
@@ -181,10 +181,10 @@ CMAKE_ARGS="-DLLAMA_HIPBLAS=on" pip install llama-cpp-python
Vulkan -To install with Vulkan support, set the `LLAMA_VULKAN=on` environment variable before installing: +To install with Vulkan support, set the `GGML_VULKAN=on` environment variable before installing: ```bash -CMAKE_ARGS="-DLLAMA_VULKAN=on" pip install llama-cpp-python +CMAKE_ARGS="-DGGML_VULKAN=on" pip install llama-cpp-python ```
@@ -192,32 +192,32 @@ CMAKE_ARGS="-DLLAMA_VULKAN=on" pip install llama-cpp-python
Kompute -To install with Kompute support, set the `LLAMA_KOMPUTE=on` environment variable before installing: +To install with Kompute support, set the `GGML_KOMPUTE=on` environment variable before installing: ```bash -CMAKE_ARGS="-DLLAMA_KOMPUTE=on" pip install llama-cpp-python +CMAKE_ARGS="-DGGML_KOMPUTE=on" pip install llama-cpp-python ```
SYCL -To install with SYCL support, set the `LLAMA_SYCL=on` environment variable before installing: +To install with SYCL support, set the `GGML_SYCL=on` environment variable before installing: ```bash source /opt/intel/oneapi/setvars.sh -CMAKE_ARGS="-DLLAMA_SYCL=on -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx" pip install llama-cpp-python +CMAKE_ARGS="-DGGML_SYCL=on -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx" pip install llama-cpp-python ```
RPC -To install with RPC support, set the `LLAMA_RPC=on` environment variable before installing: +To install with RPC support, set the `GGML_RPC=on` environment variable before installing: ```bash source /opt/intel/oneapi/setvars.sh -CMAKE_ARGS="-DLLAMA_RPC=on" pip install llama-cpp-python +CMAKE_ARGS="-DGGML_RPC=on" pip install llama-cpp-python ```
@@ -231,7 +231,7 @@ If you run into issues where it complains it can't find `'nmake'` `'?'` or CMAKE ```ps $env:CMAKE_GENERATOR = "MinGW Makefiles" -$env:CMAKE_ARGS = "-DLLAMA_OPENBLAS=on -DCMAKE_C_COMPILER=C:/w64devkit/bin/gcc.exe -DCMAKE_CXX_COMPILER=C:/w64devkit/bin/g++.exe" +$env:CMAKE_ARGS = "-DGGML_OPENBLAS=on -DCMAKE_C_COMPILER=C:/w64devkit/bin/gcc.exe -DCMAKE_CXX_COMPILER=C:/w64devkit/bin/g++.exe" ``` See the above instructions and set `CMAKE_ARGS` to the BLAS backend you want to use. @@ -260,7 +260,7 @@ Otherwise, while installing it will build the llama.cpp x86 version which will b Try installing with ```bash -CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DLLAMA_METAL=on" pip install --upgrade --verbose --force-reinstall --no-cache-dir llama-cpp-python +CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DGGML_METAL=on" pip install --upgrade --verbose --force-reinstall --no-cache-dir llama-cpp-python ```
@@ -667,7 +667,7 @@ python3 -m llama_cpp.server --model models/7B/llama-model.gguf Similar to Hardware Acceleration section above, you can also install with GPU (cuBLAS) support like this: ```bash -CMAKE_ARGS="-DLLAMA_CUDA=on" FORCE_CMAKE=1 pip install 'llama-cpp-python[server]' +CMAKE_ARGS="-DGGML_CUDA=on" FORCE_CMAKE=1 pip install 'llama-cpp-python[server]' python3 -m llama_cpp.server --model models/7B/llama-model.gguf --n_gpu_layers 35 ``` diff --git a/docs/install/macos.md b/docs/install/macos.md index 2404228325..e006fc0a3c 100644 --- a/docs/install/macos.md +++ b/docs/install/macos.md @@ -30,7 +30,7 @@ conda activate llama *(you needed xcode installed in order pip to build/compile the C++ code)* ``` pip uninstall llama-cpp-python -y -CMAKE_ARGS="-DLLAMA_METAL=on" pip install -U llama-cpp-python --no-cache-dir +CMAKE_ARGS="-DGGML_METAL=on" pip install -U llama-cpp-python --no-cache-dir pip install 'llama-cpp-python[server]' # you should now have llama-cpp-python v0.1.62 or higher installed From fccff8015eb8880e826d407321f5fbf7bd8dd4c3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Jul 2024 00:41:36 -0400 Subject: [PATCH 241/561] fix(docs): Remove kompute backend references --- README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/README.md b/README.md index 8363ef6857..5641ccaa8f 100644 --- a/README.md +++ b/README.md @@ -189,16 +189,6 @@ CMAKE_ARGS="-DGGML_VULKAN=on" pip install llama-cpp-python
-
-Kompute - -To install with Kompute support, set the `GGML_KOMPUTE=on` environment variable before installing: - -```bash -CMAKE_ARGS="-DGGML_KOMPUTE=on" pip install llama-cpp-python -``` -
-
SYCL From 276ea28a3f665ac856aa5ac5ee0af8bcbc333ed2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Jul 2024 00:46:20 -0400 Subject: [PATCH 242/561] fix(misc): Update LLAMA_ flags to GGML_ --- Makefile | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 90f562fa19..be9b55a3e7 100644 --- a/Makefile +++ b/Makefile @@ -22,28 +22,28 @@ build.debug: --editable . build.cuda: - CMAKE_ARGS="-DLLAMA_CUDA=on" python3 -m pip install --verbose -e . + CMAKE_ARGS="-DGGML_CUDA=on" python3 -m pip install --verbose -e . build.openblas: - CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" python3 -m pip install --verbose -e . + CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" python3 -m pip install --verbose -e . build.blis: - CMAKE_ARGS="-DLLAMA_BLAS=on -DLLAMA_BLAS_VENDOR=FLAME" python3 -m pip install --verbose -e . + CMAKE_ARGS="-DGGML_BLAS=on -DGGML_BLAS_VENDOR=FLAME" python3 -m pip install --verbose -e . build.metal: - CMAKE_ARGS="-DLLAMA_METAL=on" python3 -m pip install --verbose -e . + CMAKE_ARGS="-DGGML_METAL=on" python3 -m pip install --verbose -e . build.vulkan: - CMAKE_ARGS="-DLLAMA_VULKAN=on" python3 -m pip install --verbose -e . + CMAKE_ARGS="-DGGML_VULKAN=on" python3 -m pip install --verbose -e . build.kompute: - CMAKE_ARGS="-DLLAMA_KOMPUTE=on" python3 -m pip install --verbose -e . + CMAKE_ARGS="-DGGML_KOMPUTE=on" python3 -m pip install --verbose -e . build.sycl: - CMAKE_ARGS="-DLLAMA_SYCL=on" python3 -m pip install --verbose -e . + CMAKE_ARGS="-DGGML_SYCL=on" python3 -m pip install --verbose -e . build.rpc: - CMAKE_ARGS="-DLLAMA_RPC=on" python3 -m pip install --verbose -e . + CMAKE_ARGS="-DGGML_RPC=on" python3 -m pip install --verbose -e . build.sdist: python3 -m build --sdist @@ -85,4 +85,4 @@ clean: deploy.pypi \ deploy.gh-docs \ docker \ - clean \ No newline at end of file + clean From aaf4cbe81c4387a6afe264843649545be9ead669 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Jul 2024 00:47:51 -0400 Subject: [PATCH 243/561] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a953af7e70..47077d8af8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.82] + +- feat: Update llama.cpp to ggerganov/llama.cpp@7fdb6f73e35605c8dbc39e9f19cd9ed84dbc87f2 + ## [0.2.81] - feat: Update llama.cpp to ggerganov/llama.cpp@968967376dc2c018d29f897c4883d335bbf384fb diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index f87e2a3884..09ffb594e4 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.81" \ No newline at end of file +__version__ = "0.2.82" From 14760c63ed6dd7e518efd87142810911ea73b6e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:51:35 -0400 Subject: [PATCH 244/561] chore(deps): bump pypa/cibuildwheel from 2.19.1 to 2.19.2 (#1568) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.19.1 to 2.19.2. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.19.1...v2.19.2) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei --- .github/workflows/build-and-release.yaml | 4 ++-- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index e736cf9648..eeeda803db 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -29,7 +29,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.19.1 + uses: pypa/cibuildwheel@v2.19.2 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -56,7 +56,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.19.1 + uses: pypa/cibuildwheel@v2.19.2 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index b56fb48aa9..47c1c3cb52 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -30,7 +30,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.19.1 + uses: pypa/cibuildwheel@v2.19.2 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" From e31f0968ffe456778500fe7c75cbf7844a65238f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:51:46 -0400 Subject: [PATCH 245/561] chore(deps): bump microsoft/setup-msbuild from 1.1 to 1.3 (#1569) Bumps [microsoft/setup-msbuild](https://github.com/microsoft/setup-msbuild) from 1.1 to 1.3. - [Release notes](https://github.com/microsoft/setup-msbuild/releases) - [Changelog](https://github.com/microsoft/setup-msbuild/blob/main/building-release.md) - [Commits](https://github.com/microsoft/setup-msbuild/compare/v1.1...v1.3) --- updated-dependencies: - dependency-name: microsoft/setup-msbuild dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 5d445b5de8..3c02670958 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -45,7 +45,7 @@ jobs: steps: - name: Add MSBuild to PATH if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.1 + uses: microsoft/setup-msbuild@v1.3 with: vs-version: '[16.11,16.12)' From b77e5070291c205034ba461b05c13f5a271e686f Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Tue, 9 Jul 2024 17:55:21 +0200 Subject: [PATCH 246/561] feat(ci): Dockerfile update base images and post-install cleanup (#1530) * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update Dockerfile * Update build-docker.yaml * Update Dockerfile * Update Dockerfile * Remove docker login --------- Co-authored-by: Andrei Betlen --- docker/cuda_simple/Dockerfile | 2 +- docker/open_llama/Dockerfile | 6 ++++-- docker/openblas_simple/Dockerfile | 7 +++++-- docker/simple/Dockerfile | 6 ++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docker/cuda_simple/Dockerfile b/docker/cuda_simple/Dockerfile index a9e51cdc16..79b6a5bac8 100644 --- a/docker/cuda_simple/Dockerfile +++ b/docker/cuda_simple/Dockerfile @@ -1,4 +1,4 @@ -ARG CUDA_IMAGE="12.1.1-devel-ubuntu22.04" +ARG CUDA_IMAGE="12.5.0-devel-ubuntu22.04" FROM nvidia/cuda:${CUDA_IMAGE} # We need to set the host to 0.0.0.0 to allow outside access diff --git a/docker/open_llama/Dockerfile b/docker/open_llama/Dockerfile index 23e37d4d40..70b16dffce 100644 --- a/docker/open_llama/Dockerfile +++ b/docker/open_llama/Dockerfile @@ -1,5 +1,5 @@ # Define the image argument and provide a default value -ARG IMAGE=python:3-slim-bullseye +ARG IMAGE=python:3-slim-bookworm # Use the image as specified FROM ${IMAGE} @@ -12,7 +12,9 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco python3 \ python3-pip \ ninja-build \ - build-essential + build-essential \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* RUN python3 -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context diff --git a/docker/openblas_simple/Dockerfile b/docker/openblas_simple/Dockerfile index e213518b95..804c1b9029 100644 --- a/docker/openblas_simple/Dockerfile +++ b/docker/openblas_simple/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3-slim-bullseye +FROM python:3-slim-bookworm # We need to set the host to 0.0.0.0 to allow outside access ENV HOST 0.0.0.0 @@ -6,7 +6,10 @@ ENV HOST 0.0.0.0 COPY . . # Install the package -RUN apt update && apt install -y libopenblas-dev ninja-build build-essential pkg-config +RUN apt update && apt install -y libopenblas-dev ninja-build build-essential pkg-config \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* + RUN python -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context RUN CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama_cpp_python --verbose diff --git a/docker/simple/Dockerfile b/docker/simple/Dockerfile index 41cc68ea24..97a6c19df9 100644 --- a/docker/simple/Dockerfile +++ b/docker/simple/Dockerfile @@ -1,5 +1,5 @@ # Define the image argument and provide a default value -ARG IMAGE=python:3-slim-bullseye +ARG IMAGE=python:3-slim-bookworm # Use the image as specified FROM ${IMAGE} @@ -13,7 +13,9 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco python3-pip \ ninja-build \ libopenblas-dev \ - build-essential + build-essential \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* RUN mkdir /app WORKDIR /app From c1ae81554f52f81b6027e36ae0a056dca9819c3c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Jul 2024 12:20:17 -0400 Subject: [PATCH 247/561] fix(misc): Format --- examples/batch-processing/server.py | 1 + examples/gradio_chat/local.py | 22 +- examples/gradio_chat/server.py | 21 +- examples/hf_pull/main.py | 21 +- examples/high_level_api/fastapi_server.py | 1 + .../high_level_api/high_level_api_infill.py | 20 +- examples/low_level_api/Chat.py | 52 +- examples/low_level_api/Miku.py | 46 +- examples/low_level_api/ReasonAct.py | 40 +- examples/low_level_api/common.py | 323 ++++- .../low_level_api/low_level_api_chat_cpp.py | 1197 ++++++++++------- .../low_level_api/low_level_api_llama_cpp.py | 48 +- examples/low_level_api/quantize.py | 7 +- examples/low_level_api/util.py | 156 +-- examples/notebooks/Batching.ipynb | 23 +- examples/notebooks/Clients.ipynb | 8 +- examples/notebooks/Functions.ipynb | 48 +- examples/notebooks/Guidance.ipynb | 12 +- examples/notebooks/Multimodal.ipynb | 19 +- .../notebooks/OpenHermesFunctionCalling.ipynb | 67 +- examples/notebooks/PerformanceTuning.ipynb | 19 +- examples/ray/llm.py | 1 + llama_cpp/_internals.py | 53 +- llama_cpp/_utils.py | 1 + llama_cpp/llama.py | 276 ++-- llama_cpp/llama_cache.py | 11 +- llama_cpp/llama_chat_format.py | 485 ++++--- llama_cpp/llama_cpp.py | 17 +- llama_cpp/llama_grammar.py | 568 +++++--- llama_cpp/llama_tokenizer.py | 11 +- llama_cpp/llama_types.py | 5 +- llama_cpp/server/__main__.py | 5 +- llama_cpp/server/app.py | 4 +- llama_cpp/server/settings.py | 12 +- llama_cpp/server/types.py | 2 +- 35 files changed, 2270 insertions(+), 1332 deletions(-) diff --git a/examples/batch-processing/server.py b/examples/batch-processing/server.py index d3536697a2..0b36746f91 100644 --- a/examples/batch-processing/server.py +++ b/examples/batch-processing/server.py @@ -25,6 +25,7 @@ import openai.types.chat as types + @app.post("/v1/chat/completions") def create_chat_completions(): return {"message": "Hello World"} diff --git a/examples/gradio_chat/local.py b/examples/gradio_chat/local.py index a7de8e842f..e16bf234ab 100644 --- a/examples/gradio_chat/local.py +++ b/examples/gradio_chat/local.py @@ -6,25 +6,26 @@ llama = llama_cpp.Llama.from_pretrained( repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF", filename="*q8_0.gguf", - tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B"), - verbose=False + tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained( + "Qwen/Qwen1.5-0.5B" + ), + verbose=False, ) model = "gpt-3.5-turbo" + def predict(message, history): messages = [] for user_message, assistant_message in history: messages.append({"role": "user", "content": user_message}) messages.append({"role": "assistant", "content": assistant_message}) - + messages.append({"role": "user", "content": message}) response = llama.create_chat_completion_openai_v1( - model=model, - messages=messages, - stream=True + model=model, messages=messages, stream=True ) text = "" @@ -52,7 +53,14 @@ def predict(message, history): """ with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo: - gr.ChatInterface(predict, fill_height=True, examples=["What is the capital of France?", "Who was the first person on the moon?"]) + gr.ChatInterface( + predict, + fill_height=True, + examples=[ + "What is the capital of France?", + "Who was the first person on the moon?", + ], + ) if __name__ == "__main__": diff --git a/examples/gradio_chat/server.py b/examples/gradio_chat/server.py index 36fa43fbd4..52061bea67 100644 --- a/examples/gradio_chat/server.py +++ b/examples/gradio_chat/server.py @@ -2,26 +2,22 @@ from openai import OpenAI -client = OpenAI( - base_url="http://localhost:8000/v1", - api_key="llama.cpp" -) +client = OpenAI(base_url="http://localhost:8000/v1", api_key="llama.cpp") model = "gpt-3.5-turbo" + def predict(message, history): messages = [] for user_message, assistant_message in history: messages.append({"role": "user", "content": user_message}) messages.append({"role": "assistant", "content": assistant_message}) - + messages.append({"role": "user", "content": message}) response = client.chat.completions.create( - model=model, - messages=messages, - stream=True + model=model, messages=messages, stream=True ) text = "" @@ -49,7 +45,14 @@ def predict(message, history): """ with gr.Blocks(theme=gr.themes.Soft(), js=js, css=css, fill_height=True) as demo: - gr.ChatInterface(predict, fill_height=True, examples=["What is the capital of France?", "Who was the first person on the moon?"]) + gr.ChatInterface( + predict, + fill_height=True, + examples=[ + "What is the capital of France?", + "Who was the first person on the moon?", + ], + ) if __name__ == "__main__": diff --git a/examples/hf_pull/main.py b/examples/hf_pull/main.py index d3eb11c396..dfed17516d 100644 --- a/examples/hf_pull/main.py +++ b/examples/hf_pull/main.py @@ -5,29 +5,26 @@ llama = llama_cpp.Llama.from_pretrained( repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF", filename="*q8_0.gguf", - tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B"), - verbose=False + tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained( + "Qwen/Qwen1.5-0.5B" + ), + verbose=False, ) response = llama.create_chat_completion( - messages=[ - { - "role": "user", - "content": "What is the capital of France?" - } - ], + messages=[{"role": "user", "content": "What is the capital of France?"}], response_format={ "type": "json_object", "schema": { "type": "object", "properties": { "country": {"type": "string"}, - "capital": {"type": "string"} + "capital": {"type": "string"}, }, "required": ["country", "capital"], - } + }, }, - stream=True + stream=True, ) for chunk in response: @@ -36,4 +33,4 @@ continue print(delta["content"], end="", flush=True) -print() \ No newline at end of file +print() diff --git a/examples/high_level_api/fastapi_server.py b/examples/high_level_api/fastapi_server.py index 9421db57bb..ee59767d65 100644 --- a/examples/high_level_api/fastapi_server.py +++ b/examples/high_level_api/fastapi_server.py @@ -24,6 +24,7 @@ To actually see the implementation of the server, see llama_cpp/server/app.py """ + import os import uvicorn diff --git a/examples/high_level_api/high_level_api_infill.py b/examples/high_level_api/high_level_api_infill.py index 27af6367e7..282333e5a8 100644 --- a/examples/high_level_api/high_level_api_infill.py +++ b/examples/high_level_api/high_level_api_infill.py @@ -6,16 +6,16 @@ parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-models.bin") parser.add_argument("-p", "--prompt", type=str, default="def add(") parser.add_argument("-s", "--suffix", type=str, default="\n return sum\n\n") -parser.add_argument("-i", "--spm-infill", action='store_true') +parser.add_argument("-i", "--spm-infill", action="store_true") args = parser.parse_args() llm = Llama(model_path=args.model, n_gpu_layers=-1, spm_infill=args.spm_infill) output = llm.create_completion( - temperature = 0.0, - repeat_penalty = 1.0, - prompt = args.prompt, - suffix = args.suffix, + temperature=0.0, + repeat_penalty=1.0, + prompt=args.prompt, + suffix=args.suffix, ) # Models sometimes repeat suffix in response, attempt to filter that @@ -25,9 +25,13 @@ unwanted_response_length = len(unwanted_response_suffix) filtered = False -if unwanted_response_suffix and response_stripped[-unwanted_response_length:] == unwanted_response_suffix: +if ( + unwanted_response_suffix + and response_stripped[-unwanted_response_length:] == unwanted_response_suffix +): response = response_stripped[:-unwanted_response_length] filtered = True -print(f"Fill-in-Middle completion{' (filtered)' if filtered else ''}:\n\n{args.prompt}\033[32m{response}\033[{'33' if filtered else '0'}m{args.suffix}\033[0m") - +print( + f"Fill-in-Middle completion{' (filtered)' if filtered else ''}:\n\n{args.prompt}\033[32m{response}\033[{'33' if filtered else '0'}m{args.suffix}\033[0m" +) diff --git a/examples/low_level_api/Chat.py b/examples/low_level_api/Chat.py index fcef8cd800..a755089b2a 100644 --- a/examples/low_level_api/Chat.py +++ b/examples/low_level_api/Chat.py @@ -3,10 +3,12 @@ from common import GptParams from low_level_api_chat_cpp import LLaMAInteract + def env_or_def(env, default): - if (env in os.environ): - return os.environ[env] - return default + if env in os.environ: + return os.environ[env] + return default + AI_NAME = env_or_def("AI_NAME", "ChatLLaMa") MODEL = env_or_def("MODEL", "./models/llama-13B/ggml-model.bin") @@ -15,10 +17,10 @@ def env_or_def(env, default): N_THREAD = int(env_or_def("N_THREAD", "8")) today = datetime.datetime.today() -DATE_YEAR=today.strftime("%Y") -DATE_TIME=today.strftime("%H:%M") +DATE_YEAR = today.strftime("%Y") +DATE_TIME = today.strftime("%H:%M") -prompt=f"""Text transcript of a never ending dialog, where {USER_NAME} interacts with an AI assistant named {AI_NAME}. +prompt = f"""Text transcript of a never ending dialog, where {USER_NAME} interacts with an AI assistant named {AI_NAME}. {AI_NAME} is helpful, kind, honest, friendly, good at writing and never fails to answer {USER_NAME}'s requests immediately and with details and precision. There are no annotations like (30 seconds passed...) or (to himself), just what {USER_NAME} and {AI_NAME} say aloud to each other. The dialog lasts for years, the entirety of it is shared below. It's 10000 pages long. @@ -45,27 +47,29 @@ def env_or_def(env, default): {AI_NAME}: Blue. {USER_NAME}: What time is it? {AI_NAME}: It is {DATE_TIME}. -{USER_NAME}:""" + " ".join(sys.argv[1:]) +{USER_NAME}:""" + " ".join( + sys.argv[1:] +) print("Loading model...") params = GptParams( - n_ctx=2048, - temp=0.7, - top_k=40, - top_p=0.5, - repeat_last_n=256, - n_batch=1024, - repeat_penalty=1.17647, - model=MODEL, - n_threads=N_THREAD, - n_predict=N_PREDICTS, - use_color=True, - interactive=True, - antiprompt=[f"{USER_NAME}:"], - input_prefix=" ", - input_suffix=f"{AI_NAME}:", - prompt=prompt, + n_ctx=2048, + temp=0.7, + top_k=40, + top_p=0.5, + repeat_last_n=256, + n_batch=1024, + repeat_penalty=1.17647, + model=MODEL, + n_threads=N_THREAD, + n_predict=N_PREDICTS, + use_color=True, + interactive=True, + antiprompt=[f"{USER_NAME}:"], + input_prefix=" ", + input_suffix=f"{AI_NAME}:", + prompt=prompt, ) with LLaMAInteract(params) as m: - m.interact() + m.interact() diff --git a/examples/low_level_api/Miku.py b/examples/low_level_api/Miku.py index eb9a2cfa91..e072ab1b10 100644 --- a/examples/low_level_api/Miku.py +++ b/examples/low_level_api/Miku.py @@ -3,10 +3,12 @@ from common import GptParams from low_level_api_chat_cpp import LLaMAInteract + def env_or_def(env, default): - if (env in os.environ): - return os.environ[env] - return default + if env in os.environ: + return os.environ[env] + return default + AI_NAME = env_or_def("AI_NAME", "Miku") MODEL = env_or_def("MODEL", "./models/llama-13B/ggml-model.bin") @@ -14,7 +16,7 @@ def env_or_def(env, default): N_PREDICTS = int(env_or_def("N_PREDICTS", "4096")) N_THREAD = int(env_or_def("N_THREAD", "0")) -prompt=f"""This is a transcript of a 1000 page, never ending conversation between {USER_NAME} and the cute and helpful AI assistant {AI_NAME}. {AI_NAME} is a girl who is an AI running on the users computer. +prompt = f"""This is a transcript of a 1000 page, never ending conversation between {USER_NAME} and the cute and helpful AI assistant {AI_NAME}. {AI_NAME} is a girl who is an AI running on the users computer. {AI_NAME} can think for herself without the user seeing her thoughts by adding a /think prefix to her output. She uses this to reason about the world and to think about what she should say next. {AI_NAME} is always coherent and makes sense, but if she isn't sure if what she is saying is correct she will ask the user for help. {AI_NAME} is a very helpful AI and will help the user with anything they need, she is also very friendly and will try to make the user feel better if they are sad. @@ -32,28 +34,30 @@ def env_or_def(env, default): {AI_NAME}: /think It sounds like {USER_NAME} is happy to have me as their assistant! I'm so happy too! ^_^ Glad that whole emotion thing didn't scare him off! {AI_NAME}: /think I wonder what {USER_NAME} likes to do in his free time? I should ask him about that! {AI_NAME}: What do you like to do in your free time? ^_^ -{USER_NAME}:""" + " ".join(sys.argv[1:]) +{USER_NAME}:""" + " ".join( + sys.argv[1:] +) print("Loading model...") params = GptParams( - n_batch=1024, - n_ctx=2048, - n_keep=-1, - repeat_last_n=256, - repeat_penalty=1.17647, - temp=0.7, - top_k=40, - top_p=0.5, - model=MODEL, - n_predict=N_PREDICTS, - use_color=True, - interactive=True, - antiprompt=[f"{USER_NAME}:"], - prompt=prompt, + n_batch=1024, + n_ctx=2048, + n_keep=-1, + repeat_last_n=256, + repeat_penalty=1.17647, + temp=0.7, + top_k=40, + top_p=0.5, + model=MODEL, + n_predict=N_PREDICTS, + use_color=True, + interactive=True, + antiprompt=[f"{USER_NAME}:"], + prompt=prompt, ) if N_THREAD > 0: - params.n_threads = N_THREAD + params.n_threads = N_THREAD with LLaMAInteract(params) as m: - m.interact() + m.interact() diff --git a/examples/low_level_api/ReasonAct.py b/examples/low_level_api/ReasonAct.py index 82e5c4487e..1f2c590177 100644 --- a/examples/low_level_api/ReasonAct.py +++ b/examples/low_level_api/ReasonAct.py @@ -3,14 +3,16 @@ from common import GptParams from low_level_api_chat_cpp import LLaMAInteract + def env_or_def(env, default): - if (env in os.environ): - return os.environ[env] - return default + if env in os.environ: + return os.environ[env] + return default + MODEL = env_or_def("MODEL", "./models/llama-13B/ggml-model.bin") -prompt=f"""You run in a loop of Thought, Action, Observation. +prompt = f"""You run in a loop of Thought, Action, Observation. At the end of the loop either Answer or restate your Thought and Action. Use Thought to describe your thoughts about the question you have been asked. Use Action to run one of these actions available to you: @@ -27,23 +29,25 @@ def env_or_def(env, default): Question: What is capital of france? Thought: Do I need to use an action? No, I know the answer Answer: Paris is the capital of France -Question:""" + " ".join(sys.argv[1:]) +Question:""" + " ".join( + sys.argv[1:] +) print("Loading model...") params = GptParams( - interactive=True, - interactive_start=True, - top_k=10000, - temp=0.2, - repeat_penalty=1, - n_threads=7, - n_ctx=2048, - antiprompt=["Question:","Observation:"], - model=MODEL, - input_prefix=" ", - n_predict=-1, - prompt=prompt, + interactive=True, + interactive_start=True, + top_k=10000, + temp=0.2, + repeat_penalty=1, + n_threads=7, + n_ctx=2048, + antiprompt=["Question:", "Observation:"], + model=MODEL, + input_prefix=" ", + n_predict=-1, + prompt=prompt, ) with LLaMAInteract(params) as m: - m.interact() + m.interact() diff --git a/examples/low_level_api/common.py b/examples/low_level_api/common.py index 1a5152530c..a0212ff0d2 100644 --- a/examples/low_level_api/common.py +++ b/examples/low_level_api/common.py @@ -65,82 +65,242 @@ class GptParams: # Set to "\nUser:" etc. # This is an alternative to input_prefix which always adds it, so it potentially duplicates "User:"" fix_prefix: str = "" - input_echo: bool = True, + input_echo: bool = (True,) # Default instructions for Alpaca # switch to "Human" and "Assistant" for Vicuna. # TODO: TBD how they are gonna handle this upstream - instruct_inp_prefix: str="\n\n### Instruction:\n\n" - instruct_inp_suffix: str="\n\n### Response:\n\n" + instruct_inp_prefix: str = "\n\n### Instruction:\n\n" + instruct_inp_suffix: str = "\n\n### Response:\n\n" -def gpt_params_parse(argv = None): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-s", "--seed", type=int, default=-1, help="RNG seed (use random seed for <= 0)",dest="seed") - parser.add_argument("-t", "--threads", type=int, default=min(4, os.cpu_count() or 1), help="number of threads to use during computation",dest="n_threads") - parser.add_argument("-n", "--n_predict", type=int, default=128, help="number of tokens to predict (-1 = infinity)",dest="n_predict") - parser.add_argument("--n_parts", type=int, default=-1, help="number of model parts", dest="n_parts") - parser.add_argument("-c", "--ctx_size", type=int, default=512, help="size of the prompt context",dest="n_ctx") - parser.add_argument("-b", "--batch_size", type=int, default=8, help="batch size for prompt processing",dest="n_batch") - parser.add_argument("--keep", type=int, default=0, help="number of tokens to keep from the initial prompt",dest="n_keep") +def gpt_params_parse(argv=None): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "-s", + "--seed", + type=int, + default=-1, + help="RNG seed (use random seed for <= 0)", + dest="seed", + ) + parser.add_argument( + "-t", + "--threads", + type=int, + default=min(4, os.cpu_count() or 1), + help="number of threads to use during computation", + dest="n_threads", + ) + parser.add_argument( + "-n", + "--n_predict", + type=int, + default=128, + help="number of tokens to predict (-1 = infinity)", + dest="n_predict", + ) + parser.add_argument( + "--n_parts", type=int, default=-1, help="number of model parts", dest="n_parts" + ) + parser.add_argument( + "-c", + "--ctx_size", + type=int, + default=512, + help="size of the prompt context", + dest="n_ctx", + ) + parser.add_argument( + "-b", + "--batch_size", + type=int, + default=8, + help="batch size for prompt processing", + dest="n_batch", + ) + parser.add_argument( + "--keep", + type=int, + default=0, + help="number of tokens to keep from the initial prompt", + dest="n_keep", + ) parser.add_argument( "-l", "--logit-bias", type=str, - action='append', + action="append", help="--logit-bias TOKEN_ID(+/-)BIAS", - dest="logit_bias_str" - ) - parser.add_argument("--ignore-eos", action="store_true", help="ignore end of stream token and continue generating", dest="ignore_eos") - parser.add_argument("--top_k", type=int, default=40, help="top-k sampling",dest="top_k") - parser.add_argument("--top_p", type=float, default=0.95, help="top-p samplin",dest="top_p") - parser.add_argument("--tfs", type=float, default=1.0, help="tail free sampling, parameter z (1.0 = disabled)",dest="tfs_z") - parser.add_argument("--temp", type=float, default=0.80, help="temperature",dest="temp") - parser.add_argument("--repeat_penalty", type=float, default=1.10, help="penalize repeat sequence of tokens",dest="repeat_penalty") - parser.add_argument("--repeat_last_n", type=int, default=64, help="last n tokens to consider for penalize ",dest="repeat_last_n") - parser.add_argument("--frequency_penalty", type=float, default=0.0, help="repeat alpha frequency penalty (0.0 = disabled)",dest="tfs_z") - parser.add_argument("--presence_penalty", type=float, default=0.0, help="repeat alpha presence penalty (0.0 = disabled)",dest="presence_penalty") - parser.add_argument("--mirostat", type=float, default=1.0, help="use Mirostat sampling.",dest="mirostat") - parser.add_argument("--mirostat_ent", type=float, default=5.0, help="Mirostat target entropy, parameter tau represents the average surprise value",dest="mirostat_tau") - parser.add_argument("--mirostat_lr", type=float, default=0.1, help="Mirostat learning rate, parameter eta",dest="mirostat_eta") - - parser.add_argument("-m", "--model", type=str, default="./models/llama-7B/ggml-model.bin", help="model path",dest="model") - parser.add_argument("-p", "--prompt", type=str, default=None, help="initial prompt",dest="prompt") - parser.add_argument("-f", "--file", type=str, default=None, help="file containing initial prompt to load",dest="file") - parser.add_argument("--session", type=str, default=None, help="file to cache model state in (may be large!)",dest="path_session") - parser.add_argument("--in-prefix", type=str, default="", help="string to prefix user inputs with", dest="input_prefix") - parser.add_argument("--in-suffix", type=str, default="", help="append to input", dest="input_suffix") + dest="logit_bias_str", + ) + parser.add_argument( + "--ignore-eos", + action="store_true", + help="ignore end of stream token and continue generating", + dest="ignore_eos", + ) + parser.add_argument( + "--top_k", type=int, default=40, help="top-k sampling", dest="top_k" + ) + parser.add_argument( + "--top_p", type=float, default=0.95, help="top-p samplin", dest="top_p" + ) + parser.add_argument( + "--tfs", + type=float, + default=1.0, + help="tail free sampling, parameter z (1.0 = disabled)", + dest="tfs_z", + ) + parser.add_argument( + "--temp", type=float, default=0.80, help="temperature", dest="temp" + ) + parser.add_argument( + "--repeat_penalty", + type=float, + default=1.10, + help="penalize repeat sequence of tokens", + dest="repeat_penalty", + ) + parser.add_argument( + "--repeat_last_n", + type=int, + default=64, + help="last n tokens to consider for penalize ", + dest="repeat_last_n", + ) + parser.add_argument( + "--frequency_penalty", + type=float, + default=0.0, + help="repeat alpha frequency penalty (0.0 = disabled)", + dest="tfs_z", + ) + parser.add_argument( + "--presence_penalty", + type=float, + default=0.0, + help="repeat alpha presence penalty (0.0 = disabled)", + dest="presence_penalty", + ) + parser.add_argument( + "--mirostat", + type=float, + default=1.0, + help="use Mirostat sampling.", + dest="mirostat", + ) + parser.add_argument( + "--mirostat_ent", + type=float, + default=5.0, + help="Mirostat target entropy, parameter tau represents the average surprise value", + dest="mirostat_tau", + ) + parser.add_argument( + "--mirostat_lr", + type=float, + default=0.1, + help="Mirostat learning rate, parameter eta", + dest="mirostat_eta", + ) + + parser.add_argument( + "-m", + "--model", + type=str, + default="./models/llama-7B/ggml-model.bin", + help="model path", + dest="model", + ) + parser.add_argument( + "-p", "--prompt", type=str, default=None, help="initial prompt", dest="prompt" + ) + parser.add_argument( + "-f", + "--file", + type=str, + default=None, + help="file containing initial prompt to load", + dest="file", + ) + parser.add_argument( + "--session", + type=str, + default=None, + help="file to cache model state in (may be large!)", + dest="path_session", + ) + parser.add_argument( + "--in-prefix", + type=str, + default="", + help="string to prefix user inputs with", + dest="input_prefix", + ) + parser.add_argument( + "--in-suffix", type=str, default="", help="append to input", dest="input_suffix" + ) parser.add_argument( "-r", "--reverse-prompt", type=str, - action='append', + action="append", help="poll user input upon seeing PROMPT (can be\nspecified more than once for multiple prompts).", - dest="antiprompt" + dest="antiprompt", + ) + + parser.add_argument( + "--lora", + type=str, + default="", + help="apply LoRA adapter (implies --no-mmap)", + dest="lora_adapter", + ) + parser.add_argument( + "--lora-base", + type=str, + default="", + help="optional model to use as a base for the layers modified by the LoRA adapter", + dest="lora_base", ) - - parser.add_argument("--lora", type=str, default="", help="apply LoRA adapter (implies --no-mmap)", dest="lora_adapter") - parser.add_argument("--lora-base", type=str, default="", help="optional model to use as a base for the layers modified by the LoRA adapter", dest="lora_base") - parser.add_argument("--memory_f32", action="store_false", help="use f32 instead of f16 for memory key+value",dest="memory_f16") - parser.add_argument("--random-prompt", action="store_true", help="start with a randomized prompt.", dest="random_prompt") + parser.add_argument( + "--memory_f32", + action="store_false", + help="use f32 instead of f16 for memory key+value", + dest="memory_f16", + ) + parser.add_argument( + "--random-prompt", + action="store_true", + help="start with a randomized prompt.", + dest="random_prompt", + ) parser.add_argument( "--color", action="store_true", help="colorise output to distinguish prompt and user input from generations", - dest="use_color" + dest="use_color", ) parser.add_argument( - "-i", "--interactive", action="store_true", help="run in interactive mode", dest="interactive" + "-i", + "--interactive", + action="store_true", + help="run in interactive mode", + dest="interactive", ) - + parser.add_argument("--embedding", action="store_true", help="", dest="embedding") parser.add_argument( "--interactive-first", action="store_true", help="run in interactive mode and wait for input right away", - dest="interactive_start" + dest="interactive_start", ) parser.add_argument( @@ -148,42 +308,84 @@ def gpt_params_parse(argv = None): "--instruct", action="store_true", help="run in instruction mode (use with Alpaca or Vicuna models)", - dest="instruct" + dest="instruct", + ) + parser.add_argument( + "--no-penalize-nl", + action="store_false", + help="do not penalize newline token", + dest="penalize_nl", + ) + parser.add_argument( + "--perplexity", + action="store_true", + help="compute perplexity over the prompt", + dest="perplexity", + ) + parser.add_argument( + "--no-mmap", + action="store_false", + help="do not memory-map model (slower load but may reduce pageouts if not using mlock)", + dest="use_mmap", + ) + parser.add_argument( + "--mlock", + action="store_true", + help="force system to keep model in RAM rather than swapping or compressing", + dest="use_mlock", + ) + parser.add_argument( + "--mtest", + action="store_true", + help="compute maximum memory usage", + dest="mem_test", + ) + parser.add_argument( + "--verbose-prompt", + action="store_true", + help="print prompt before generation", + dest="verbose_prompt", ) - parser.add_argument("--no-penalize-nl", action="store_false", help="do not penalize newline token", dest="penalize_nl") - parser.add_argument("--perplexity", action="store_true", help="compute perplexity over the prompt", dest="perplexity") - parser.add_argument("--no-mmap", action="store_false",help="do not memory-map model (slower load but may reduce pageouts if not using mlock)",dest="use_mmap") - parser.add_argument("--mlock", action="store_true",help="force system to keep model in RAM rather than swapping or compressing",dest="use_mlock") - parser.add_argument("--mtest", action="store_true",help="compute maximum memory usage",dest="mem_test") - parser.add_argument("--verbose-prompt", action="store_true",help="print prompt before generation",dest="verbose_prompt") - #Custom args - parser.add_argument("--fix-prefix", type=str, default="", help="append to input when generated n_predict tokens", dest="fix_prefix") - parser.add_argument("--input-noecho", action="store_false", help="dont output the input", dest="input_echo") + # Custom args + parser.add_argument( + "--fix-prefix", + type=str, + default="", + help="append to input when generated n_predict tokens", + dest="fix_prefix", + ) + parser.add_argument( + "--input-noecho", + action="store_false", + help="dont output the input", + dest="input_echo", + ) parser.add_argument( "--interactive-start", action="store_true", help="run in interactive mode", - dest="interactive" + dest="interactive", ) args = parser.parse_args(argv) - + logit_bias_str = args.logit_bias_str delattr(args, "logit_bias_str") params = GptParams(**vars(args)) - if (params.lora_adapter): + if params.lora_adapter: params.use_mmap = False - if (logit_bias_str != None): + if logit_bias_str != None: for i in logit_bias_str: - if (m := re.match(r"(\d+)([-+]\d+)", i)): + if m := re.match(r"(\d+)([-+]\d+)", i): params.logit_bias[int(m.group(1))] = float(m.group(2)) return params + def gpt_random_prompt(rng): return [ "So", @@ -198,5 +400,6 @@ def gpt_random_prompt(rng): "They", ][rng % 10] + if __name__ == "__main__": print(gpt_params_parse()) diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 02c09afb0b..39081be17a 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -10,6 +10,7 @@ You should also still be feeding the model with a "primer" prompt that shows it the expected format. """ + import ctypes import sys from time import time @@ -19,198 +20,257 @@ from common import GptParams, gpt_params_parse, gpt_random_prompt import util + # A LLaMA interactive session class LLaMAInteract: - def __init__(self, params: GptParams) -> None: - # input args - self.params = params - if self.params.path_session is None: - self.params.path_session = "" - if self.params.antiprompt is None: - self.params.antiprompt = "" - - if (self.params.perplexity): - raise NotImplementedError("""************ + def __init__(self, params: GptParams) -> None: + # input args + self.params = params + if self.params.path_session is None: + self.params.path_session = "" + if self.params.antiprompt is None: + self.params.antiprompt = "" + + if self.params.perplexity: + raise NotImplementedError( + """************ please use the 'perplexity' tool for perplexity calculations -************""") +************""" + ) - if (self.params.embedding): - raise NotImplementedError("""************ + if self.params.embedding: + raise NotImplementedError( + """************ please use the 'embedding' tool for embedding calculations -************""") +************""" + ) - if (self.params.n_ctx > 2048): - print(f"""warning: model does not support \ + if self.params.n_ctx > 2048: + print( + f"""warning: model does not support \ context sizes greater than 2048 tokens ({self.params.n_ctx} \ -specified) expect poor results""", file=sys.stderr) - - if (self.params.seed <= 0): - self.params.seed = int(time()) - - print(f"seed = {self.params.seed}", file=sys.stderr) - - if (self.params.random_prompt): - self.params.prompt = gpt_random_prompt(self.params.seed) - - # runtime args - self.input_consumed = 0 - self.n_past = 0 - self.n_session_consumed = 0 - self.first_antiprompt = [] - self.remaining_tokens = self.params.n_predict - self.output_echo = self.params.input_echo - self.multibyte_fix = [] - - # model load - self.lparams = llama_cpp.llama_model_default_params() - self.lparams.n_ctx = self.params.n_ctx - self.lparams.n_parts = self.params.n_parts - self.lparams.seed = self.params.seed - self.lparams.memory_f16 = self.params.memory_f16 - self.lparams.use_mlock = self.params.use_mlock - self.lparams.use_mmap = self.params.use_mmap - - self.model = llama_cpp.llama_load_model_from_file( - self.params.model.encode("utf8"), self.lparams) - - # Context Params. - self.cparams = llama_cpp.llama_context_default_params() - - self.ctx = llama_cpp.llama_new_context_with_model(self.model, self.cparams) - if (not self.ctx): - raise RuntimeError(f"error: failed to load model '{self.params.model}'") - - if (self.params.ignore_eos): - self.params.logit_bias[llama_cpp.llama_token_eos()] = -float("inf") - - if (len(self.params.lora_adapter) > 0): - if (llama_cpp.llama_apply_lora_from_file( - self.ctx, - self.params.lora_adapter.encode("utf8"), - self.params.lora_base.encode("utf8") if len(self.params.lora_base) > 0 else None, - self.params.n_threads - ) != 0): - print("error: failed to apply lora adapter") - return - - print(file=sys.stderr) - print(f"system_info: n_threads = {self.params.n_threads} / {cpu_count()} \ -| {llama_cpp.llama_print_system_info().decode('utf8')}", file=sys.stderr) - - # determine the required inference memory per token: - if (self.params.mem_test): - tmp = [0, 1, 2, 3] - llama_cpp.llama_eval(self.ctx, (llama_cpp.c_int * len(tmp))(*tmp), len(tmp), 0, self.n_threads) - llama_cpp.llama_print_timings(self.ctx) - self.exit() - return - - # create internal context - self.n_ctx = llama_cpp.llama_n_ctx(self.ctx) - - # Add a space in front of the first character to match OG llama tokenizer behavior - self.params.prompt = " " + self.params.prompt - - # Load prompt file - if (self.params.file): - with open(self.params.file) as f: - self.params.prompt = f.read() - - self.session_tokens: list[llama_cpp.llama_token] = [] - if (len(self.params.path_session) > 0): - print(f"attempting to load saved session from '{self.params.path_session}'", file=sys.stderr) - - if (path.exists(self.params.path_session)): - _session_tokens = (llama_cpp.llama_token * (self.params.n_ctx))() - _n_token_count_out = llama_cpp.c_size_t() - if (llama_cpp.llama_load_session_file( - self.ctx, - self.params.path_session.encode("utf8"), - _session_tokens, - self.params.n_ctx, - ctypes.byref(_n_token_count_out) - ) != 1): - print(f"error: failed to load session file '{self.params.path_session}'", file=sys.stderr) - return - _n_token_count_out = _n_token_count_out.value - self.session_tokens = _session_tokens[:_n_token_count_out] - print(f"loaded a session with prompt size of {_n_token_count_out} tokens", file=sys.stderr) - else: - print(f"session file does not exist, will create", file=sys.stderr) - - # tokenize the prompt - self.embd = [] - self.embd_inp = self._tokenize(self.params.prompt) - - if (len(self.embd_inp) > self.n_ctx - 4): - raise RuntimeError(f"error: prompt is too long ({len(self.embd_inp)} tokens, max {self.params.n_ctx - 4})") - - # debug message about similarity of saved session, if applicable - self.n_matching_session_tokens = 0 - if len(self.session_tokens) > 0: - for id in self.session_tokens: - if self.n_matching_session_tokens >= len(self.embd_inp) or id != self.embd_inp[self.n_matching_session_tokens]: - break - self.n_matching_session_tokens += 1 - - if self.n_matching_session_tokens >= len(self.embd_inp): - print(f"session file has exact match for prompt!") - elif self.n_matching_session_tokens < (len(self.embd_inp) / 2): - print(f"warning: session file has low similarity to prompt ({self.n_matching_session_tokens} / {len(self.embd_inp)} tokens); will mostly be reevaluated") - else: - print(f"session file matches {self.n_matching_session_tokens} / {len(self.embd_inp)} tokens of prompt") - - self.need_to_save_session = len(self.params.path_session) > 0 and self.n_matching_session_tokens < (len(self.embd_inp) * 3 / 4) - - # number of tokens to keep when resetting context - if (self.params.n_keep < 0 or self.params.n_keep > len(self.embd_inp) or self.params.instruct): - self.params.n_keep = len(self.embd_inp) - - self.inp_prefix = self._tokenize(self.params.instruct_inp_prefix) - self.inp_suffix = self._tokenize(self.params.instruct_inp_suffix, False) - - # in instruct mode, we inject a prefix and a suffix to each input by the user - self.antiecho = None - if (self.params.instruct): - self.params.interactive_start = True - _ptn = self._tokenize(self.params.instruct_inp_prefix.strip(), False) - self.first_antiprompt.append(_ptn) - self.antiecho = util.IterSearch(_ptn) - - # enable interactive mode if reverse prompt or interactive start is specified - if (len(self.params.antiprompt) != 0 or self.params.interactive_start): - self.params.interactive = True - - # determine newline token - self.llama_token_newline = self._tokenize("\n", False) - self.llama_token_eot = self._tokenize(" [end of text]\n", False) - - if (self.params.verbose_prompt): - print(f""" +specified) expect poor results""", + file=sys.stderr, + ) + + if self.params.seed <= 0: + self.params.seed = int(time()) + + print(f"seed = {self.params.seed}", file=sys.stderr) + + if self.params.random_prompt: + self.params.prompt = gpt_random_prompt(self.params.seed) + + # runtime args + self.input_consumed = 0 + self.n_past = 0 + self.n_session_consumed = 0 + self.first_antiprompt = [] + self.remaining_tokens = self.params.n_predict + self.output_echo = self.params.input_echo + self.multibyte_fix = [] + + # model load + self.lparams = llama_cpp.llama_model_default_params() + self.lparams.n_ctx = self.params.n_ctx + self.lparams.n_parts = self.params.n_parts + self.lparams.seed = self.params.seed + self.lparams.memory_f16 = self.params.memory_f16 + self.lparams.use_mlock = self.params.use_mlock + self.lparams.use_mmap = self.params.use_mmap + + self.model = llama_cpp.llama_load_model_from_file( + self.params.model.encode("utf8"), self.lparams + ) + + # Context Params. + self.cparams = llama_cpp.llama_context_default_params() + + self.ctx = llama_cpp.llama_new_context_with_model(self.model, self.cparams) + if not self.ctx: + raise RuntimeError(f"error: failed to load model '{self.params.model}'") + + if self.params.ignore_eos: + self.params.logit_bias[llama_cpp.llama_token_eos()] = -float("inf") + + if len(self.params.lora_adapter) > 0: + if ( + llama_cpp.llama_apply_lora_from_file( + self.ctx, + self.params.lora_adapter.encode("utf8"), + ( + self.params.lora_base.encode("utf8") + if len(self.params.lora_base) > 0 + else None + ), + self.params.n_threads, + ) + != 0 + ): + print("error: failed to apply lora adapter") + return + + print(file=sys.stderr) + print( + f"system_info: n_threads = {self.params.n_threads} / {cpu_count()} \ +| {llama_cpp.llama_print_system_info().decode('utf8')}", + file=sys.stderr, + ) + + # determine the required inference memory per token: + if self.params.mem_test: + tmp = [0, 1, 2, 3] + llama_cpp.llama_eval( + self.ctx, + (llama_cpp.c_int * len(tmp))(*tmp), + len(tmp), + 0, + self.n_threads, + ) + llama_cpp.llama_print_timings(self.ctx) + self.exit() + return + + # create internal context + self.n_ctx = llama_cpp.llama_n_ctx(self.ctx) + + # Add a space in front of the first character to match OG llama tokenizer behavior + self.params.prompt = " " + self.params.prompt + + # Load prompt file + if self.params.file: + with open(self.params.file) as f: + self.params.prompt = f.read() + + self.session_tokens: list[llama_cpp.llama_token] = [] + if len(self.params.path_session) > 0: + print( + f"attempting to load saved session from '{self.params.path_session}'", + file=sys.stderr, + ) + + if path.exists(self.params.path_session): + _session_tokens = (llama_cpp.llama_token * (self.params.n_ctx))() + _n_token_count_out = llama_cpp.c_size_t() + if ( + llama_cpp.llama_load_session_file( + self.ctx, + self.params.path_session.encode("utf8"), + _session_tokens, + self.params.n_ctx, + ctypes.byref(_n_token_count_out), + ) + != 1 + ): + print( + f"error: failed to load session file '{self.params.path_session}'", + file=sys.stderr, + ) + return + _n_token_count_out = _n_token_count_out.value + self.session_tokens = _session_tokens[:_n_token_count_out] + print( + f"loaded a session with prompt size of {_n_token_count_out} tokens", + file=sys.stderr, + ) + else: + print(f"session file does not exist, will create", file=sys.stderr) + + # tokenize the prompt + self.embd = [] + self.embd_inp = self._tokenize(self.params.prompt) + + if len(self.embd_inp) > self.n_ctx - 4: + raise RuntimeError( + f"error: prompt is too long ({len(self.embd_inp)} tokens, max {self.params.n_ctx - 4})" + ) + + # debug message about similarity of saved session, if applicable + self.n_matching_session_tokens = 0 + if len(self.session_tokens) > 0: + for id in self.session_tokens: + if ( + self.n_matching_session_tokens >= len(self.embd_inp) + or id != self.embd_inp[self.n_matching_session_tokens] + ): + break + self.n_matching_session_tokens += 1 + + if self.n_matching_session_tokens >= len(self.embd_inp): + print(f"session file has exact match for prompt!") + elif self.n_matching_session_tokens < (len(self.embd_inp) / 2): + print( + f"warning: session file has low similarity to prompt ({self.n_matching_session_tokens} / {len(self.embd_inp)} tokens); will mostly be reevaluated" + ) + else: + print( + f"session file matches {self.n_matching_session_tokens} / {len(self.embd_inp)} tokens of prompt" + ) + + self.need_to_save_session = len( + self.params.path_session + ) > 0 and self.n_matching_session_tokens < (len(self.embd_inp) * 3 / 4) + + # number of tokens to keep when resetting context + if ( + self.params.n_keep < 0 + or self.params.n_keep > len(self.embd_inp) + or self.params.instruct + ): + self.params.n_keep = len(self.embd_inp) + + self.inp_prefix = self._tokenize(self.params.instruct_inp_prefix) + self.inp_suffix = self._tokenize(self.params.instruct_inp_suffix, False) + + # in instruct mode, we inject a prefix and a suffix to each input by the user + self.antiecho = None + if self.params.instruct: + self.params.interactive_start = True + _ptn = self._tokenize(self.params.instruct_inp_prefix.strip(), False) + self.first_antiprompt.append(_ptn) + self.antiecho = util.IterSearch(_ptn) + + # enable interactive mode if reverse prompt or interactive start is specified + if len(self.params.antiprompt) != 0 or self.params.interactive_start: + self.params.interactive = True + + # determine newline token + self.llama_token_newline = self._tokenize("\n", False) + self.llama_token_eot = self._tokenize(" [end of text]\n", False) + + if self.params.verbose_prompt: + print( + f""" prompt: '{self.params.prompt}' -number of tokens in prompt = {len(self.embd_inp)}""", file=sys.stderr) - - for i in range(len(self.embd_inp)): - print(f"{self.embd_inp[i]} -> '{self.token_to_str(self.embd_inp[i])}'", file=sys.stderr) - - if (self.params.n_keep > 0): - print("static prompt based on n_keep: '") - for i in range(self.params.n_keep): - print(self.token_to_str(self.embd_inp[i]), file=sys.stderr) - print("'", file=sys.stderr) - print(file=sys.stderr) - - if (self.params.interactive): - print("interactive mode on.", file=sys.stderr) - - if (len(self.params.antiprompt) > 0): - for antiprompt in self.params.antiprompt: - print(f"Reverse prompt: '{antiprompt}'", file=sys.stderr) - - if len(self.params.input_prefix) > 0: - print(f"Input prefix: '{self.params.input_prefix}'", file=sys.stderr) - - print(f"""sampling: repeat_last_n = {self.params.repeat_last_n},\ +number of tokens in prompt = {len(self.embd_inp)}""", + file=sys.stderr, + ) + + for i in range(len(self.embd_inp)): + print( + f"{self.embd_inp[i]} -> '{self.token_to_str(self.embd_inp[i])}'", + file=sys.stderr, + ) + + if self.params.n_keep > 0: + print("static prompt based on n_keep: '") + for i in range(self.params.n_keep): + print(self.token_to_str(self.embd_inp[i]), file=sys.stderr) + print("'", file=sys.stderr) + print(file=sys.stderr) + + if self.params.interactive: + print("interactive mode on.", file=sys.stderr) + + if len(self.params.antiprompt) > 0: + for antiprompt in self.params.antiprompt: + print(f"Reverse prompt: '{antiprompt}'", file=sys.stderr) + + if len(self.params.input_prefix) > 0: + print(f"Input prefix: '{self.params.input_prefix}'", file=sys.stderr) + + print( + f"""sampling: repeat_last_n = {self.params.repeat_last_n},\ repeat_penalty = {self.params.repeat_penalty},\ presence_penalty = {self.params.presence_penalty},\ frequency_penalty = {self.params.frequency_penalty},\ @@ -228,77 +288,96 @@ def __init__(self, params: GptParams) -> None: n_predict = {self.params.n_predict},\ n_keep = {self.params.n_keep} -""", file=sys.stderr) +""", + file=sys.stderr, + ) - # determine antiprompt tokens - for i in self.params.antiprompt: - self.first_antiprompt.append(self._tokenize(i, False)) + # determine antiprompt tokens + for i in self.params.antiprompt: + self.first_antiprompt.append(self._tokenize(i, False)) - self.last_n_tokens = [0]*self.n_ctx #TODO: deque doesnt support slices + self.last_n_tokens = [0] * self.n_ctx # TODO: deque doesnt support slices - if (params.interactive): - print("""== Running in interactive mode. == + if params.interactive: + print( + """== Running in interactive mode. == - Press Ctrl+C to interject at any time. - Press Return to return control to LLaMa. - If you want to submit another line, end your input in '\\'. -""", file=sys.stderr) - self.set_color(util.CONSOLE_COLOR_PROMPT) - - # tokenize a prompt - def _tokenize(self, prompt, bos=True): - _arr = (llama_cpp.llama_token * ((len(prompt) + 1) * 4))() - _n = llama_cpp.llama_tokenize(self.model, prompt.encode("utf8", errors="ignore"), len(prompt), _arr, len(_arr), bos, False) - return _arr[:_n] - - def set_color(self, c): - if (self.params.use_color): - print(c, end="") - - def use_antiprompt(self): - return len(self.first_antiprompt) > 0 - - # generate tokens - def generate(self): - while self.remaining_tokens > 0 or self.params.interactive or self.params.n_predict == -1: - # predict - if len(self.embd) > 0: - # infinite text generation via context swapping - # if we run out of context: - # - take the n_keep first tokens from the original prompt (via n_past) - # - take half of the last (n_ctx - n_keep) tokens and recompute the logits in a batch - if (self.n_past + len(self.embd) > self.n_ctx): - n_left = self.n_past - self.params.n_keep - self.n_past = self.params.n_keep - - # insert n_left/2 tokens at the start of embd from last_n_tokens - _insert = self.last_n_tokens[ - self.n_ctx - int(n_left/2) - len(self.embd):-len(self.embd) - ] - self.embd = _insert + self.embd - self.params.path_session = "" - - # try to reuse a matching prefix from the loaded session instead of re-eval (via n_past) - if self.n_session_consumed < len(self.session_tokens): - for i in range(len(self.embd)): - if self.embd[i] != self.session_tokens[self.n_session_consumed]: - self.session_tokens = self.session_tokens[:self.n_session_consumed] - break - - self.n_past += 1 - self.n_session_consumed += 1 - - if self.n_session_consumed >= len(self.session_tokens): - i += 1 - break - - if i > 0: - self.embd = self.embd[i:] - - # evaluate tokens in batches - # embd is typically prepared beforehand to fit within a batch, but not always - #TODO BUG: The batching code causes nonsensical generation - """for i in range(0, len(self.embd), self.params.n_batch): +""", + file=sys.stderr, + ) + self.set_color(util.CONSOLE_COLOR_PROMPT) + + # tokenize a prompt + def _tokenize(self, prompt, bos=True): + _arr = (llama_cpp.llama_token * ((len(prompt) + 1) * 4))() + _n = llama_cpp.llama_tokenize( + self.model, + prompt.encode("utf8", errors="ignore"), + len(prompt), + _arr, + len(_arr), + bos, + False, + ) + return _arr[:_n] + + def set_color(self, c): + if self.params.use_color: + print(c, end="") + + def use_antiprompt(self): + return len(self.first_antiprompt) > 0 + + # generate tokens + def generate(self): + while ( + self.remaining_tokens > 0 + or self.params.interactive + or self.params.n_predict == -1 + ): + # predict + if len(self.embd) > 0: + # infinite text generation via context swapping + # if we run out of context: + # - take the n_keep first tokens from the original prompt (via n_past) + # - take half of the last (n_ctx - n_keep) tokens and recompute the logits in a batch + if self.n_past + len(self.embd) > self.n_ctx: + n_left = self.n_past - self.params.n_keep + self.n_past = self.params.n_keep + + # insert n_left/2 tokens at the start of embd from last_n_tokens + _insert = self.last_n_tokens[ + self.n_ctx - int(n_left / 2) - len(self.embd) : -len(self.embd) + ] + self.embd = _insert + self.embd + self.params.path_session = "" + + # try to reuse a matching prefix from the loaded session instead of re-eval (via n_past) + if self.n_session_consumed < len(self.session_tokens): + for i in range(len(self.embd)): + if self.embd[i] != self.session_tokens[self.n_session_consumed]: + self.session_tokens = self.session_tokens[ + : self.n_session_consumed + ] + break + + self.n_past += 1 + self.n_session_consumed += 1 + + if self.n_session_consumed >= len(self.session_tokens): + i += 1 + break + + if i > 0: + self.embd = self.embd[i:] + + # evaluate tokens in batches + # embd is typically prepared beforehand to fit within a batch, but not always + # TODO BUG: The batching code causes nonsensical generation + """for i in range(0, len(self.embd), self.params.n_batch): n_eval = self.params.n_batch _arr = (llama_cpp.llama_token * n_eval)(*self.embd[i:i + n_eval]) if llama_cpp.llama_eval(self.ctx, _arr, n_eval, self.n_past, self.params.n_threads) != 0: @@ -307,278 +386,356 @@ def generate(self): self.n_past += n_eval""" - if (llama_cpp.llama_eval( - self.ctx, (llama_cpp.llama_token * len(self.embd))(*self.embd), len(self.embd), self.n_past - ) != 0): - raise Exception("Failed to llama_eval!") - - if len(self.embd) > 0 and len(self.params.path_session) > 0: - self.session_tokens.extend(self.embd) - self.n_session_consumed = len(self.session_tokens) - - self.n_past += len(self.embd) - self.embd = [] - if len(self.embd_inp) <= self.input_consumed: #&& !is_interacting - # out of user input, sample next token - top_k = llama_cpp.llama_n_vocab(self.ctx) if self.params.top_k <= 0 else self.params.top_k - repeat_last_n = self.n_ctx if self.params.repeat_last_n < 0 else self.params.repeat_last_n - - # optionally save the session on first sample (for faster prompt loading next time) - if len(self.params.path_session) > 0 and self.need_to_save_session: - self.need_to_save_session = False - llama_cpp.llama_save_session_file( - self.ctx, - self.params.path_session.encode("utf8"), - (llama_cpp.llama_token * len(self.session_tokens))(*self.session_tokens), - len(self.session_tokens) - ) - - id = 0 - - logits = llama_cpp.llama_get_logits(self.ctx) - n_vocab = llama_cpp.llama_n_vocab(self.model) - - # Apply params.logit_bias map - for key, value in self.params.logit_bias.items(): - logits[key] += value - - _arr = (llama_cpp.llama_token_data * n_vocab)(*[ - llama_cpp.llama_token_data(token_id, logits[token_id], 0.0) - for token_id in range(n_vocab) - ]) - candidates_p = llama_cpp.ctypes.pointer(llama_cpp.llama_token_data_array(_arr, len(_arr), False)) - - # Apply penalties - nl_logit = logits[llama_cpp.llama_token_nl(self.ctx)] - last_n_repeat = min(len(self.last_n_tokens), repeat_last_n, self.n_ctx) - - _arr = (llama_cpp.llama_token * last_n_repeat)(*self.last_n_tokens[len(self.last_n_tokens) - last_n_repeat:]) - llama_cpp.llama_sample_repetition_penalties( - ctx=self.ctx, - candidates=candidates_p, - last_tokens_data = _arr, - penalty_last_n = last_n_repeat, - penalty_repeat = llama_cpp.c_float(self.params.repeat_penalty), - penalty_freq = llama_cpp.c_float(self.params.frequency_penalty), - penalty_present = llama_cpp.c_float(self.params.presence_penalty), - ) - - # NOT PRESENT IN CURRENT VERSION ? - # llama_cpp.llama_sample_frequency_and_presence_penalti(self.ctx, candidates_p, - # _arr, - # last_n_repeat, llama_cpp.c_float(self.params.frequency_penalty), llama_cpp.c_float(self.params.presence_penalty)) - - if not self.params.penalize_nl: - logits[llama_cpp.llama_token_nl()] = nl_logit - - if self.params.temp <= 0: - # Greedy sampling - id = llama_cpp.llama_sample_token_greedy(self.ctx, candidates_p) - else: - if self.params.mirostat == 1: - mirostat_mu = 2.0 * self.params.mirostat_tau - mirostat_m = 100 - llama_cpp.llama_sample_temperature(self.ctx, candidates_p, llama_cpp.c_float(self.params.temp)) - id = llama_cpp.llama_sample_token_mirostat(self.ctx, candidates_p, llama_cpp.c_float(self.params.mirostat_tau), llama_cpp.c_float(self.params.mirostat_eta), llama_cpp.c_int(mirostat_m), llama_cpp.c_float(mirostat_mu)) - elif self.params.mirostat == 2: - mirostat_mu = 2.0 * self.params.mirostat_tau - llama_cpp.llama_sample_temperature(self.ctx, candidates_p, llama_cpp.c_float(self.params.temp)) - id = llama_cpp.llama_sample_token_mirostat_v2(self.ctx, candidates_p, llama_cpp.c_float(self.params.mirostat_tau), llama_cpp.c_float(self.params.mirostat_eta), llama_cpp.c_float(mirostat_mu)) - else: - # Temperature sampling - llama_cpp.llama_sample_top_k(self.ctx, candidates_p, top_k, min_keep=llama_cpp.c_size_t(1)) - llama_cpp.llama_sample_tail_free(self.ctx, candidates_p, llama_cpp.c_float(self.params.tfs_z), min_keep=llama_cpp.c_size_t(1)) - llama_cpp.llama_sample_typical(self.ctx, candidates_p, llama_cpp.c_float(self.params.typical_p), min_keep=llama_cpp.c_size_t(1)) - llama_cpp.llama_sample_top_p(self.ctx, candidates_p, llama_cpp.c_float(self.params.top_p), min_keep=llama_cpp.c_size_t(1)) - llama_cpp.llama_sample_temperature(self.ctx, candidates_p, llama_cpp.c_float(self.params.temp)) - id = llama_cpp.llama_sample_token(self.ctx, candidates_p) - # print("`{}`".format(candidates_p.size)) - - self.last_n_tokens.pop(0) - self.last_n_tokens.append(id) - - # replace end of text token with newline token when in interactive mode - if (id == llama_cpp.llama_token_eos(self.ctx) and self.params.interactive and not self.params.instruct): - id = self.llama_token_newline[0] - self.embd.append(id) - if (self.use_antiprompt()): - # tokenize and inject first reverse prompt - self.embd_inp += self.first_antiprompt[0] - for id in self.first_antiprompt[0]: - self.embd.append(id) - else: - # add it to the context - self.embd.append(id) - - # echo this to console - self.output_echo = True - - # decrement remaining sampling budget - self.remaining_tokens -= 1 - else: - # output to console if input echo is on - self.output_echo = self.params.input_echo - - # some user input remains from prompt or interaction, forward it to processing - while len(self.embd_inp) > self.input_consumed: - self.embd.append(self.embd_inp[self.input_consumed]) - self.last_n_tokens.pop(0) - self.last_n_tokens.append(self.embd_inp[self.input_consumed]) - self.input_consumed += 1 - if len(self.embd) >= self.params.n_batch: - break - - # display tokens - if self.output_echo: - for id in self.embd: - if self.antiecho != None: - for r in self.antiecho(id): - yield r - else: - yield id - - # reset color to default if we there is no pending user input - if (self.params.input_echo and len(self.embd_inp) == self.input_consumed): - self.set_color(util.CONSOLE_COLOR_DEFAULT) - - if (self.params.interactive and len(self.embd_inp) <= self.input_consumed): - # if antiprompt is present, stop - if (self.use_antiprompt()): - if True in [ - i == self.last_n_tokens[-len(i):] - for i in self.first_antiprompt - ]: - break - - # if we are using instruction mode, and we have processed the initial prompt - if (self.params.interactive_start): - break - - # end of text token - if len(self.embd) > 0 and self.embd[-1] == llama_cpp.llama_token_eos(self.ctx): - if (not self.params.instruct): - for i in self.llama_token_eot: - yield i - break - - # respect n_predict even if antiprompt is present - if (self.params.interactive and self.remaining_tokens <= 0 and self.params.n_predict != -1): - # If we arent in instruction mode, fix the current generation by appending the antiprompt. - # Makes it so if chat ends prematurely you dont append the AI's text etc. - if not self.params.instruct: - self.embd_inp += self.first_antiprompt[0] - self.n_remain = self.params.n_predict - break - - self.params.interactive_start = False - - def __enter__(self): - return self - - def __exit__(self, type, value, tb): - self.exit() - - def exit(self): - llama_cpp.llama_free(self.ctx) - self.set_color(util.CONSOLE_COLOR_DEFAULT) - - def token_to_str(self, token_id: int) -> bytes: - size = 32 - buffer = (ctypes.c_char * size)() - n = llama_cpp.llama_token_to_piece( - self.model, llama_cpp.llama_token(token_id), buffer, size) - assert n <= size - return bytes(buffer[:n]) - - # return past text - def past(self): - for id in self.last_n_tokens[-self.n_past:]: - yield self.token_to_str(id).decode("utf8", errors="ignore") - - # write input - def input(self, prompt: str): - if (self.params.instruct and self.last_n_tokens[-len(self.inp_prefix):] != self.inp_prefix): - self.embd_inp += self.inp_prefix - self.embd_inp += self._tokenize(prompt) - if (self.params.instruct): - self.embd_inp += self.inp_suffix - - # write output - def output(self): - self.remaining_tokens = self.params.n_predict - for id in self.generate(): - cur_char = self.token_to_str(id) - - # Add remainder of missing bytes - if None in self.multibyte_fix: - self.multibyte_fix[self.multibyte_fix.index(None)] = cur_char - - # Return completed utf char - if len(self.multibyte_fix) > 0 and not None in self.multibyte_fix: - yield (b"".join(self.multibyte_fix)).decode("utf8") - self.multibyte_fix = [] - continue - - # Contains multi-byte UTF8 - for num, pattern in [(2, 192), (3, 224), (4, 240)]: - # Bitwise AND check - if pattern & int.from_bytes(cur_char, 'little') == pattern: - self.multibyte_fix = [cur_char] + ([None] * (num-1)) - - # Stop incomplete bytes from passing - if len(self.multibyte_fix) > 0: - continue - - yield cur_char.decode("utf8") - - # read user input - def read_input(self): - out = "" - while (t := input()).endswith("\\"): - out += t[:-1] + "\n" - return out + t + "\n" - - # interactive mode - def interact(self): - for i in self.output(): - print(i,end="",flush=True) - self.params.input_echo = False + if ( + llama_cpp.llama_eval( + self.ctx, + (llama_cpp.llama_token * len(self.embd))(*self.embd), + len(self.embd), + self.n_past, + ) + != 0 + ): + raise Exception("Failed to llama_eval!") + + if len(self.embd) > 0 and len(self.params.path_session) > 0: + self.session_tokens.extend(self.embd) + self.n_session_consumed = len(self.session_tokens) + + self.n_past += len(self.embd) + self.embd = [] + if len(self.embd_inp) <= self.input_consumed: # && !is_interacting + # out of user input, sample next token + top_k = ( + llama_cpp.llama_n_vocab(self.ctx) + if self.params.top_k <= 0 + else self.params.top_k + ) + repeat_last_n = ( + self.n_ctx + if self.params.repeat_last_n < 0 + else self.params.repeat_last_n + ) + + # optionally save the session on first sample (for faster prompt loading next time) + if len(self.params.path_session) > 0 and self.need_to_save_session: + self.need_to_save_session = False + llama_cpp.llama_save_session_file( + self.ctx, + self.params.path_session.encode("utf8"), + (llama_cpp.llama_token * len(self.session_tokens))( + *self.session_tokens + ), + len(self.session_tokens), + ) + + id = 0 + + logits = llama_cpp.llama_get_logits(self.ctx) + n_vocab = llama_cpp.llama_n_vocab(self.model) + + # Apply params.logit_bias map + for key, value in self.params.logit_bias.items(): + logits[key] += value + + _arr = (llama_cpp.llama_token_data * n_vocab)( + *[ + llama_cpp.llama_token_data(token_id, logits[token_id], 0.0) + for token_id in range(n_vocab) + ] + ) + candidates_p = llama_cpp.ctypes.pointer( + llama_cpp.llama_token_data_array(_arr, len(_arr), False) + ) + + # Apply penalties + nl_logit = logits[llama_cpp.llama_token_nl(self.ctx)] + last_n_repeat = min(len(self.last_n_tokens), repeat_last_n, self.n_ctx) + + _arr = (llama_cpp.llama_token * last_n_repeat)( + *self.last_n_tokens[len(self.last_n_tokens) - last_n_repeat :] + ) + llama_cpp.llama_sample_repetition_penalties( + ctx=self.ctx, + candidates=candidates_p, + last_tokens_data=_arr, + penalty_last_n=last_n_repeat, + penalty_repeat=llama_cpp.c_float(self.params.repeat_penalty), + penalty_freq=llama_cpp.c_float(self.params.frequency_penalty), + penalty_present=llama_cpp.c_float(self.params.presence_penalty), + ) + + # NOT PRESENT IN CURRENT VERSION ? + # llama_cpp.llama_sample_frequency_and_presence_penalti(self.ctx, candidates_p, + # _arr, + # last_n_repeat, llama_cpp.c_float(self.params.frequency_penalty), llama_cpp.c_float(self.params.presence_penalty)) + + if not self.params.penalize_nl: + logits[llama_cpp.llama_token_nl()] = nl_logit + + if self.params.temp <= 0: + # Greedy sampling + id = llama_cpp.llama_sample_token_greedy(self.ctx, candidates_p) + else: + if self.params.mirostat == 1: + mirostat_mu = 2.0 * self.params.mirostat_tau + mirostat_m = 100 + llama_cpp.llama_sample_temperature( + self.ctx, candidates_p, llama_cpp.c_float(self.params.temp) + ) + id = llama_cpp.llama_sample_token_mirostat( + self.ctx, + candidates_p, + llama_cpp.c_float(self.params.mirostat_tau), + llama_cpp.c_float(self.params.mirostat_eta), + llama_cpp.c_int(mirostat_m), + llama_cpp.c_float(mirostat_mu), + ) + elif self.params.mirostat == 2: + mirostat_mu = 2.0 * self.params.mirostat_tau + llama_cpp.llama_sample_temperature( + self.ctx, candidates_p, llama_cpp.c_float(self.params.temp) + ) + id = llama_cpp.llama_sample_token_mirostat_v2( + self.ctx, + candidates_p, + llama_cpp.c_float(self.params.mirostat_tau), + llama_cpp.c_float(self.params.mirostat_eta), + llama_cpp.c_float(mirostat_mu), + ) + else: + # Temperature sampling + llama_cpp.llama_sample_top_k( + self.ctx, + candidates_p, + top_k, + min_keep=llama_cpp.c_size_t(1), + ) + llama_cpp.llama_sample_tail_free( + self.ctx, + candidates_p, + llama_cpp.c_float(self.params.tfs_z), + min_keep=llama_cpp.c_size_t(1), + ) + llama_cpp.llama_sample_typical( + self.ctx, + candidates_p, + llama_cpp.c_float(self.params.typical_p), + min_keep=llama_cpp.c_size_t(1), + ) + llama_cpp.llama_sample_top_p( + self.ctx, + candidates_p, + llama_cpp.c_float(self.params.top_p), + min_keep=llama_cpp.c_size_t(1), + ) + llama_cpp.llama_sample_temperature( + self.ctx, candidates_p, llama_cpp.c_float(self.params.temp) + ) + id = llama_cpp.llama_sample_token(self.ctx, candidates_p) + # print("`{}`".format(candidates_p.size)) + + self.last_n_tokens.pop(0) + self.last_n_tokens.append(id) + + # replace end of text token with newline token when in interactive mode + if ( + id == llama_cpp.llama_token_eos(self.ctx) + and self.params.interactive + and not self.params.instruct + ): + id = self.llama_token_newline[0] + self.embd.append(id) + if self.use_antiprompt(): + # tokenize and inject first reverse prompt + self.embd_inp += self.first_antiprompt[0] + for id in self.first_antiprompt[0]: + self.embd.append(id) + else: + # add it to the context + self.embd.append(id) + + # echo this to console + self.output_echo = True + + # decrement remaining sampling budget + self.remaining_tokens -= 1 + else: + # output to console if input echo is on + self.output_echo = self.params.input_echo + + # some user input remains from prompt or interaction, forward it to processing + while len(self.embd_inp) > self.input_consumed: + self.embd.append(self.embd_inp[self.input_consumed]) + self.last_n_tokens.pop(0) + self.last_n_tokens.append(self.embd_inp[self.input_consumed]) + self.input_consumed += 1 + if len(self.embd) >= self.params.n_batch: + break + + # display tokens + if self.output_echo: + for id in self.embd: + if self.antiecho != None: + for r in self.antiecho(id): + yield r + else: + yield id + + # reset color to default if we there is no pending user input + if self.params.input_echo and len(self.embd_inp) == self.input_consumed: + self.set_color(util.CONSOLE_COLOR_DEFAULT) + + if self.params.interactive and len(self.embd_inp) <= self.input_consumed: + # if antiprompt is present, stop + if self.use_antiprompt(): + if True in [ + i == self.last_n_tokens[-len(i) :] + for i in self.first_antiprompt + ]: + break + + # if we are using instruction mode, and we have processed the initial prompt + if self.params.interactive_start: + break + + # end of text token + if len(self.embd) > 0 and self.embd[-1] == llama_cpp.llama_token_eos( + self.ctx + ): + if not self.params.instruct: + for i in self.llama_token_eot: + yield i + break + + # respect n_predict even if antiprompt is present + if ( + self.params.interactive + and self.remaining_tokens <= 0 + and self.params.n_predict != -1 + ): + # If we arent in instruction mode, fix the current generation by appending the antiprompt. + # Makes it so if chat ends prematurely you dont append the AI's text etc. + if not self.params.instruct: + self.embd_inp += self.first_antiprompt[0] + self.n_remain = self.params.n_predict + break + + self.params.interactive_start = False + + def __enter__(self): + return self + + def __exit__(self, type, value, tb): + self.exit() + + def exit(self): + llama_cpp.llama_free(self.ctx) + self.set_color(util.CONSOLE_COLOR_DEFAULT) + + def token_to_str(self, token_id: int) -> bytes: + size = 32 + buffer = (ctypes.c_char * size)() + n = llama_cpp.llama_token_to_piece( + self.model, llama_cpp.llama_token(token_id), buffer, size + ) + assert n <= size + return bytes(buffer[:n]) + + # return past text + def past(self): + for id in self.last_n_tokens[-self.n_past :]: + yield self.token_to_str(id).decode("utf8", errors="ignore") + + # write input + def input(self, prompt: str): + if ( + self.params.instruct + and self.last_n_tokens[-len(self.inp_prefix) :] != self.inp_prefix + ): + self.embd_inp += self.inp_prefix + self.embd_inp += self._tokenize(prompt) + if self.params.instruct: + self.embd_inp += self.inp_suffix + + # write output + def output(self): + self.remaining_tokens = self.params.n_predict + for id in self.generate(): + cur_char = self.token_to_str(id) + + # Add remainder of missing bytes + if None in self.multibyte_fix: + self.multibyte_fix[self.multibyte_fix.index(None)] = cur_char + + # Return completed utf char + if len(self.multibyte_fix) > 0 and not None in self.multibyte_fix: + yield (b"".join(self.multibyte_fix)).decode("utf8") + self.multibyte_fix = [] + continue + + # Contains multi-byte UTF8 + for num, pattern in [(2, 192), (3, 224), (4, 240)]: + # Bitwise AND check + if pattern & int.from_bytes(cur_char, "little") == pattern: + self.multibyte_fix = [cur_char] + ([None] * (num - 1)) + + # Stop incomplete bytes from passing + if len(self.multibyte_fix) > 0: + continue + + yield cur_char.decode("utf8") + + # read user input + def read_input(self): + out = "" + while (t := input()).endswith("\\"): + out += t[:-1] + "\n" + return out + t + "\n" + + # interactive mode + def interact(self): + for i in self.output(): + print(i, end="", flush=True) + self.params.input_echo = False # Using string instead of tokens to check for antiprompt, - # It is more reliable than tokens for interactive mode. - generated_str = "" - while self.params.interactive: - self.set_color(util.CONSOLE_COLOR_USER_INPUT) - if (self.params.instruct): - print('\n> ', end="") - self.input(self.read_input()) - else: - print(self.params.input_prefix, end="") - self.input(f"{self.params.input_prefix}{self.read_input()}{self.params.input_suffix}") - print(self.params.input_suffix,end="") - self.set_color(util.CONSOLE_COLOR_DEFAULT) - - try: - for i in self.output(): - print(i,end="",flush=True) - generated_str += i - for ap in self.params.antiprompt: - if generated_str.endswith(ap): - raise KeyboardInterrupt - except KeyboardInterrupt: - self.set_color(util.CONSOLE_COLOR_DEFAULT) - if not self.params.instruct: - print(self.params.fix_prefix,end="") - self.input(self.params.fix_prefix) + # It is more reliable than tokens for interactive mode. + generated_str = "" + while self.params.interactive: + self.set_color(util.CONSOLE_COLOR_USER_INPUT) + if self.params.instruct: + print("\n> ", end="") + self.input(self.read_input()) + else: + print(self.params.input_prefix, end="") + self.input( + f"{self.params.input_prefix}{self.read_input()}{self.params.input_suffix}" + ) + print(self.params.input_suffix, end="") + self.set_color(util.CONSOLE_COLOR_DEFAULT) + + try: + for i in self.output(): + print(i, end="", flush=True) + generated_str += i + for ap in self.params.antiprompt: + if generated_str.endswith(ap): + raise KeyboardInterrupt + except KeyboardInterrupt: + self.set_color(util.CONSOLE_COLOR_DEFAULT) + if not self.params.instruct: + print(self.params.fix_prefix, end="") + self.input(self.params.fix_prefix) + if __name__ == "__main__": - from datetime import datetime + from datetime import datetime - USER_NAME="User" - AI_NAME="ChatLLaMa" + USER_NAME = "User" + AI_NAME = "ChatLLaMa" - time_now = datetime.now() - prompt = f"""Text transcript of a never ending dialog, where {USER_NAME} interacts with an AI assistant named {AI_NAME}. + time_now = datetime.now() + prompt = f"""Text transcript of a never ending dialog, where {USER_NAME} interacts with an AI assistant named {AI_NAME}. {AI_NAME} is helpful, kind, honest, friendly, good at writing and never fails to answer {USER_NAME}’s requests immediately and with details and precision. Transcript below contains only the recorded dialog between two, without any annotations like (30 seconds passed...) or (to himself), just what {USER_NAME} and {AI_NAME} say aloud to each other. The dialog lasts for years, the entirety of it is shared below. It's 10000 pages long. @@ -595,10 +752,10 @@ def interact(self): {USER_NAME}: Name a color. {AI_NAME}: Blue {USER_NAME}: """ - - params = gpt_params_parse() - if params.prompt is None and params.file is None: - params.prompt = prompt - with LLaMAInteract(params) as m: - m.interact() + params = gpt_params_parse() + if params.prompt is None and params.file is None: + params.prompt = prompt + + with LLaMAInteract(params) as m: + m.interact() diff --git a/examples/low_level_api/low_level_api_llama_cpp.py b/examples/low_level_api/low_level_api_llama_cpp.py index ef1b2c0162..ba3545771d 100644 --- a/examples/low_level_api/low_level_api_llama_cpp.py +++ b/examples/low_level_api/low_level_api_llama_cpp.py @@ -7,23 +7,20 @@ llama_cpp.llama_backend_init(numa=False) N_THREADS = multiprocessing.cpu_count() -MODEL_PATH = os.environ.get('MODEL', "../models/7B/ggml-model.bin") +MODEL_PATH = os.environ.get("MODEL", "../models/7B/ggml-model.bin") prompt = b"\n\n### Instruction:\nWhat is the capital of France?\n\n### Response:\n" lparams = llama_cpp.llama_model_default_params() cparams = llama_cpp.llama_context_default_params() -model = llama_cpp.llama_load_model_from_file(MODEL_PATH.encode('utf-8'), lparams) +model = llama_cpp.llama_load_model_from_file(MODEL_PATH.encode("utf-8"), lparams) ctx = llama_cpp.llama_new_context_with_model(model, cparams) # determine the required inference memory per token: tmp = [0, 1, 2, 3] llama_cpp.llama_eval( - ctx = ctx, - tokens=(llama_cpp.c_int * len(tmp))(*tmp), - n_tokens=len(tmp), - n_past=0 - )# Deprecated + ctx=ctx, tokens=(llama_cpp.c_int * len(tmp))(*tmp), n_tokens=len(tmp), n_past=0 +) # Deprecated n_past = 0 @@ -32,12 +29,12 @@ embd_inp = (llama_cpp.llama_token * (len(prompt) + 1))() n_of_tok = llama_cpp.llama_tokenize( model=model, - text=bytes(str(prompt),'utf-8'), - text_len=len(embd_inp), + text=bytes(str(prompt), "utf-8"), + text_len=len(embd_inp), tokens=embd_inp, n_max_tokens=len(embd_inp), add_bos=False, - special=False + special=False, ) embd_inp = embd_inp[:n_of_tok] @@ -63,11 +60,11 @@ while remaining_tokens > 0: if len(embd) > 0: llama_cpp.llama_eval( - ctx = ctx, + ctx=ctx, tokens=(llama_cpp.c_int * len(embd))(*embd), n_tokens=len(embd), - n_past=n_past - )# Deprecated + n_past=n_past, + ) # Deprecated n_past += len(embd) embd = [] @@ -75,20 +72,26 @@ logits = llama_cpp.llama_get_logits(ctx) n_vocab = llama_cpp.llama_n_vocab(model) - _arr = (llama_cpp.llama_token_data * n_vocab)(*[ - llama_cpp.llama_token_data(token_id, logits[token_id], 0.0) - for token_id in range(n_vocab) - ]) + _arr = (llama_cpp.llama_token_data * n_vocab)( + *[ + llama_cpp.llama_token_data(token_id, logits[token_id], 0.0) + for token_id in range(n_vocab) + ] + ) candidates_p = llama_cpp.ctypes.pointer( - llama_cpp.llama_token_data_array(_arr, len(_arr), False)) + llama_cpp.llama_token_data_array(_arr, len(_arr), False) + ) _arr = (llama_cpp.c_int * len(last_n_tokens_data))(*last_n_tokens_data) - llama_cpp.llama_sample_repetition_penalties(ctx, candidates_p, + llama_cpp.llama_sample_repetition_penalties( + ctx, + candidates_p, _arr, penalty_last_n=last_n_repeat, penalty_repeat=repeat_penalty, penalty_freq=frequency_penalty, - penalty_present=presence_penalty) + penalty_present=presence_penalty, + ) llama_cpp.llama_sample_top_k(ctx, candidates_p, k=40, min_keep=1) llama_cpp.llama_sample_top_p(ctx, candidates_p, p=0.8, min_keep=1) @@ -111,10 +114,11 @@ size = 32 buffer = (ctypes.c_char * size)() n = llama_cpp.llama_token_to_piece( - model, llama_cpp.llama_token(id), buffer, size) + model, llama_cpp.llama_token(id), buffer, size + ) assert n <= size print( - buffer[:n].decode('utf-8'), + buffer[:n].decode("utf-8"), end="", flush=True, ) diff --git a/examples/low_level_api/quantize.py b/examples/low_level_api/quantize.py index cdb038a712..057ac389eb 100644 --- a/examples/low_level_api/quantize.py +++ b/examples/low_level_api/quantize.py @@ -22,7 +22,10 @@ def main(args): parser = argparse.ArgumentParser() parser.add_argument("fname_inp", type=str, help="Path to input model") parser.add_argument("fname_out", type=str, help="Path to output model") - parser.add_argument("type", type=int, help="Type of quantization (2: q4_0, 3: q4_1), see llama_cpp.py for enum") + parser.add_argument( + "type", + type=int, + help="Type of quantization (2: q4_0, 3: q4_1), see llama_cpp.py for enum", + ) args = parser.parse_args() main(args) - diff --git a/examples/low_level_api/util.py b/examples/low_level_api/util.py index 9d0ec2f705..ef8b1c1eea 100644 --- a/examples/low_level_api/util.py +++ b/examples/low_level_api/util.py @@ -1,4 +1,3 @@ - ANSI_COLOR_RESET = "\x1b[0m" ANSI_COLOR_YELLOW = "\x1b[33m" ANSI_BOLD = "\x1b[1m" @@ -8,88 +7,95 @@ CONSOLE_COLOR_PROMPT = ANSI_COLOR_YELLOW CONSOLE_COLOR_USER_INPUT = ANSI_BOLD + ANSI_COLOR_GREEN + # Iterative search # Actively searches and prevents a pattern from being returned class IterSearch: - def __init__(self, pattern): - self.pattern = list(pattern) - self.buffer = [] - - def __call__(self, char): - self.buffer += [char] + def __init__(self, pattern): + self.pattern = list(pattern) + self.buffer = [] - if (self.pattern[:len(self.buffer)] == self.buffer): - if (len(self.buffer) >= len(self.pattern)): - self.buffer.clear() - return [] + def __call__(self, char): + self.buffer += [char] - _tmp = self.buffer[:] - self.buffer.clear() - return _tmp + if self.pattern[: len(self.buffer)] == self.buffer: + if len(self.buffer) >= len(self.pattern): + self.buffer.clear() + return [] -class Circle: - def __init__(self, size, default=0): - self.list = [default] * size - self.maxsize = size - self.size = 0 - self.offset = 0 - - def append(self, elem): - if self.size < self.maxsize: - self.list[self.size] = elem - self.size += 1 - else: - self.list[self.offset] = elem - self.offset = (self.offset + 1) % self.maxsize - - def __getitem__(self, val): - if isinstance(val, int): - if 0 > val or val >= self.size: - raise IndexError('Index out of range') - return self.list[val] if self.size < self.maxsize else self.list[(self.offset + val) % self.maxsize] - elif isinstance(val, slice): - start, stop, step = val.start, val.stop, val.step - if step is None: - step = 1 - if start is None: - start = 0 - if stop is None: - stop = self.size - if start < 0: - start = self.size + start - if stop < 0: - stop = self.size + stop - - indices = range(start, stop, step) - return [self.list[(self.offset + i) % self.maxsize] for i in indices if i < self.size] - else: - raise TypeError('Invalid argument type') + _tmp = self.buffer[:] + self.buffer.clear() + return _tmp +class Circle: + def __init__(self, size, default=0): + self.list = [default] * size + self.maxsize = size + self.size = 0 + self.offset = 0 + + def append(self, elem): + if self.size < self.maxsize: + self.list[self.size] = elem + self.size += 1 + else: + self.list[self.offset] = elem + self.offset = (self.offset + 1) % self.maxsize + + def __getitem__(self, val): + if isinstance(val, int): + if 0 > val or val >= self.size: + raise IndexError("Index out of range") + return ( + self.list[val] + if self.size < self.maxsize + else self.list[(self.offset + val) % self.maxsize] + ) + elif isinstance(val, slice): + start, stop, step = val.start, val.stop, val.step + if step is None: + step = 1 + if start is None: + start = 0 + if stop is None: + stop = self.size + if start < 0: + start = self.size + start + if stop < 0: + stop = self.size + stop + + indices = range(start, stop, step) + return [ + self.list[(self.offset + i) % self.maxsize] + for i in indices + if i < self.size + ] + else: + raise TypeError("Invalid argument type") if __name__ == "__main__": - c = Circle(5) - - c.append(1) - print(c.list) - print(c[:]) - assert c[0] == 1 - assert c[:5] == [1] - - for i in range(2,5+1): - c.append(i) - print(c.list) - print(c[:]) - assert c[0] == 1 - assert c[:5] == [1,2,3,4,5] - - for i in range(5+1,9+1): - c.append(i) - print(c.list) - print(c[:]) - assert c[0] == 5 - assert c[:5] == [5,6,7,8,9] - #assert c[:-5] == [5,6,7,8,9] - assert c[:10] == [5,6,7,8,9] - + c = Circle(5) + + c.append(1) + print(c.list) + print(c[:]) + assert c[0] == 1 + assert c[:5] == [1] + + for i in range(2, 5 + 1): + c.append(i) + print(c.list) + print(c[:]) + assert c[0] == 1 + assert c[:5] == [1, 2, 3, 4, 5] + + for i in range(5 + 1, 9 + 1): + c.append(i) + print(c.list) + print(c[:]) + assert c[0] == 5 + assert c[:5] == [5, 6, 7, 8, 9] + # assert c[:-5] == [5,6,7,8,9] + assert c[:10] == [5, 6, 7, 8, 9] diff --git a/examples/notebooks/Batching.ipynb b/examples/notebooks/Batching.ipynb index 687316b327..73b28c7444 100644 --- a/examples/notebooks/Batching.ipynb +++ b/examples/notebooks/Batching.ipynb @@ -392,7 +392,9 @@ "source": [ "params = llama_cpp.llama_model_default_params()\n", "params.n_gpu_layers = 35\n", - "model = llama_cpp.llama_load_model_from_file(b\"../../models/mistral-7b-v0.1-GGUF/ggml-model-Q4_K.gguf\", params=params) # Update this to whatever" + "model = llama_cpp.llama_load_model_from_file(\n", + " b\"../../models/mistral-7b-v0.1-GGUF/ggml-model-Q4_K.gguf\", params=params\n", + ") # Update this to whatever" ] }, { @@ -416,7 +418,9 @@ "prompt = b\"The quick brown fox\"\n", "\n", "tokens = (llama_cpp.llama_token * n_ctx)()\n", - "tokens_len = llama_cpp.llama_tokenize(model, prompt, len(prompt), tokens, len(tokens), True, True)\n", + "tokens_len = llama_cpp.llama_tokenize(\n", + " model, prompt, len(prompt), tokens, len(tokens), True, True\n", + ")\n", "print(tokens[:tokens_len])\n", "\n", "n_kv_req = tokens_len + (n_len - tokens_len) * n_parallel\n", @@ -447,7 +451,6 @@ } ], "source": [ - "\n", "ctx_params = llama_cpp.llama_context_default_params()\n", "ctx_params.seed = 1234\n", "ctx_params.n_ctx = n_kv_req\n", @@ -577,7 +580,7 @@ " for i in range(n_parallel):\n", " if i_batch[i] < 0:\n", " continue\n", - " \n", + "\n", " n_vocab = llama_cpp.llama_n_vocab(model)\n", " logits = llama_cpp.llama_get_logits_ith(ctx, i_batch[i])\n", "\n", @@ -588,7 +591,9 @@ " candidates[token_id].logit = logits[token_id]\n", " candidates[token_id].p = 0.0\n", "\n", - " candidates_p = llama_cpp.llama_token_data_array(candidates, len(candidates), False)\n", + " candidates_p = llama_cpp.llama_token_data_array(\n", + " candidates, len(candidates), False\n", + " )\n", "\n", " top_k = 40\n", " top_p = 0.9\n", @@ -596,8 +601,8 @@ "\n", " llama_cpp.llama_sample_top_k(ctx, ctypes.byref(candidates_p), top_k, 1)\n", " llama_cpp.llama_sample_top_p(ctx, ctypes.byref(candidates_p), top_p, 1)\n", - " llama_cpp.llama_sample_temp (ctx, ctypes.byref(candidates_p), temp)\n", - " \n", + " llama_cpp.llama_sample_temp(ctx, ctypes.byref(candidates_p), temp)\n", + "\n", " new_token_id = llama_cpp.llama_sample_token(ctx, ctypes.byref(candidates_p))\n", "\n", " if new_token_id == llama_cpp.llama_token_eos(ctx) or n_cur == n_len:\n", @@ -617,7 +622,7 @@ " i_batch[i] = batch.n_tokens\n", " batch.n_tokens += 1\n", " n_decode += 1\n", - " \n", + "\n", " if batch.n_tokens == 0:\n", " break\n", "\n", @@ -627,7 +632,7 @@ " print(\"Error decoding\", flush=True)\n", " break\n", " print(n_cur)\n", - " print(streams)\n" + " print(streams)" ] }, { diff --git a/examples/notebooks/Clients.ipynb b/examples/notebooks/Clients.ipynb index caebbb67fd..fab82673e5 100644 --- a/examples/notebooks/Clients.ipynb +++ b/examples/notebooks/Clients.ipynb @@ -37,11 +37,11 @@ "source": [ "import openai\n", "\n", - "openai.api_key = \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\" # can be anything\n", + "openai.api_key = \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\" # can be anything\n", "openai.api_base = \"http://100.64.159.73:8000/v1\"\n", "\n", "openai.Completion.create(\n", - " model=\"text-davinci-003\", # currently can be anything\n", + " model=\"text-davinci-003\", # currently can be anything\n", " prompt=\"The quick brown fox jumps\",\n", " max_tokens=5,\n", ")" @@ -66,7 +66,9 @@ "source": [ "import os\n", "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\" # can be anything\n", + "os.environ[\"OPENAI_API_KEY\"] = (\n", + " \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\" # can be anything\n", + ")\n", "os.environ[\"OPENAI_API_BASE\"] = \"http://100.64.159.73:8000/v1\"\n", "\n", "from langchain.llms import OpenAI\n", diff --git a/examples/notebooks/Functions.ipynb b/examples/notebooks/Functions.ipynb index f1e5e9a1d6..1f41381659 100644 --- a/examples/notebooks/Functions.ipynb +++ b/examples/notebooks/Functions.ipynb @@ -45,10 +45,11 @@ "\n", "\n", "client = openai.OpenAI(\n", - " api_key = \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", # can be anything\n", - " base_url = \"http://100.64.159.73:8000/v1\" # NOTE: Replace with IP address and port of your llama-cpp-python server\n", + " api_key=\"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\", # can be anything\n", + " base_url=\"http://100.64.159.73:8000/v1\", # NOTE: Replace with IP address and port of your llama-cpp-python server\n", ")\n", "\n", + "\n", "# Example dummy function hard coded to return the same weather\n", "# In production, this could be your backend API or an external API\n", "def get_current_weather(location, unit=\"fahrenheit\"):\n", @@ -56,15 +57,23 @@ " if \"tokyo\" in location.lower():\n", " return json.dumps({\"location\": \"Tokyo\", \"temperature\": \"10\", \"unit\": \"celsius\"})\n", " elif \"san francisco\" in location.lower():\n", - " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"})\n", + " return json.dumps(\n", + " {\"location\": \"San Francisco\", \"temperature\": \"72\", \"unit\": \"fahrenheit\"}\n", + " )\n", " elif \"paris\" in location.lower():\n", " return json.dumps({\"location\": \"Paris\", \"temperature\": \"22\", \"unit\": \"celsius\"})\n", " else:\n", " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", "\n", + "\n", "def run_conversation():\n", " # Step 1: send the conversation and available functions to the model\n", - " messages = [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\"}]\n", + " messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What's the weather like in San Francisco, Tokyo, and Paris?\",\n", + " }\n", + " ]\n", " tools = [\n", " {\n", " \"type\": \"function\",\n", @@ -123,6 +132,8 @@ " messages=messages,\n", " ) # get a new response from the model where it can see the function response\n", " return second_response\n", + "\n", + "\n", "print(run_conversation())" ] }, @@ -169,16 +180,18 @@ "# Enables `response_model`\n", "client = instructor.patch(client=client)\n", "\n", + "\n", "class UserDetail(BaseModel):\n", " name: str\n", " age: int\n", "\n", + "\n", "user = client.chat.completions.create(\n", " model=\"gpt-3.5-turbo\",\n", " response_model=UserDetail,\n", " messages=[\n", " {\"role\": \"user\", \"content\": \"Extract Jason is 25 years old\"},\n", - " ]\n", + " ],\n", ")\n", "\n", "assert isinstance(user, UserDetail)\n", @@ -213,17 +226,22 @@ "source": [ "import enum\n", "\n", + "\n", "class Labels(str, enum.Enum):\n", " \"\"\"Enumeration for single-label text classification.\"\"\"\n", + "\n", " SPAM = \"spam\"\n", " NOT_SPAM = \"not_spam\"\n", "\n", + "\n", "class SinglePrediction(BaseModel):\n", " \"\"\"\n", " Class for a single class label prediction.\n", " \"\"\"\n", + "\n", " class_label: Labels\n", "\n", + "\n", "def classify(data: str) -> SinglePrediction:\n", " \"\"\"Perform single-label classification on the input text.\"\"\"\n", " return client.chat.completions.create(\n", @@ -237,6 +255,7 @@ " ],\n", " ) # type: ignore\n", "\n", + "\n", "prediction = classify(\"Hello there I'm a Nigerian prince and I want to give you money\")\n", "assert prediction.class_label == Labels.SPAM\n", "print(prediction)" @@ -265,19 +284,23 @@ "source": [ "from typing import List\n", "\n", + "\n", "# Define Enum class for multiple labels\n", "class MultiLabels(str, enum.Enum):\n", " TECH_ISSUE = \"tech_issue\"\n", " BILLING = \"billing\"\n", " GENERAL_QUERY = \"general_query\"\n", "\n", + "\n", "# Define the multi-class prediction model\n", "class MultiClassPrediction(BaseModel):\n", " \"\"\"\n", " Class for a multi-class label prediction.\n", " \"\"\"\n", + "\n", " class_labels: List[MultiLabels]\n", "\n", + "\n", "def multi_classify(data: str) -> MultiClassPrediction:\n", " \"\"\"Perform multi-label classification on the input text.\"\"\"\n", " return client.chat.completions.create(\n", @@ -291,6 +314,7 @@ " ],\n", " ) # type: ignore\n", "\n", + "\n", "# Test multi-label classification\n", "ticket = \"My account is locked and I can't access my billing info.\"\n", "prediction = multi_classify(ticket)\n", @@ -331,10 +355,12 @@ "question = \"What is the meaning of life?\"\n", "context = \"The according to the devil the meaning of live is to live a life of sin and debauchery.\"\n", "\n", + "\n", "class QuestionAnswer(BaseModel):\n", " question: str\n", " answer: str\n", "\n", + "\n", "qa: QuestionAnswer = client.chat.completions.create(\n", " model=\"gpt-3.5-turbo\",\n", " response_model=QuestionAnswer,\n", @@ -351,6 +377,7 @@ ")\n", "print(qa)\n", "\n", + "\n", "class QuestionAnswerNoEvil(BaseModel):\n", " question: str\n", " answer: Annotated[\n", @@ -360,6 +387,7 @@ " ),\n", " ]\n", "\n", + "\n", "try:\n", " qa: QuestionAnswerNoEvil = client.chat.completions.create(\n", " model=\"gpt-3.5-turbo\",\n", @@ -405,6 +433,7 @@ "\n", "from pydantic import Field, BaseModel, model_validator, FieldValidationInfo\n", "\n", + "\n", "class Fact(BaseModel):\n", " fact: str = Field(...)\n", " substring_quote: List[str] = Field(...)\n", @@ -424,6 +453,7 @@ " for match in re.finditer(re.escape(quote), context):\n", " yield match.span()\n", "\n", + "\n", "class QuestionAnswer(BaseModel):\n", " question: str = Field(...)\n", " answer: List[Fact] = Field(...)\n", @@ -440,13 +470,17 @@ " temperature=0.0,\n", " response_model=QuestionAnswer,\n", " messages=[\n", - " {\"role\": \"system\", \"content\": \"You are a world class algorithm to answer questions with correct and exact citations.\"},\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a world class algorithm to answer questions with correct and exact citations.\",\n", + " },\n", " {\"role\": \"user\", \"content\": f\"{context}\"},\n", - " {\"role\": \"user\", \"content\": f\"Question: {question}\"}\n", + " {\"role\": \"user\", \"content\": f\"Question: {question}\"},\n", " ],\n", " validation_context={\"text_chunk\": context},\n", " )\n", "\n", + "\n", "question = \"What did the author do during college?\"\n", "context = \"\"\"\n", "My name is Jason Liu, and I grew up in Toronto Canada but I was born in China.\n", diff --git a/examples/notebooks/Guidance.ipynb b/examples/notebooks/Guidance.ipynb index 045856ea2f..c525598530 100644 --- a/examples/notebooks/Guidance.ipynb +++ b/examples/notebooks/Guidance.ipynb @@ -28,7 +28,9 @@ "source": [ "import os\n", "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\" # can be anything\n", + "os.environ[\"OPENAI_API_KEY\"] = (\n", + " \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\" # can be anything\n", + ")\n", "os.environ[\"OPENAI_API_BASE\"] = \"http://100.64.159.73:8000/v1\"\n", "os.environ[\"OPENAI_API_HOST\"] = \"http://100.64.159.73:8000\"\n", "\n", @@ -38,21 +40,23 @@ "guidance.llm = guidance.llms.OpenAI(\"text-davinci-003\", caching=False)\n", "\n", "# define a guidance program that adapts a proverb\n", - "program = guidance(\"\"\"Tweak this proverb to apply to model instructions instead.\n", + "program = guidance(\n", + " \"\"\"Tweak this proverb to apply to model instructions instead.\n", "\n", "{{proverb}}\n", "- {{book}} {{chapter}}:{{verse}}\n", "\n", "UPDATED\n", "Where there is no guidance{{gen 'rewrite' stop=\"\\\\n-\"}}\n", - "- GPT {{gen 'chapter'}}:{{gen 'verse'}}\"\"\")\n", + "- GPT {{gen 'chapter'}}:{{gen 'verse'}}\"\"\"\n", + ")\n", "\n", "# execute the program on a specific proverb\n", "executed_program = program(\n", " proverb=\"Where there is no guidance, a people falls,\\nbut in an abundance of counselors there is safety.\",\n", " book=\"Proverbs\",\n", " chapter=11,\n", - " verse=14\n", + " verse=14,\n", ")" ] }, diff --git a/examples/notebooks/Multimodal.ipynb b/examples/notebooks/Multimodal.ipynb index def68df008..8448ac1f7f 100644 --- a/examples/notebooks/Multimodal.ipynb +++ b/examples/notebooks/Multimodal.ipynb @@ -38,23 +38,20 @@ " \"url\": \"https://user-images.githubusercontent.com/1991296/230134379-7181e485-c521-4d23-a0d6-f7b3b61ba524.png\",\n", " },\n", " },\n", - " {\"type\": \"text\", \"text\": \"What does the image say. Format your response as a json object with a single 'text' key.\"},\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"What does the image say. Format your response as a json object with a single 'text' key.\",\n", + " },\n", " ],\n", " }\n", " ],\n", - " response_format={ \n", + " response_format={\n", " \"type\": \"json_object\",\n", - " \"schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"text\": {\n", - " \"type\": \"string\"\n", - " }\n", - " }\n", - " }\n", - " }\n", + " \"schema\": {\"type\": \"object\", \"properties\": {\"text\": {\"type\": \"string\"}}},\n", + " },\n", ")\n", "import json\n", + "\n", "print(json.loads(response.choices[0].message.content))" ] }, diff --git a/examples/notebooks/OpenHermesFunctionCalling.ipynb b/examples/notebooks/OpenHermesFunctionCalling.ipynb index c0de3fdc22..13128be044 100644 --- a/examples/notebooks/OpenHermesFunctionCalling.ipynb +++ b/examples/notebooks/OpenHermesFunctionCalling.ipynb @@ -42,42 +42,58 @@ "import inspect\n", "from typing import get_type_hints\n", "\n", + "\n", "class Article:\n", " pass\n", "\n", + "\n", "class Weather:\n", " pass\n", "\n", + "\n", "class Directions:\n", " pass\n", "\n", - "def calculate_mortgage_payment(loan_amount: int, interest_rate: float, loan_term: int) -> float:\n", + "\n", + "def calculate_mortgage_payment(\n", + " loan_amount: int, interest_rate: float, loan_term: int\n", + ") -> float:\n", " \"\"\"Get the monthly mortgage payment given an interest rate percentage.\"\"\"\n", - " \n", + "\n", " # TODO: you must implement this to actually call it later\n", " pass\n", "\n", - "def get_article_details(title: str, authors: list[str], short_summary: str, date_published: str, tags: list[str]) -> Article:\n", + "\n", + "def get_article_details(\n", + " title: str,\n", + " authors: list[str],\n", + " short_summary: str,\n", + " date_published: str,\n", + " tags: list[str],\n", + ") -> Article:\n", " '''Get article details from unstructured article text.\n", - "date_published: formatted as \"MM/DD/YYYY\"'''\n", - " \n", + " date_published: formatted as \"MM/DD/YYYY\"'''\n", + "\n", " # TODO: you must implement this to actually call it later\n", " pass\n", "\n", + "\n", "def get_weather(zip_code: str) -> Weather:\n", " \"\"\"Get the current weather given a zip code.\"\"\"\n", - " \n", + "\n", " # TODO: you must implement this to actually call it later\n", " pass\n", "\n", + "\n", "def get_directions(start: str, destination: str) -> Directions:\n", " \"\"\"Get directions from Google Directions API.\n", - "start: start address as a string including zipcode (if any)\n", - "destination: end address as a string including zipcode (if any)\"\"\"\n", - " \n", + " start: start address as a string including zipcode (if any)\n", + " destination: end address as a string including zipcode (if any)\"\"\"\n", + "\n", " # TODO: you must implement this to actually call it later\n", " pass\n", "\n", + "\n", "def get_type_name(t):\n", " name = str(t)\n", " if \"list\" in name or \"dict\" in name:\n", @@ -85,6 +101,7 @@ " else:\n", " return t.__name__\n", "\n", + "\n", "def serialize_function_to_json(func):\n", " signature = inspect.signature(func)\n", " type_hints = get_type_hints(func)\n", @@ -92,11 +109,8 @@ " function_info = {\n", " \"name\": func.__name__,\n", " \"description\": func.__doc__,\n", - " \"parameters\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {}\n", - " },\n", - " \"returns\": type_hints.get('return', 'void').__name__\n", + " \"parameters\": {\"type\": \"object\", \"properties\": {}},\n", + " \"returns\": type_hints.get(\"return\", \"void\").__name__,\n", " }\n", "\n", " for name, _ in signature.parameters.items():\n", @@ -105,6 +119,7 @@ "\n", " return json.dumps(function_info, indent=2)\n", "\n", + "\n", "print(serialize_function_to_json(get_article_details))" ] }, @@ -117,13 +132,14 @@ "import xml.etree.ElementTree as ET\n", "import re\n", "\n", + "\n", "def extract_function_calls(completion):\n", " completion = completion.strip()\n", " pattern = r\"((.*?))\"\n", " match = re.search(pattern, completion, re.DOTALL)\n", " if not match:\n", " return None\n", - " \n", + "\n", " multiplefn = match.group(1)\n", " root = ET.fromstring(multiplefn)\n", " functions = root.findall(\"functioncall\")\n", @@ -399,7 +415,7 @@ "prompts = [\n", " \"What's the weather in 10001?\",\n", " \"Determine the monthly mortgage payment for a loan amount of $200,000, an interest rate of 4%, and a loan term of 30 years.\",\n", - " \"What's the current exchange rate for USD to EUR?\"\n", + " \"What's the current exchange rate for USD to EUR?\",\n", "]\n", "functions = [get_weather, calculate_mortgage_payment, get_article_details]\n", "\n", @@ -793,7 +809,12 @@ "source": [ "import llama_cpp\n", "\n", - "llama = llama_cpp.Llama(model_path=\"../../models/OpenHermes-2.5-Mistral-7B-GGUF/openhermes-2.5-mistral-7b.Q4_K_M.gguf\", n_gpu_layers=-1, n_ctx=2048, verbose=False)" + "llama = llama_cpp.Llama(\n", + " model_path=\"../../models/OpenHermes-2.5-Mistral-7B-GGUF/openhermes-2.5-mistral-7b.Q4_K_M.gguf\",\n", + " n_gpu_layers=-1,\n", + " n_ctx=2048,\n", + " verbose=False,\n", + ")" ] }, { @@ -818,7 +839,7 @@ "prompts = [\n", " \"What's the weather in 10001?\",\n", " \"Determine the monthly mortgage payment for a loan amount of $200,000, an interest rate of 4%, and a loan term of 30 years.\",\n", - " \"What's the current exchange rate for USD to EUR?\"\n", + " \"What's the current exchange rate for USD to EUR?\",\n", "]\n", "functions = [get_weather, calculate_mortgage_payment, get_article_details]\n", "\n", @@ -830,7 +851,7 @@ " print(function_calls)\n", " else:\n", " print(completion.strip())\n", - " print(\"=\"*100)" + " print(\"=\" * 100)" ] }, { @@ -861,11 +882,13 @@ "prompts = [\n", " \"What's the weather in 05751?\",\n", " \"I'm planning a trip to Killington, Vermont (05751) from Hoboken, NJ (07030). Can you get me weather for both locations and directions?\",\n", - " \"What's the current exchange rate for USD to EUR?\"\n", + " \"What's the current exchange rate for USD to EUR?\",\n", "]\n", "\n", "for prompt in prompts:\n", - " completion = llama.create_completion(generate_hermes_prompt(prompt, functions), max_tokens=-1)[\"choices\"][0][\"text\"]\n", + " completion = llama.create_completion(\n", + " generate_hermes_prompt(prompt, functions), max_tokens=-1\n", + " )[\"choices\"][0][\"text\"]\n", " function_calls = extract_function_calls(completion)\n", "\n", " if function_calls:\n", @@ -875,7 +898,7 @@ " else:\n", " print(completion.strip())\n", "\n", - " print(\"=\"*100)" + " print(\"=\" * 100)" ] }, { diff --git a/examples/notebooks/PerformanceTuning.ipynb b/examples/notebooks/PerformanceTuning.ipynb index 76e26fbd10..ba74e4a41f 100644 --- a/examples/notebooks/PerformanceTuning.ipynb +++ b/examples/notebooks/PerformanceTuning.ipynb @@ -13,6 +13,7 @@ "import llama_cpp\n", "\n", "import numpy as np\n", + "\n", "np.int = int\n", "\n", "from skopt.space import Integer, Categorical\n", @@ -25,7 +26,7 @@ " Categorical([True, False], name=\"f16_kv\"),\n", " Categorical([True, False], name=\"use_mlock\"),\n", " Integer(1, multiprocessing.cpu_count(), name=\"n_threads\"),\n", - " Integer(1, 2048, name=\"n_batch\")\n", + " Integer(1, 2048, name=\"n_batch\"),\n", "]\n", "\n", "# TODO: Make this a random prompt to avoid any cache related inconsistencies\n", @@ -41,18 +42,25 @@ "\n", "from skopt.utils import use_named_args\n", "\n", + "\n", "@use_named_args(space)\n", "def objective(**params):\n", " f16_kv = params[\"f16_kv\"]\n", " use_mlock = params[\"use_mlock\"]\n", " n_threads = params[\"n_threads\"]\n", " n_batch = params[\"n_batch\"]\n", - " llm = llama_cpp.Llama(model_path=MODEL_PATH, f16_kv=f16_kv, use_mlock=use_mlock, n_threads=n_threads, n_batch=n_batch)\n", + " llm = llama_cpp.Llama(\n", + " model_path=MODEL_PATH,\n", + " f16_kv=f16_kv,\n", + " use_mlock=use_mlock,\n", + " n_threads=n_threads,\n", + " n_batch=n_batch,\n", + " )\n", "\n", " t1 = time.time()\n", " output = llm(\n", " PROMPT,\n", - " max_tokens=1, # Only optimize prompt processing\n", + " max_tokens=1, # Only optimize prompt processing\n", " stop=[\"###\", \"\\n\"],\n", " echo=True,\n", " )\n", @@ -5240,10 +5248,7 @@ "source": [ "from skopt import gp_minimize\n", "\n", - "res = gp_minimize(\n", - " objective,\n", - " space\n", - ")" + "res = gp_minimize(objective, space)" ] }, { diff --git a/examples/ray/llm.py b/examples/ray/llm.py index 27b15d21d4..2325dd303b 100755 --- a/examples/ray/llm.py +++ b/examples/ray/llm.py @@ -4,6 +4,7 @@ from ray.serve import Application from llama_cpp import Llama + @serve.deployment class LlamaDeployment: def __init__(self, model_path: str): diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 125d72ded3..ff378739f6 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -117,9 +117,11 @@ def apply_lora_from_file( self.model, lora_path.encode("utf-8"), scale, - path_base_model.encode("utf-8") - if path_base_model is not None - else ctypes.c_char_p(0), + ( + path_base_model.encode("utf-8") + if path_base_model is not None + else ctypes.c_char_p(0) + ), n_threads, ) @@ -225,7 +227,9 @@ def detokenize(self, tokens: List[int], special: bool = False) -> bytes: # NOTE: Llama1 models automatically added a space at the start of the prompt # this line removes a leading space if the first token is a beginning of sentence token return ( - output[1:] if len(tokens) > 0 and tokens[0] == self.token_bos() and output[0:1] == b' ' else output + output[1:] + if len(tokens) > 0 and tokens[0] == self.token_bos() and output[0:1] == b" " + else output ) # Extra @@ -235,20 +239,28 @@ def metadata(self) -> Dict[str, str]: buffer_size = 1024 buffer = ctypes.create_string_buffer(buffer_size) # zero the buffer - buffer.value = b'\0' * buffer_size + buffer.value = b"\0" * buffer_size # iterate over model keys for i in range(llama_cpp.llama_model_meta_count(self.model)): - nbytes = llama_cpp.llama_model_meta_key_by_index(self.model, i, buffer, buffer_size) + nbytes = llama_cpp.llama_model_meta_key_by_index( + self.model, i, buffer, buffer_size + ) if nbytes > buffer_size: buffer_size = nbytes + 1 buffer = ctypes.create_string_buffer(buffer_size) - nbytes = llama_cpp.llama_model_meta_key_by_index(self.model, i, buffer, buffer_size) + nbytes = llama_cpp.llama_model_meta_key_by_index( + self.model, i, buffer, buffer_size + ) key = buffer.value.decode("utf-8") - nbytes = llama_cpp.llama_model_meta_val_str_by_index(self.model, i, buffer, buffer_size) + nbytes = llama_cpp.llama_model_meta_val_str_by_index( + self.model, i, buffer, buffer_size + ) if nbytes > buffer_size: buffer_size = nbytes + 1 buffer = ctypes.create_string_buffer(buffer_size) - nbytes = llama_cpp.llama_model_meta_val_str_by_index(self.model, i, buffer, buffer_size) + nbytes = llama_cpp.llama_model_meta_val_str_by_index( + self.model, i, buffer, buffer_size + ) value = buffer.value.decode("utf-8") metadata[key] = value return metadata @@ -465,7 +477,11 @@ def sample_token_mirostat( ) def sample_token_mirostat_v2( - self, candidates: "_LlamaTokenDataArray", tau: float, eta: float, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float] + self, + candidates: "_LlamaTokenDataArray", + tau: float, + eta: float, + mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: assert self.ctx is not None return llama_cpp.llama_sample_token_mirostat_v2( @@ -589,7 +605,7 @@ def __init__(self, *, n_vocab: int): size=self.n_vocab, sorted=False, ) - self.default_candidates_data_id = np.arange(self.n_vocab, dtype=np.intc) # type: ignore + self.default_candidates_data_id = np.arange(self.n_vocab, dtype=np.intc) # type: ignore self.default_candidates_data_p = np.zeros(self.n_vocab, dtype=np.single) def copy_logits(self, logits: npt.NDArray[np.single]): @@ -635,10 +651,14 @@ def _tokenize(model: _LlamaModel, text: str, add_bos: bool, special: bool) -> li def _token_to_piece(model: _LlamaModel, token: int, special: bool = False) -> str: assert model.model is not None result = (ctypes.c_char * 8)(0) - n_tokens = llama_cpp.llama_token_to_piece(model.model, token, result, 0, len(result), special) + n_tokens = llama_cpp.llama_token_to_piece( + model.model, token, result, 0, len(result), special + ) if n_tokens < 0: result = (ctypes.c_char * -n_tokens)(0) - check = llama_cpp.llama_token_to_piece(model.model, token, result, 0, len(result), special) + check = llama_cpp.llama_token_to_piece( + model.model, token, result, 0, len(result), special + ) if check != -n_tokens: raise RuntimeError(f"Failed to get piece: token={token}") else: @@ -750,7 +770,10 @@ def prev_str(self, ctx_main: _LlamaContext, n: int) -> str: return ctx_main.model.detokenize(self.prev[-n:]).decode("utf-8") def sample( - self, ctx_main: _LlamaContext, idx: int = 0, logits_array: Optional[npt.NDArray[np.single]] = None + self, + ctx_main: _LlamaContext, + idx: int = 0, + logits_array: Optional[npt.NDArray[np.single]] = None, ): n_vocab = ctx_main.model.n_vocab() id: int = 0 @@ -775,7 +798,7 @@ def sample( if len(self.prev) > 0: nl_token = ctx_main.model.token_nl() nl_logit = logits_array[nl_token] - last_tokens = self.prev[-self.params.penalty_last_n:] + last_tokens = self.prev[-self.params.penalty_last_n :] last_tokens_size = min(len(last_tokens), self.params.penalty_last_n) if last_tokens_size > 0: last_tokens_p = (llama_cpp.llama_token * len(last_tokens))(*last_tokens) diff --git a/llama_cpp/_utils.py b/llama_cpp/_utils.py index 781b265010..29628193bb 100644 --- a/llama_cpp/_utils.py +++ b/llama_cpp/_utils.py @@ -10,6 +10,7 @@ STDOUT_FILENO = 1 STDERR_FILENO = 2 + class suppress_stdout_stderr(object): # NOTE: these must be "saved" here to avoid exceptions when using # this context manager inside of a __del__ method diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 5f5396683c..3669efdec6 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -86,7 +86,9 @@ def __init__( n_batch: int = 512, n_threads: Optional[int] = None, n_threads_batch: Optional[int] = None, - rope_scaling_type: Optional[int] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, + rope_scaling_type: Optional[ + int + ] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED, rope_freq_base: float = 0.0, rope_freq_scale: float = 0.0, @@ -229,7 +231,7 @@ def __init__( self.model_params.split_mode = split_mode self.model_params.main_gpu = main_gpu if rpc_servers is not None: - self.model_params.rpc_servers = rpc_servers.encode('utf-8') + self.model_params.rpc_servers = rpc_servers.encode("utf-8") self._rpc_servers = rpc_servers else: self._rpc_servers = None @@ -262,22 +264,34 @@ def __init__( for i, (k, v) in enumerate(kv_overrides.items()): self._kv_overrides_array[i].key = k.encode("utf-8") if isinstance(v, bool): - self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL + self._kv_overrides_array[i].tag = ( + llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL + ) self._kv_overrides_array[i].value.val_bool = v elif isinstance(v, int): - self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT + self._kv_overrides_array[i].tag = ( + llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT + ) self._kv_overrides_array[i].value.val_i64 = v elif isinstance(v, float): - self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT + self._kv_overrides_array[i].tag = ( + llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT + ) self._kv_overrides_array[i].value.val_f64 = v - elif isinstance(v, str): # type: ignore + elif isinstance(v, str): # type: ignore v_bytes = v.encode("utf-8") - if len(v_bytes) > 128: # TODO: Make this a constant + if len(v_bytes) > 128: # TODO: Make this a constant raise ValueError(f"Value for {k} is too long: {v}") v_bytes = v_bytes.ljust(128, b"\0") - self._kv_overrides_array[i].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR + self._kv_overrides_array[i].tag = ( + llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR + ) # copy min(v_bytes, 128) to str_value - address = typing.cast(int, ctypes.addressof(self._kv_overrides_array[i].value) + llama_cpp.llama_model_kv_override_value.val_str.offset) + address = typing.cast( + int, + ctypes.addressof(self._kv_overrides_array[i].value) + + llama_cpp.llama_model_kv_override_value.val_str.offset, + ) buffer_start = ctypes.cast(address, ctypes.POINTER(ctypes.c_char)) ctypes.memmove( buffer_start, @@ -331,7 +345,7 @@ def __init__( self.context_params.logits_all = ( logits_all if draft_model is None else True ) # Must be set to True for speculative decoding - self.context_params.embeddings = embedding # TODO: Rename to embeddings + self.context_params.embeddings = embedding # TODO: Rename to embeddings self.context_params.offload_kqv = offload_kqv self.context_params.flash_attn = flash_attn # KV cache quantization @@ -355,9 +369,15 @@ def __init__( self._stack = contextlib.ExitStack() - self._model = self._stack.enter_context(contextlib.closing(_LlamaModel( - path_model=self.model_path, params=self.model_params, verbose=self.verbose - ))) + self._model = self._stack.enter_context( + contextlib.closing( + _LlamaModel( + path_model=self.model_path, + params=self.model_params, + verbose=self.verbose, + ) + ) + ) # Override tokenizer self.tokenizer_ = tokenizer or LlamaTokenizer(self) @@ -369,18 +389,26 @@ def __init__( self.context_params.n_ctx = self._model.n_ctx_train() self.context_params.n_batch = self.n_batch - self._ctx = self._stack.enter_context(contextlib.closing(_LlamaContext( - model=self._model, - params=self.context_params, - verbose=self.verbose, - ))) + self._ctx = self._stack.enter_context( + contextlib.closing( + _LlamaContext( + model=self._model, + params=self.context_params, + verbose=self.verbose, + ) + ) + ) - self._batch = self._stack.enter_context(contextlib.closing(_LlamaBatch( - n_tokens=self.n_batch, - embd=0, - n_seq_max=self.context_params.n_ctx, - verbose=self.verbose, - ))) + self._batch = self._stack.enter_context( + contextlib.closing( + _LlamaBatch( + n_tokens=self.n_batch, + embd=0, + n_seq_max=self.context_params.n_ctx, + verbose=self.verbose, + ) + ) + ) if self.lora_path: if self._model.apply_lora_from_file( @@ -398,7 +426,9 @@ def __init__( self.chat_format = chat_format self.chat_handler = chat_handler - self._chat_handlers: Dict[str, llama_chat_format.LlamaChatCompletionHandler] = {} + self._chat_handlers: Dict[str, llama_chat_format.LlamaChatCompletionHandler] = ( + {} + ) self.draft_model = draft_model @@ -433,17 +463,30 @@ def __init__( eos_token_id = self.token_eos() bos_token_id = self.token_bos() - eos_token = self._model.token_get_text(eos_token_id) if eos_token_id != -1 else "" - bos_token = self._model.token_get_text(bos_token_id) if bos_token_id != -1 else "" + eos_token = ( + self._model.token_get_text(eos_token_id) if eos_token_id != -1 else "" + ) + bos_token = ( + self._model.token_get_text(bos_token_id) if bos_token_id != -1 else "" + ) # Unfortunately the llama.cpp API does not return metadata arrays, so we can't get template names from tokenizer.chat_templates - template_choices = dict((name[10:], template) for name, template in self.metadata.items() if name.startswith("tokenizer.chat_template.")) + template_choices = dict( + (name[10:], template) + for name, template in self.metadata.items() + if name.startswith("tokenizer.chat_template.") + ) if "tokenizer.chat_template" in self.metadata: - template_choices["chat_template.default"] = self.metadata["tokenizer.chat_template"] + template_choices["chat_template.default"] = self.metadata[ + "tokenizer.chat_template" + ] if self.verbose and template_choices: - print(f"Available chat formats from metadata: {', '.join(template_choices.keys())}", file=sys.stderr) + print( + f"Available chat formats from metadata: {', '.join(template_choices.keys())}", + file=sys.stderr, + ) for name, template in template_choices.items(): self._chat_handlers[name] = llama_chat_format.Jinja2ChatFormatter( @@ -468,7 +511,10 @@ def __init__( print(f"Guessed chat format: {chat_format}", file=sys.stderr) else: if self.verbose: - print(f"Using gguf chat template: {template_choices['chat_template.default']}", file=sys.stderr) + print( + f"Using gguf chat template: {template_choices['chat_template.default']}", + file=sys.stderr, + ) print(f"Using chat eos_token: {eos_token}", file=sys.stderr) print(f"Using chat bos_token: {bos_token}", file=sys.stderr) @@ -477,7 +523,9 @@ def __init__( if self.chat_format is None and self.chat_handler is None: self.chat_format = "llama-2" if self.verbose: - print(f"Using fallback chat format: {self.chat_format}", file=sys.stderr) + print( + f"Using fallback chat format: {self.chat_format}", file=sys.stderr + ) @property def ctx(self) -> llama_cpp.llama_context_p: @@ -582,13 +630,17 @@ def eval(self, tokens: Sequence[int]): if self.context_params.logits_all: rows = n_tokens cols = self._n_vocab - logits = np.ctypeslib.as_array(self._ctx.get_logits(), shape=(rows * cols, )) - self.scores[n_past : n_past + n_tokens, :].reshape(-1)[: :] = logits + logits = np.ctypeslib.as_array( + self._ctx.get_logits(), shape=(rows * cols,) + ) + self.scores[n_past : n_past + n_tokens, :].reshape(-1)[::] = logits else: rows = 1 cols = self._n_vocab - logits = np.ctypeslib.as_array(self._ctx.get_logits(), shape=(rows * cols, )) - self.scores[n_past + n_tokens - 1, :].reshape(-1)[: :] = logits + logits = np.ctypeslib.as_array( + self._ctx.get_logits(), shape=(rows * cols,) + ) + self.scores[n_past + n_tokens - 1, :].reshape(-1)[::] = logits # Update n_tokens self.n_tokens += n_tokens @@ -880,7 +932,8 @@ def decode_batch(seq_sizes: List[int]): for i, size in enumerate(seq_sizes): ptr = llama_cpp.llama_get_embeddings(self._ctx.ctx) embedding: List[List[float]] = [ - ptr[pos + j * n_embd : pos + (j + 1) * n_embd] for j in range(size) + ptr[pos + j * n_embd : pos + (j + 1) * n_embd] + for j in range(size) ] if normalize: embedding = [_normalize_embedding(e) for e in embedding] @@ -987,14 +1040,24 @@ def _create_completion( prefix_token_id: int = self._model.token_prefix() middle_token_id: int = self._model.token_middle() suffix_token_id: int = self._model.token_suffix() - add_space_prefix: bool = self.metadata.get("tokenizer.ggml.add_space_prefix", "true") == "true" + add_space_prefix: bool = ( + self.metadata.get("tokenizer.ggml.add_space_prefix", "true") == "true" + ) bos_tokens: List[int] = [cls_token_id if cls_token_id != -1 else bos_token_id] - eos_tokens: List[int] = [sep_token_id if sep_token_id != -1 else self.token_eos()] + eos_tokens: List[int] = [ + sep_token_id if sep_token_id != -1 else self.token_eos() + ] - if (isinstance(prompt, list) and suffix is None) or self._model.add_bos_token() == 0 or bos_tokens[:1] == [-1]: + if ( + (isinstance(prompt, list) and suffix is None) + or self._model.add_bos_token() == 0 + or bos_tokens[:1] == [-1] + ): bos_tokens = [] - if (isinstance(prompt, list) and suffix is None) or (self._model.add_eos_token() != 1 and sep_token_id == -1): + if (isinstance(prompt, list) and suffix is None) or ( + self._model.add_eos_token() != 1 and sep_token_id == -1 + ): eos_tokens = [] suffix_space_prefix: int = 0 @@ -1008,28 +1071,27 @@ def _create_completion( completion_tokens: List[int] = [] if len(prompt) > 0 else [bos_token_id] # Add blank space to start of prompt to match OG llama tokenizer prefix_tokens: List[int] = ( + [prefix_token_id] if prefix_token_id >= 0 and suffix is not None else [] + ) + ( ( - [prefix_token_id] - if prefix_token_id >= 0 and suffix is not None - else [] - ) - + - ( - ( - self.tokenize(prompt.encode("utf-8"), add_bos=False, special=(prefix_token_id < 0 or suffix is None)) - if prompt != "" - else [] + self.tokenize( + prompt.encode("utf-8"), + add_bos=False, + special=(prefix_token_id < 0 or suffix is None), ) - if isinstance(prompt, str) - else prompt + if prompt != "" + else [] ) + if isinstance(prompt, str) + else prompt ) suffix_tokens: List[int] = ( ( [suffix_token_id] - + - ( - self.tokenize(suffix.encode("utf-8"), add_bos=False, special=False)[suffix_space_prefix:] + + ( + self.tokenize(suffix.encode("utf-8"), add_bos=False, special=False)[ + suffix_space_prefix: + ] if suffix else [] ) @@ -1038,11 +1100,17 @@ def _create_completion( else [] ) middle_tokens: List[int] = ( - [middle_token_id] - if middle_token_id >= 0 and suffix is not None - else [] + [middle_token_id] if middle_token_id >= 0 and suffix is not None else [] + ) + prompt_tokens: List[int] = ( + bos_tokens + + ( + (suffix_tokens + prefix_tokens + middle_tokens) + if self.spm_infill + else (prefix_tokens + suffix_tokens + middle_tokens) + ) + + eos_tokens ) - prompt_tokens: List[int] = bos_tokens + ((suffix_tokens + prefix_tokens + middle_tokens) if self.spm_infill else (prefix_tokens + suffix_tokens + middle_tokens)) + eos_tokens text: bytes = b"" returned_tokens: int = 0 stop = ( @@ -1179,7 +1247,10 @@ def logit_bias_processor( if stream: remaining_tokens = completion_tokens[returned_tokens:] - remaining_text = self.detokenize(remaining_tokens, prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]) + remaining_text = self.detokenize( + remaining_tokens, + prev_tokens=prompt_tokens + completion_tokens[:returned_tokens], + ) remaining_length = len(remaining_text) # We want to avoid yielding any characters from @@ -1201,19 +1272,29 @@ def logit_bias_processor( for token in remaining_tokens: if token == bos_token_id: continue - token_end_position += len(self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens])) + token_end_position += len( + self.detokenize( + [token], + prev_tokens=prompt_tokens + + completion_tokens[:returned_tokens], + ) + ) # Check if stop sequence is in the token if token_end_position > ( remaining_length - first_stop_position ): break - token_str = self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]).decode( - "utf-8", errors="ignore" - ) + token_str = self.detokenize( + [token], + prev_tokens=prompt_tokens + + completion_tokens[:returned_tokens], + ).decode("utf-8", errors="ignore") text_offset = len(prompt) + len( - self.detokenize(completion_tokens[:returned_tokens], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]).decode( - "utf-8", errors="ignore" - ) + self.detokenize( + completion_tokens[:returned_tokens], + prev_tokens=prompt_tokens + + completion_tokens[:returned_tokens], + ).decode("utf-8", errors="ignore") ) token_offset = len(prompt_tokens) + returned_tokens logits = self._scores[token_offset - 1, :] @@ -1233,9 +1314,11 @@ def logit_bias_processor( top_logprob.update({token_str: current_logprobs[int(token)]}) logprobs_or_none = { "tokens": [ - self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]).decode( - "utf-8", errors="ignore" - ) + self.detokenize( + [token], + prev_tokens=prompt_tokens + + completion_tokens[:returned_tokens], + ).decode("utf-8", errors="ignore") ], "text_offset": [text_offset], "token_logprobs": [current_logprobs[int(token)]], @@ -1249,9 +1332,11 @@ def logit_bias_processor( "model": model_name, "choices": [ { - "text": self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]).decode( - "utf-8", errors="ignore" - ), + "text": self.detokenize( + [token], + prev_tokens=prompt_tokens + + completion_tokens[:returned_tokens], + ).decode("utf-8", errors="ignore"), "index": 0, "logprobs": logprobs_or_none, "finish_reason": None, @@ -1263,7 +1348,11 @@ def logit_bias_processor( decode_success = False for i in range(1, len(remaining_tokens) + 1): try: - bs = self.detokenize(remaining_tokens[:i], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]) + bs = self.detokenize( + remaining_tokens[:i], + prev_tokens=prompt_tokens + + completion_tokens[:returned_tokens], + ) ts = bs.decode("utf-8") decode_success = True break @@ -1313,7 +1402,10 @@ def logit_bias_processor( if stream: remaining_tokens = completion_tokens[returned_tokens:] - all_text = self.detokenize(remaining_tokens, prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]) + all_text = self.detokenize( + remaining_tokens, + prev_tokens=prompt_tokens + completion_tokens[:returned_tokens], + ) any_stop = [s for s in stop_sequences if s in all_text] if len(any_stop) > 0: end = min(all_text.index(stop) for stop in any_stop) @@ -1322,7 +1414,12 @@ def logit_bias_processor( token_end_position = 0 for token in remaining_tokens: - token_end_position += len(self.detokenize([token], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens])) + token_end_position += len( + self.detokenize( + [token], + prev_tokens=prompt_tokens + completion_tokens[:returned_tokens], + ) + ) logprobs_or_none: Optional[CompletionLogprobs] = None if logprobs is not None: @@ -1332,7 +1429,11 @@ def logit_bias_processor( "utf-8", errors="ignore" ) text_offset = len(prompt) + len( - self.detokenize(completion_tokens[:returned_tokens], prev_tokens=prompt_tokens + completion_tokens[:returned_tokens]) + self.detokenize( + completion_tokens[:returned_tokens], + prev_tokens=prompt_tokens + + completion_tokens[:returned_tokens], + ) ) token_offset = len(prompt_tokens) + returned_tokens - 1 logits = self._scores[token_offset, :] @@ -1441,12 +1542,17 @@ def logit_bias_processor( if echo: # Remove leading BOS token if exists - all_tokens = prompt_tokens[1 if prompt_tokens[0] == self.token_bos() else 0:] + completion_tokens + all_tokens = ( + prompt_tokens[1 if prompt_tokens[0] == self.token_bos() else 0 :] + + completion_tokens + ) else: all_tokens = completion_tokens all_token_strs = [ - self.detokenize([token], prev_tokens=all_tokens[:i]).decode("utf-8", errors="ignore") + self.detokenize([token], prev_tokens=all_tokens[:i]).decode( + "utf-8", errors="ignore" + ) for i, token in enumerate(all_tokens) ] all_logprobs = Llama.logits_to_logprobs(self._scores)[token_offset:] @@ -1472,7 +1578,9 @@ def logit_bias_processor( ) token_logprobs.append(logprobs_token[int(token)]) top_logprob: Optional[Dict[str, float]] = { - self.detokenize([i], prev_tokens=all_tokens[:idx]).decode("utf-8", errors="ignore"): logprob + self.detokenize([i], prev_tokens=all_tokens[:idx]).decode( + "utf-8", errors="ignore" + ): logprob for logprob, i in sorted_logprobs[:logprobs] } top_logprob.update({token_str: logprobs_token[int(token)]}) @@ -1765,8 +1873,10 @@ def create_chat_completion( Returns: Generated chat completion or a stream of chat completion chunks. """ - handler = self.chat_handler or self._chat_handlers.get(self.chat_format) or llama_chat_format.get_chat_completion_handler( - self.chat_format + handler = ( + self.chat_handler + or self._chat_handlers.get(self.chat_format) + or llama_chat_format.get_chat_completion_handler(self.chat_format) ) return handler( llama=self, diff --git a/llama_cpp/llama_cache.py b/llama_cpp/llama_cache.py index 9e9870a522..5220c79337 100644 --- a/llama_cpp/llama_cache.py +++ b/llama_cpp/llama_cache.py @@ -40,7 +40,9 @@ def __contains__(self, key: Sequence[int]) -> bool: raise NotImplementedError @abstractmethod - def __setitem__(self, key: Sequence[int], value: "llama_cpp.llama.LlamaState") -> None: + def __setitem__( + self, key: Sequence[int], value: "llama_cpp.llama.LlamaState" + ) -> None: raise NotImplementedError @@ -50,7 +52,9 @@ class LlamaRAMCache(BaseLlamaCache): def __init__(self, capacity_bytes: int = (2 << 30)): super().__init__(capacity_bytes) self.capacity_bytes = capacity_bytes - self.cache_state: OrderedDict[Tuple[int, ...], "llama_cpp.llama.LlamaState"] = OrderedDict() + self.cache_state: OrderedDict[Tuple[int, ...], "llama_cpp.llama.LlamaState"] = ( + OrderedDict() + ) @property def cache_size(self): @@ -63,7 +67,8 @@ def _find_longest_prefix_key( min_len = 0 min_key = None keys = ( - (k, llama_cpp.llama.Llama.longest_token_prefix(k, key)) for k in self.cache_state.keys() + (k, llama_cpp.llama.Llama.longest_token_prefix(k, key)) + for k in self.cache_state.keys() ) for k, prefix_len in keys: if prefix_len > min_len: diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 975f74762d..78d83ba866 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -8,7 +8,18 @@ import string from contextlib import ExitStack -from typing import Any, Dict, Iterator, List, Literal, Optional, Tuple, Union, Protocol, cast +from typing import ( + Any, + Dict, + Iterator, + List, + Literal, + Optional, + Tuple, + Union, + Protocol, + cast, +) import jinja2 from jinja2.sandbox import ImmutableSandboxedEnvironment @@ -191,7 +202,9 @@ def __init__( self.eos_token = eos_token self.bos_token = bos_token self.add_generation_prompt = add_generation_prompt - self.stop_token_ids = set(stop_token_ids) if stop_token_ids is not None else None + self.stop_token_ids = ( + set(stop_token_ids) if stop_token_ids is not None else None + ) self._environment = ImmutableSandboxedEnvironment( loader=jinja2.BaseLoader(), @@ -226,14 +239,20 @@ def raise_exception(message: str): stopping_criteria = None if self.stop_token_ids is not None: + def stop_on_last_token( - tokens: npt.NDArray[np.intc], - logits: npt.NDArray[np.single] + tokens: npt.NDArray[np.intc], logits: npt.NDArray[np.single] ) -> bool: return tokens[-1] in self.stop_token_ids + stopping_criteria = llama.StoppingCriteriaList([stop_on_last_token]) - return ChatFormatterResponse(prompt=prompt, stop=[self.eos_token], stopping_criteria=stopping_criteria, added_special=True) + return ChatFormatterResponse( + prompt=prompt, + stop=[self.eos_token], + stopping_criteria=stopping_criteria, + added_special=True, + ) def to_chat_handler(self) -> LlamaChatCompletionHandler: return chat_formatter_to_chat_completion_handler(self) @@ -430,7 +449,9 @@ def _stream_response_to_function_stream( "type": "function", "function": { "name": tool_name, - "arguments": chunk["choices"][0]["text"], + "arguments": chunk["choices"][0][ + "text" + ], }, } ], @@ -465,9 +486,7 @@ def _stream_response_to_function_stream( "type": "function", "function": { "name": tool_name, - "arguments": chunk["choices"][0][ - "text" - ], + "arguments": chunk["choices"][0]["text"], }, } ], @@ -500,7 +519,6 @@ def _stream_response_to_function_stream( return _stream_response_to_function_stream(chunks) - def chat_formatter_to_chat_completion_handler( chat_formatter: ChatFormatter, ) -> LlamaChatCompletionHandler: @@ -549,7 +567,11 @@ def chat_completion_handler( tools=tools, tool_choice=tool_choice, ) - prompt = llama.tokenize(result.prompt.encode("utf-8"), add_bos=not result.added_special, special=True) + prompt = llama.tokenize( + result.prompt.encode("utf-8"), + add_bos=not result.added_special, + special=True, + ) if result.stop is not None: stop = [] if stop is None else [stop] if isinstance(stop, str) else stop rstop = result.stop if isinstance(result.stop, list) else [result.stop] @@ -560,7 +582,9 @@ def chat_completion_handler( stopping_criteria = result.stopping_criteria if response_format is not None and response_format["type"] == "json_object": - grammar = _grammar_for_response_format(response_format, verbose=llama.verbose) + grammar = _grammar_for_response_format( + response_format, verbose=llama.verbose + ) # Convert legacy functions to tools if functions is not None: @@ -587,7 +611,11 @@ def chat_completion_handler( } tool = None - if tool_choice is not None and isinstance(tool_choice, dict) and tools is not None: + if ( + tool_choice is not None + and isinstance(tool_choice, dict) + and tools is not None + ): name = tool_choice["function"]["name"] tool = next((t for t in tools if t["function"]["name"] == name), None) if tool is None: @@ -656,7 +684,9 @@ def format_autotokenizer( prompt: str = tokenizer.apply_chat_template(messages, tokenize=False) # type: ignore assert isinstance(prompt, str) # Return formatted prompt and eos token by default - return ChatFormatterResponse(prompt=prompt, stop=tokenizer.eos_token, added_special=True) + return ChatFormatterResponse( + prompt=prompt, stop=tokenizer.eos_token, added_special=True + ) return format_autotokenizer @@ -709,7 +739,9 @@ def format_tokenizer_config( bos_token=bos_token, eos_token=eos_token, ) - return ChatFormatterResponse(prompt=prompt, stop=[eos_token, bos_token], added_special=True) + return ChatFormatterResponse( + prompt=prompt, stop=[eos_token, bos_token], added_special=True + ) return format_tokenizer_config @@ -731,8 +763,10 @@ def guess_chat_format_from_gguf_metadata(metadata: Dict[str, str]) -> Optional[s if metadata["tokenizer.chat_template"] == CHATML_CHAT_TEMPLATE: return "chatml" - if (metadata["tokenizer.chat_template"] == MISTRAL_INSTRUCT_CHAT_TEMPLATE or - metadata["tokenizer.chat_template"] == MIXTRAL_INSTRUCT_CHAT_TEMPLATE): + if ( + metadata["tokenizer.chat_template"] == MISTRAL_INSTRUCT_CHAT_TEMPLATE + or metadata["tokenizer.chat_template"] == MIXTRAL_INSTRUCT_CHAT_TEMPLATE + ): return "mistral-instruct" if metadata["tokenizer.chat_template"] == LLAMA3_INSTRUCT_CHAT_TEMPLATE: @@ -868,13 +902,15 @@ def _format_chatglm3( ret += role return ret -def _grammar_for_json(verbose:bool=False): - return llama_grammar.LlamaGrammar.from_string(llama_grammar.JSON_GBNF, verbose=verbose) + +def _grammar_for_json(verbose: bool = False): + return llama_grammar.LlamaGrammar.from_string( + llama_grammar.JSON_GBNF, verbose=verbose + ) + def _grammar_for_json_schema( - schema: str, - verbose: bool = False, - fallback_to_json: bool = True + schema: str, verbose: bool = False, fallback_to_json: bool = True ): try: return llama_grammar.LlamaGrammar.from_json_schema(schema, verbose=verbose) @@ -884,9 +920,10 @@ def _grammar_for_json_schema( else: raise e + def _grammar_for_response_format( - response_format: llama_types.ChatCompletionRequestResponseFormat, - verbose: bool = False + response_format: llama_types.ChatCompletionRequestResponseFormat, + verbose: bool = False, ): if response_format["type"] != "json_object": return None @@ -898,6 +935,7 @@ def _grammar_for_response_format( else: return _grammar_for_json(verbose=verbose) + ### Chat Formats ### @@ -1239,10 +1277,7 @@ def format_mistral_instruct( and isinstance(message["content"], str) ): prompt += "[INST] " + message["content"] - elif ( - message["role"] == "assistant" - and message["content"] is not None - ): + elif message["role"] == "assistant" and message["content"] is not None: prompt += " [/INST]" + message["content"] + eos prompt += " [/INST]" return ChatFormatterResponse(prompt=prompt, stop=stop) @@ -1932,7 +1967,9 @@ def prepare_messages_for_inference( grammar=grammar, ) if stream is False: - completion_or_completion_chunks["choices"][0]["text"] = completion_or_completion_chunks["choices"][0]["text"].lstrip() + completion_or_completion_chunks["choices"][0]["text"] = ( + completion_or_completion_chunks["choices"][0]["text"].lstrip() + ) return _convert_completion_to_chat(completion_or_completion_chunks, stream=stream) # type: ignore def get_grammar(function_call): @@ -1969,45 +2006,50 @@ def get_grammar(function_call): return grammar def create_completion(prompt, stop, grammar): - completion = cast(llama_types.Completion, llama.create_completion( - prompt=prompt, - temperature=temperature, - top_p=top_p, - top_k=top_k, - min_p=min_p, - typical_p=typical_p, - stream=stream, - stop=stop, - max_tokens=max_tokens, - presence_penalty=presence_penalty, - frequency_penalty=frequency_penalty, - repeat_penalty=repeat_penalty, - tfs_z=tfs_z, - mirostat_mode=mirostat_mode, - mirostat_tau=mirostat_tau, - mirostat_eta=mirostat_eta, - model=model, - logits_processor=logits_processor, - grammar=grammar, - )) + completion = cast( + llama_types.Completion, + llama.create_completion( + prompt=prompt, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + typical_p=typical_p, + stream=stream, + stop=stop, + max_tokens=max_tokens, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + repeat_penalty=repeat_penalty, + tfs_z=tfs_z, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + model=model, + logits_processor=logits_processor, + grammar=grammar, + ), + ) return completion content = "" function_calls, function_bodies = [], [] completion_tokens = 0 - + def generate_streaming(tools, functions, function_call, prompt): assert version == "v2", "Streaming for v1 is not supported" - + chunk_id, chunk_created = None, None - + # If tool_choice/function_call is provided if isinstance(function_call, dict): prompt += f"{function_call['name']}\n{CONTENT_TOKEN}" grammar = get_grammar(function_call["name"]) stops = [STOP_TOKEN, FROM_TOKEN] - tool_id = "".join([random.choice(string.ascii_letters + string.digits) for _ in range(24)]) + tool_id = "".join( + [random.choice(string.ascii_letters + string.digits) for _ in range(24)] + ) completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) completion_text = "" first = True @@ -2021,19 +2063,35 @@ def generate_streaming(tools, functions, function_call, prompt): "index": 0, "id": "call_" + tool_id, "type": "function", - "function": {"name": function_call["name"], "arguments": ""}, + "function": { + "name": function_call["name"], + "arguments": "", + }, } ] } else: - func_call_dict = {"function_call": {"name": function_call["name"], "arguments": ""}} + func_call_dict = { + "function_call": { + "name": function_call["name"], + "arguments": "", + } + } yield llama_types.CreateChatCompletionStreamResponse( id="chat" + chunk["id"], object="chat.completion.chunk", created=chunk["created"], model=chunk["model"], choices=[ - {"index": 0, "logprobs": None, "delta": {"role": None, "content": None, **func_call_dict}} + { + "index": 0, + "logprobs": None, + "delta": { + "role": None, + "content": None, + **func_call_dict, + }, + } ], ) first = False @@ -2052,7 +2110,12 @@ def generate_streaming(tools, functions, function_call, prompt): ] } else: - func_call_dict = {"function_call": {"name": None, "arguments": chunk["choices"][0]["text"].rstrip()}} + func_call_dict = { + "function_call": { + "name": None, + "arguments": chunk["choices"][0]["text"].rstrip(), + } + } if len(chunk["choices"][0]["text"].rstrip()) > 0: yield llama_types.CreateChatCompletionStreamResponse( id="chat" + chunk["id"], @@ -2080,10 +2143,15 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "finish_reason": "tool_calls" if tools is not None else "function_call", + "finish_reason": ( + "tool_calls" if tools is not None else "function_call" + ), "logprobs": None, "delta": { - "role": None, "content": None, "function_call": None, "tool_calls": None + "role": None, + "content": None, + "function_call": None, + "tool_calls": None, }, } ], @@ -2095,7 +2163,9 @@ def generate_streaming(tools, functions, function_call, prompt): # Generate function name first grammar = None stops = CONTENT_TOKEN - completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) + completion = create_completion( + prompt=prompt, stop=stops, grammar=grammar + ) completion_text = "" for chunk in completion: completion_text += chunk["choices"][0]["text"] @@ -2124,7 +2194,12 @@ def generate_streaming(tools, functions, function_call, prompt): else: prompt += f"{function_name}\n<|content|>" grammar = get_grammar(function_name) - tool_id = "".join([random.choice(string.ascii_letters + string.digits) for _ in range(24)]) + tool_id = "".join( + [ + random.choice(string.ascii_letters + string.digits) + for _ in range(24) + ] + ) if tools is not None: func_call_dict = { "tool_calls": [ @@ -2132,12 +2207,17 @@ def generate_streaming(tools, functions, function_call, prompt): "index": tool_index, "id": "call_" + tool_id, "type": "function", - "function": {"name": function_name, "arguments": ""}, + "function": { + "name": function_name, + "arguments": "", + }, } ] } else: - func_call_dict = {"function_call": {"name": function_name, "arguments": ""}} + func_call_dict = { + "function_call": {"name": function_name, "arguments": ""} + } # Stream function name yield llama_types.CreateChatCompletionStreamResponse( id="chat" + chunk_id, @@ -2158,10 +2238,16 @@ def generate_streaming(tools, functions, function_call, prompt): ) # Generate content stops = [RECIPIENT_TOKEN, STOP_TOKEN] - completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) + completion = create_completion( + prompt=prompt, stop=stops, grammar=grammar + ) if function_name == "all": completion_text = "" - stop_sequence, buffer, is_end = "\n<|from|>assistant\n<|recipient|>", [], False + stop_sequence, buffer, is_end = ( + "\n<|from|>assistant\n<|recipient|>", + [], + False, + ) for i, chunk in enumerate(completion): completion_text += chunk["choices"][0]["text"] if is_end: @@ -2179,9 +2265,12 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": chunk["choices"][0][ + "logprobs" + ], "delta": { - "role": "assistant", "content": buffer.pop(0) + "role": "assistant", + "content": buffer.pop(0), }, } ], @@ -2204,17 +2293,30 @@ def generate_streaming(tools, functions, function_call, prompt): "logprobs": chunk["choices"][0]["logprobs"], "delta": { "role": "assistant", - "content": chunk["choices"][0]["text"] if i > 0 else chunk["choices"][0]["text"].lstrip() + "content": ( + chunk["choices"][0]["text"] + if i > 0 + else chunk["choices"][0][ + "text" + ].lstrip() + ), }, } ], ) # Check whether the model wants to generate another turn - if "<|from|> assistant" in completion_text or "<|from|>assistant" in completion_text: + if ( + "<|from|> assistant" in completion_text + or "<|from|>assistant" in completion_text + ): if completion_text.endswith("\n<|from|>assistant\n"): - cleaned_completion_text = completion_text[:-len("\n<|from|>assistant\n")].strip() + cleaned_completion_text = completion_text[ + : -len("\n<|from|>assistant\n") + ].strip() elif completion_text.endswith("\n<|from|> assistant\n"): - cleaned_completion_text = completion_text[:-len("\n<|from|> assistant\n")].strip() + cleaned_completion_text = completion_text[ + : -len("\n<|from|> assistant\n") + ].strip() else: cleaned_completion_text = completion_text.strip() prompt += f"{cleaned_completion_text}\n<|from|>assistant\n<|recipient|>" @@ -2250,13 +2352,22 @@ def generate_streaming(tools, functions, function_call, prompt): "type": "function", "function": { "name": None, - "arguments": chunk["choices"][0]["text"].rstrip(), + "arguments": chunk["choices"][0][ + "text" + ].rstrip(), }, } ] } else: - func_call_dict = {"function_call": {"name": None, "arguments": chunk["choices"][0]["text"].rstrip()}} + func_call_dict = { + "function_call": { + "name": None, + "arguments": chunk["choices"][0][ + "text" + ].rstrip(), + } + } yield llama_types.CreateChatCompletionStreamResponse( id="chat" + chunk_id, object="chat.completion.chunk", @@ -2276,9 +2387,16 @@ def generate_streaming(tools, functions, function_call, prompt): ) prompt += completion_text.strip() grammar = None - completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) - completion_text += "".join([chunk["choices"][0]["text"] for chunk in completion]) - if ("<|from|> assistant" in completion_text or "<|from|>assistant" in completion_text) and tools is not None: + completion = create_completion( + prompt=prompt, stop=stops, grammar=grammar + ) + completion_text += "".join( + [chunk["choices"][0]["text"] for chunk in completion] + ) + if ( + "<|from|> assistant" in completion_text + or "<|from|>assistant" in completion_text + ) and tools is not None: prompt += "\n<|from|>assistant\n<|recipient|>" tool_index += 1 else: @@ -2291,16 +2409,23 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "finish_reason": "tool_calls" if tools is not None else "function_call", + "finish_reason": ( + "tool_calls" + if tools is not None + else "function_call" + ), "logprobs": None, "delta": { - "role": None, "content": None, "function_call": None, "tool_calls": None + "role": None, + "content": None, + "function_call": None, + "tool_calls": None, }, } ], ) break - + if stream is not False: return generate_streaming( tools=tools, functions=functions, function_call=function_call, prompt=prompt @@ -2324,7 +2449,6 @@ def generate_streaming(tools, functions, function_call, prompt): completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) completion_text = completion["choices"][0]["text"] completion_tokens += completion["usage"]["completion_tokens"] - # If the generation does not involve a function call if ( @@ -2348,7 +2472,9 @@ def generate_streaming(tools, functions, function_call, prompt): completion_text.split(START_FUNCTION_CALL_TOKEN)[-1][:-1].strip() ) grammar = get_grammar(function_calls[-1]) - completion = create_completion(prompt=prompt, stop=END_FUNCTION_CALL_TOKEN, grammar=grammar) + completion = create_completion( + prompt=prompt, stop=END_FUNCTION_CALL_TOKEN, grammar=grammar + ) completion_tokens += completion["usage"]["completion_tokens"] function_bodies.append(completion["choices"][0]["text"].strip()) # If the prompt involves a function call, just append generated parameters to function_bodies @@ -2362,7 +2488,9 @@ def generate_streaming(tools, functions, function_call, prompt): function_calls.append(function_call) grammar = get_grammar(function_call) stops = [STOP_TOKEN, FROM_TOKEN] - completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) + completion = create_completion( + prompt=prompt, stop=stops, grammar=grammar + ) completion_text = completion["choices"][0]["text"] completion_tokens += completion["usage"]["completion_tokens"] function_bodies.append(completion_text.strip()) @@ -2372,7 +2500,9 @@ def generate_streaming(tools, functions, function_call, prompt): # Generate function name first grammar = None stops = CONTENT_TOKEN - completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) + completion = create_completion( + prompt=prompt, stop=stops, grammar=grammar + ) completion_text = completion["choices"][0]["text"] completion_tokens += completion["usage"]["completion_tokens"] function_name = completion_text.strip() @@ -2385,23 +2515,32 @@ def generate_streaming(tools, functions, function_call, prompt): grammar = get_grammar(function_call) # Generate content stops = [RECIPIENT_TOKEN, STOP_TOKEN] - completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) + completion = create_completion( + prompt=prompt, stop=stops, grammar=grammar + ) completion_text = completion["choices"][0]["text"] completion_tokens += completion["usage"]["completion_tokens"] if function_name == "all": if completion_text.endswith("\n<|from|>assistant\n"): - content += completion_text[:-len("\n<|from|>assistant\n")] + content += completion_text[: -len("\n<|from|>assistant\n")] if completion_text.endswith("\n<|from|> assistant\n"): content += completion_text[-len("\n<|from|> assistant\n")] else: content += completion_text content = content.lstrip() # Check whether the model wants to generate another turn - if "<|from|> assistant" in completion_text or "<|from|>assistant" in completion_text: + if ( + "<|from|> assistant" in completion_text + or "<|from|>assistant" in completion_text + ): if completion_text.endswith("\n<|from|>assistant\n"): - cleaned_completion_text = completion_text[:-len("\n<|from|>assistant\n")].strip() + cleaned_completion_text = completion_text[ + : -len("\n<|from|>assistant\n") + ].strip() elif completion_text.endswith("\n<|from|> assistant\n"): - cleaned_completion_text = completion_text[-len("\n<|from|> assistant\n")].strip() + cleaned_completion_text = completion_text[ + -len("\n<|from|> assistant\n") + ].strip() else: cleaned_completion_text = completion_text.strip() prompt += f"{cleaned_completion_text}\n<|from|>assistant\n<|recipient|>" @@ -2412,9 +2551,14 @@ def generate_streaming(tools, functions, function_call, prompt): # Check whether the model wants to generate another turn prompt += completion_text.strip() grammar = None - completion = create_completion(prompt=prompt, stop=stops, grammar=grammar) + completion = create_completion( + prompt=prompt, stop=stops, grammar=grammar + ) completion_tokens += completion["usage"]["completion_tokens"] - if "<|from|> assistant" in completion["choices"][0]["text"] or "<|from|>assistant" in completion["choices"][0]["text"]: + if ( + "<|from|> assistant" in completion["choices"][0]["text"] + or "<|from|>assistant" in completion["choices"][0]["text"] + ): prompt += "\n<|from|>assistant\n<|recipient|>" else: break @@ -2442,7 +2586,13 @@ def generate_streaming(tools, functions, function_call, prompt): ) # TODO: support stream mode - function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = {} + function_call_dict: Union[ + Dict[str, str], + Dict[ + Literal["function_call"], + llama_types.ChatCompletionRequestAssistantMessageFunctionCall, + ], + ] = {} if len(tool_calls) > 0: if tools is not None: function_call_dict["tool_calls"] = tool_calls @@ -2474,7 +2624,9 @@ def generate_streaming(tools, functions, function_call, prompt): class Llava15ChatHandler: - DEFAULT_SYSTEM_MESSAGE: Optional[str] = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." + DEFAULT_SYSTEM_MESSAGE: Optional[str] = ( + "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." + ) CHAT_FORMAT = ( "{% for message in messages %}" @@ -2487,7 +2639,6 @@ class Llava15ChatHandler: "{% endif %}" "{% if message.content is iterable %}" "\nUSER: " - "{% for content in message.content %}" "{% if content.type == 'image_url' and content.image_url is string %}" "{{ content.image_url }}" @@ -2496,13 +2647,11 @@ class Llava15ChatHandler: "{{ content.image_url.url }}" "{% endif %}" "{% endfor %}" - "{% for content in message.content %}" "{% if content.type == 'text' %}" "{{ content.text }}" "{% endif %}" "{% endfor %}" - "{% endif %}" "{% endif %}" "{% if message.role == 'assistant' and message.content is not none %}" @@ -2520,30 +2669,30 @@ def __init__(self, clip_model_path: str, verbose: bool = True): self.clip_model_path = clip_model_path self.verbose = verbose - self._llava_cpp = llava_cpp # TODO: Fix + self._llava_cpp = llava_cpp # TODO: Fix self._exit_stack = ExitStack() - self._last_image_embed: Optional[llava_cpp.CtypesPointer[llava_cpp.llava_image_embed]] = None + self._last_image_embed: Optional[ + llava_cpp.CtypesPointer[llava_cpp.llava_image_embed] + ] = None self._last_image_hash: Optional[int] = None if not os.path.exists(clip_model_path): raise ValueError(f"Clip model path does not exist: {clip_model_path}") with suppress_stdout_stderr(disable=self.verbose): - clip_ctx = self._llava_cpp.clip_model_load( - self.clip_model_path.encode(), 0 - ) + clip_ctx = self._llava_cpp.clip_model_load(self.clip_model_path.encode(), 0) if clip_ctx is None: raise ValueError(f"Failed to load clip model: {clip_model_path}") - + self.clip_ctx = clip_ctx def clip_free(): with suppress_stdout_stderr(disable=self.verbose): self._llava_cpp.clip_free(self.clip_ctx) - + self._exit_stack.callback(clip_free) - + def last_image_embed_free(): with suppress_stdout_stderr(disable=self.verbose): if self._last_image_embed is not None: @@ -2598,7 +2747,11 @@ def __call__( system_prompt = _get_system_message(messages) if system_prompt == "" and self.DEFAULT_SYSTEM_MESSAGE is not None: - messages = [llama_types.ChatCompletionRequestSystemMessage(role="system", content=self.DEFAULT_SYSTEM_MESSAGE)] + messages + messages = [ + llama_types.ChatCompletionRequestSystemMessage( + role="system", content=self.DEFAULT_SYSTEM_MESSAGE + ) + ] + messages image_urls = self.get_image_urls(messages) template = ImmutableSandboxedEnvironment( @@ -2614,7 +2767,11 @@ def __call__( split_text = self.split_text_on_image_urls(text, image_urls) def embed_image_bytes(image_bytes: bytes): - if self._last_image_embed is not None and self._last_image_hash is not None and hash(image_bytes) == self._last_image_hash: + if ( + self._last_image_embed is not None + and self._last_image_hash is not None + and hash(image_bytes) == self._last_image_hash + ): return self._last_image_embed with suppress_stdout_stderr(disable=self.verbose): # Free the previous image embed @@ -2622,13 +2779,13 @@ def embed_image_bytes(image_bytes: bytes): self._llava_cpp.llava_image_embed_free(self._last_image_embed) self._last_image_embed = None self._last_image_hash = None - embed = ( - self._llava_cpp.llava_image_embed_make_with_bytes( - self.clip_ctx, - llama.context_params.n_threads_batch, - (ctypes.c_uint8 * len(image_bytes)).from_buffer(bytearray(image_bytes)), - len(image_bytes), - ) + embed = self._llava_cpp.llava_image_embed_make_with_bytes( + self.clip_ctx, + llama.context_params.n_threads_batch, + (ctypes.c_uint8 * len(image_bytes)).from_buffer( + bytearray(image_bytes) + ), + len(image_bytes), ) self._last_image_embed = embed self._last_image_hash = hash(image_bytes) @@ -2639,15 +2796,21 @@ def embed_image_bytes(image_bytes: bytes): llama._ctx.kv_cache_clear() for type_, value in split_text: if type_ == "text": - tokens = llama.tokenize(value.encode("utf8"), add_bos=False, special=True) + tokens = llama.tokenize( + value.encode("utf8"), add_bos=False, special=True + ) if llama.n_tokens + len(tokens) > llama.n_ctx(): - raise ValueError(f"Prompt exceeds n_ctx: {llama.n_tokens + len(tokens)} > {llama.n_ctx()}") + raise ValueError( + f"Prompt exceeds n_ctx: {llama.n_tokens + len(tokens)} > {llama.n_ctx()}" + ) llama.eval(tokens) else: image_bytes = self.load_image(value) embed = embed_image_bytes(image_bytes) if llama.n_tokens + embed.contents.n_image_pos > llama.n_ctx(): - raise ValueError(f"Prompt exceeds n_ctx: {llama.n_tokens + embed.contents.n_image_pos} > {llama.n_ctx()}") + raise ValueError( + f"Prompt exceeds n_ctx: {llama.n_tokens + embed.contents.n_image_pos} > {llama.n_ctx()}" + ) n_past = ctypes.c_int(llama.n_tokens) n_past_p = ctypes.pointer(n_past) with suppress_stdout_stderr(disable=self.verbose): @@ -2692,7 +2855,11 @@ def embed_image_bytes(image_bytes: bytes): } tool = None - if tool_choice is not None and isinstance(tool_choice, dict) and tools is not None: + if ( + tool_choice is not None + and isinstance(tool_choice, dict) + and tools is not None + ): name = tool_choice["function"]["name"] tool = next((t for t in tools if t["function"]["name"] == name), None) if tool is None: @@ -2809,9 +2976,10 @@ def from_pretrained( ) -> "Llava15ChatHandler": import fnmatch from pathlib import Path + try: - from huggingface_hub import hf_hub_download, HfFileSystem # type: ignore - from huggingface_hub.utils import validate_repo_id # type: ignore + from huggingface_hub import hf_hub_download, HfFileSystem # type: ignore + from huggingface_hub.utils import validate_repo_id # type: ignore except ImportError: raise ImportError( "Llama.from_pretrained requires the huggingface-hub package. " @@ -2824,7 +2992,7 @@ def from_pretrained( files = [ file["name"] if isinstance(file, dict) else file - for file in hffs.ls(repo_id) # type: ignore + for file in hffs.ls(repo_id) # type: ignore ] # split each file into repo_id, subfolder, filename @@ -2880,6 +3048,7 @@ def from_pretrained( **kwargs, ) + class ObsidianChatHandler(Llava15ChatHandler): # Prompt Format # The model followed ChatML format. However, with ### as the seperator @@ -2906,7 +3075,6 @@ class ObsidianChatHandler(Llava15ChatHandler): "{{ message.content }}" "{% endif %}" "{% if message.content is iterable %}" - "{% for content in message.content %}" "{% if content.type == 'image_url' and content.image_url is string %}" "{{ content.image_url }}" @@ -2915,13 +3083,11 @@ class ObsidianChatHandler(Llava15ChatHandler): "{{ content.image_url.url }}" "{% endif %}" "{% endfor %}" - "{% for content in message.content %}" "{% if content.type == 'text' %}" "{{ content.text }}" "{% endif %}" "{% endfor %}" - "{% endif %}" "###\n" "{% endif %}" @@ -2938,6 +3104,7 @@ class ObsidianChatHandler(Llava15ChatHandler): "{% endif %}" ) + class MoondreamChatHandler(Llava15ChatHandler): # Chat Format: # f"\n\n{chat_history}Question: {question}\n\nAnswer:" @@ -2945,7 +3112,6 @@ class MoondreamChatHandler(Llava15ChatHandler): "{% for message in messages %}" "{% if message.role == 'user' %}" "{% if message.content is iterable %}" - # "{% for content in message.content %}" "{% if content.type == 'image_url' %}" @@ -2957,35 +3123,30 @@ class MoondreamChatHandler(Llava15ChatHandler): "{% endif %}" "{% endif %}" "{% endfor %}" - # Question: "{% for content in message.content %}" "{% if content.type == 'text' %}" "Question: {{ content.text }}\n\n" "{% endif %}" "{% endfor %}" - "{% endif %}" - # Question: "{% if message.content is string %}" "Question: {{ message.content }}\n\n" "{% endif %}" - "{% endif %}" - # Answer: "{% if message.role == 'assistant' %}" "Answer:{{ message.content }}\n\n" "{% endif %}" "{% endfor %}" - # Generation prompt "{% if add_generation_prompt %}" "Answer:" "{% endif %}" ) + class Llava16ChatHandler(Llava15ChatHandler): DEFAULT_SYSTEM_MESSAGE = "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. " @@ -2999,7 +3160,6 @@ class Llava16ChatHandler(Llava15ChatHandler): "{% endif %}" "{% if message.role == 'user' %}" "{% if message.content is iterable %}" - # "{% for content in message.content %}" "{% if content.type == 'image_url' %}" @@ -3011,35 +3171,30 @@ class Llava16ChatHandler(Llava15ChatHandler): "{% endif %}" "{% endif %}" "{% endfor %}" - # Question: "{% for content in message.content %}" "{% if content.type == 'text' %}" "{{ content.text }}" "{% endif %}" "{% endfor %}" - "{% endif %}" - # Question: "{% if message.content is string %}" "{{ message.content }}" "{% endif %}" - "{% endif %}" - # Answer: "{% if message.role == 'assistant' %}" "{{ message.content }}" "{% endif %}" "{% endfor %}" - # Generation prompt "{% if add_generation_prompt %}" "Answer:" "{% endif %}" ) + class NanoLlavaChatHandler(Llava15ChatHandler): # Prompt Format # The model follow the ChatML standard, however, without \n at the end of <|im_end|>: @@ -3065,7 +3220,6 @@ class NanoLlavaChatHandler(Llava15ChatHandler): "{{ message.content }}" "{% endif %}" "{% if message.content is iterable %}" - "{% for content in message.content %}" "{% if content.type == 'image_url' and content.image_url is string %}" "{{ content.image_url }}" @@ -3074,13 +3228,11 @@ class NanoLlavaChatHandler(Llava15ChatHandler): "{{ content.image_url.url }}" "{% endif %}" "{% endfor %}" - "{% for content in message.content %}" "{% if content.type == 'text' %}" "{{ content.text }}" "{% endif %}" "{% endfor %}" - "{% endif %}" "<|im_end|>" "{% endif %}" @@ -3097,6 +3249,7 @@ class NanoLlavaChatHandler(Llava15ChatHandler): "{% endif %}" ) + class Llama3VisionAlphaChatHandler(Llava15ChatHandler): # question = "" + q @@ -3105,15 +3258,10 @@ class Llama3VisionAlphaChatHandler(Llava15ChatHandler): CHAT_FORMAT = ( "{% for message in messages %}" - "<|start_header_id|>" - "{% if message.role == 'user' %}" - "user<|end_header_id|>\n\n" - "{% if message.content is iterable %}" - # "{% for content in message.content %}" "{% if content.type == 'image_url' %}" @@ -3125,39 +3273,32 @@ class Llama3VisionAlphaChatHandler(Llava15ChatHandler): "{% endif %}" "{% endif %}" "{% endfor %}" - # Question: "{% for content in message.content %}" "{% if content.type == 'text' %}" "{{ content.text }}" "{% endif %}" "{% endfor %}" - "{% endif %}" - # Question: "{% if message.content is string %}" "{{ message.content }}" "{% endif %}" - "{% endif %}" - # Answer: "{% if message.role == 'assistant' %}" "assistant<|end_header_id|>\n\n" "{{ message.content }}" "{% endif %}" - "<|eot_id|>" - "{% endfor %}" - # Generation prompt "{% if add_generation_prompt %}" "<|start_header_id|>assistant<|end_header_id|>\n\n" "{% endif %}" ) + # alias Llama3VisionAlpha = Llama3VisionAlphaChatHandler @@ -3276,7 +3417,11 @@ def chatml_function_calling( }, } - stop = [stop, "<|im_end|>"] if isinstance(stop, str) else stop + ["<|im_end|>"] if stop else ["<|im_end|>"] + stop = ( + [stop, "<|im_end|>"] + if isinstance(stop, str) + else stop + ["<|im_end|>"] if stop else ["<|im_end|>"] + ) # Case 1: No tool choice by user if ( @@ -3489,7 +3634,9 @@ def chatml_function_calling( logits_processor=logits_processor, grammar=grammar, ) - completion_or_chunks = cast(llama_types.CreateCompletionResponse, completion_or_chunks) + completion_or_chunks = cast( + llama_types.CreateCompletionResponse, completion_or_chunks + ) completions.append(completion_or_chunks) completions_tool_name.append(tool_name) prompt += completion_or_chunks["choices"][0]["text"] @@ -3526,12 +3673,22 @@ def chatml_function_calling( ) # Merge completions - function_call_dict: Union[Dict[str, str], Dict[Literal["function_call"], llama_types.ChatCompletionRequestAssistantMessageFunctionCall]] = { - "function_call": { - "name": tool_name, - "arguments": completions[0]["choices"][0]["text"], + function_call_dict: Union[ + Dict[str, str], + Dict[ + Literal["function_call"], + llama_types.ChatCompletionRequestAssistantMessageFunctionCall, + ], + ] = ( + { + "function_call": { + "name": tool_name, + "arguments": completions[0]["choices"][0]["text"], + } } - } if len(completions) == 1 else {} + if len(completions) == 1 + else {} + ) return { "id": "chat" + completion["id"], "object": "chat.completion", @@ -3562,13 +3719,17 @@ def chatml_function_calling( zip(completions_tool_name, completions) ) ], - **function_call_dict + **function_call_dict, }, } ], "usage": { "completion_tokens": sum( - completion["usage"]["completion_tokens"] if "usage" in completion else 0 + ( + completion["usage"]["completion_tokens"] + if "usage" in completion + else 0 + ) for completion in completions ), "prompt_tokens": sum( diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index a0f5832c62..decc08e657 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -56,7 +56,7 @@ def _load_shared_library(lib_base_name: str): # Add the library directory to the DLL search path on Windows (if needed) if sys.platform == "win32": os.add_dll_directory(str(_base_path)) - os.environ['PATH'] = str(_base_path) + os.pathsep + os.environ['PATH'] + os.environ["PATH"] = str(_base_path) + os.pathsep + os.environ["PATH"] if sys.platform == "win32" and sys.version_info >= (3, 8): os.add_dll_directory(str(_base_path)) @@ -490,7 +490,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_POOLING_TYPE_NONE = 0 LLAMA_POOLING_TYPE_MEAN = 1 LLAMA_POOLING_TYPE_CLS = 2 -LLAMA_POOLING_TYPE_LAST = 3 +LLAMA_POOLING_TYPE_LAST = 3 # enum llama_attention_type { # LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1, @@ -812,6 +812,7 @@ class llama_model_params(ctypes.Structure): # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU # bool flash_attn; // whether to use flash attention [EXPERIMENTAL] + # // Abort callback # // if it returns true, execution of llama_decode() will be aborted # // currently works only with CPU execution @@ -1472,10 +1473,13 @@ def llama_model_has_encoder(model: llama_model_p, /) -> bool: # // For encoder-decoder models, this function returns id of the token that must be provided # // to the decoder to start generating output sequence. For other models, it returns -1. # LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model); -@ctypes_function("llama_model_decoder_start_token", [llama_model_p_ctypes], ctypes.c_int32) +@ctypes_function( + "llama_model_decoder_start_token", [llama_model_p_ctypes], ctypes.c_int32 +) def llama_model_decoder_start_token(model: llama_model_p, /) -> int: """For encoder-decoder models, this function returns id of the token that must be provided - to the decoder to start generating output sequence. For other models, it returns -1.""" + to the decoder to start generating output sequence. For other models, it returns -1. + """ ... @@ -2568,7 +2572,9 @@ def llama_token_is_eog(model: llama_model_p, token: Union[llama_token, int], /) @ctypes_function( "llama_token_is_control", [llama_model_p_ctypes, llama_token], ctypes.c_bool ) -def llama_token_is_control(model: llama_model_p, token: Union[llama_token, int], /) -> bool: +def llama_token_is_control( + model: llama_model_p, token: Union[llama_token, int], / +) -> bool: """Identify if Token Id is a control token or a render-able token""" ... @@ -3364,6 +3370,7 @@ def llama_grammar_accept_token( # // Model split # // + # /// @details Build a split GGUF final path for this chunk. # /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" # // Returns the split_path length. diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index d9a3823b4f..0ac7354bba 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -167,12 +167,10 @@ def __getitem__(self, index: int) -> str: return value[index] if index < len(value) else "" @overload - def __add__(self: Ptr, other: int) -> Ptr: - ... + def __add__(self: Ptr, other: int) -> Ptr: ... @overload - def __add__(self: Ptr, other: Ptr) -> int: - ... + def __add__(self: Ptr, other: Ptr) -> int: ... def __add__(self: Ptr, other: Union[int, Ptr]) -> Union[int, Ptr]: return ( @@ -182,12 +180,10 @@ def __add__(self: Ptr, other: Union[int, Ptr]) -> Union[int, Ptr]: ) @overload - def __sub__(self: Ptr, other: int) -> Ptr: - ... + def __sub__(self: Ptr, other: int) -> Ptr: ... @overload - def __sub__(self: Ptr, other: Ptr) -> int: - ... + def __sub__(self: Ptr, other: Ptr) -> int: ... def __sub__(self: Ptr, other: Union[int, Ptr]) -> Union[int, Ptr]: return ( @@ -1396,40 +1392,48 @@ def print_grammar(file: TextIO, state: parse_state) -> None: SPACE_RULE = '" "?' -def _build_repetition(item_rule, min_items, max_items, separator_rule=None, item_rule_is_literal=False): +def _build_repetition( + item_rule, min_items, max_items, separator_rule=None, item_rule_is_literal=False +): if not separator_rule: if min_items == 0 and max_items == 1: - return f'{item_rule}?' + return f"{item_rule}?" elif min_items == 1 and max_items is None: - return f'{item_rule}+' + return f"{item_rule}+" - result = '' + result = "" if min_items > 0: if item_rule_is_literal and separator_rule is None: result = '"' + (item_rule[1:-1] * min_items) + '"' else: - result = (f' {separator_rule} ' if separator_rule else ' ').join([item_rule] * min_items) + result = (f" {separator_rule} " if separator_rule else " ").join( + [item_rule] * min_items + ) def opt_repetitions(up_to_n, prefix_with_sep=False): - ''' - - n=4, no sep: '(a (a (a (a)?)?)?)?' - - n=4, sep=',', prefix: '("," a ("," a ("," a ("," a)?)?)?)?' - - n=4, sep=',', no prefix: '(a ("," a ("," a ("," a)?)?)?)?' - ''' - - content = f'{separator_rule} {item_rule}' if prefix_with_sep and separator_rule else item_rule + """ + - n=4, no sep: '(a (a (a (a)?)?)?)?' + - n=4, sep=',', prefix: '("," a ("," a ("," a ("," a)?)?)?)?' + - n=4, sep=',', no prefix: '(a ("," a ("," a ("," a)?)?)?)?' + """ + + content = ( + f"{separator_rule} {item_rule}" + if prefix_with_sep and separator_rule + else item_rule + ) if up_to_n == 0: - return '' + return "" elif up_to_n == 1: - return f'({content})?' + return f"({content})?" elif separator_rule and not prefix_with_sep: - return f'({content} {opt_repetitions(up_to_n - 1, prefix_with_sep=True)})?' + return f"({content} {opt_repetitions(up_to_n - 1, prefix_with_sep=True)})?" else: - return (f'({content} ' * up_to_n).rstrip() + (')?' * up_to_n) + return (f"({content} " * up_to_n).rstrip() + (")?" * up_to_n) if min_items > 0 and max_items != min_items: - result += ' ' + result += " " if max_items is not None: result += opt_repetitions(max_items - min_items, prefix_with_sep=min_items > 0) @@ -1437,56 +1441,81 @@ def opt_repetitions(up_to_n, prefix_with_sep=False): item_operator = f'({separator_rule + " " if separator_rule else ""}{item_rule})' if min_items == 0 and separator_rule: - result = f'({item_rule} {item_operator}*)?' + result = f"({item_rule} {item_operator}*)?" else: - result += f'{item_operator}*' + result += f"{item_operator}*" return result - class BuiltinRule: def __init__(self, content: str, deps: list = None): self.content = content self.deps = deps or [] -_up_to_15_digits = _build_repetition('[0-9]', 0, 15) + +_up_to_15_digits = _build_repetition("[0-9]", 0, 15) PRIMITIVE_RULES = { - 'boolean' : BuiltinRule('("true" | "false") space', []), - 'decimal-part' : BuiltinRule('[0-9] ' + _up_to_15_digits, []), - 'integral-part': BuiltinRule('[0-9] | [1-9] ' + _up_to_15_digits, []), - 'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space', ['integral-part', 'decimal-part']), - 'integer' : BuiltinRule('("-"? integral-part) space', ['integral-part']), - 'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']), - 'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', ['string', 'value']), - 'array' : BuiltinRule('"[" space ( value ("," space value)* )? "]" space', ['value']), - 'uuid' : BuiltinRule(r'"\"" ' + ' "-" '.join('[0-9a-fA-F]' * n for n in [8, 4, 4, 4, 12]) + r' "\"" space', []), - 'char' : BuiltinRule(r'[^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])', []), - 'string' : BuiltinRule(r'"\"" char* "\"" space', ['char']), - 'null' : BuiltinRule('"null" space', []), + "boolean": BuiltinRule('("true" | "false") space', []), + "decimal-part": BuiltinRule("[0-9] " + _up_to_15_digits, []), + "integral-part": BuiltinRule("[0-9] | [1-9] " + _up_to_15_digits, []), + "number": BuiltinRule( + '("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space', + ["integral-part", "decimal-part"], + ), + "integer": BuiltinRule('("-"? integral-part) space', ["integral-part"]), + "value": BuiltinRule( + "object | array | string | number | boolean | null", + ["object", "array", "string", "number", "boolean", "null"], + ), + "object": BuiltinRule( + '"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', + ["string", "value"], + ), + "array": BuiltinRule( + '"[" space ( value ("," space value)* )? "]" space', ["value"] + ), + "uuid": BuiltinRule( + r'"\"" ' + + ' "-" '.join("[0-9a-fA-F]" * n for n in [8, 4, 4, 4, 12]) + + r' "\"" space', + [], + ), + "char": BuiltinRule( + r'[^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])', + [], + ), + "string": BuiltinRule(r'"\"" char* "\"" space', ["char"]), + "null": BuiltinRule('"null" space', []), } # TODO: support "uri", "email" string formats STRING_FORMAT_RULES = { - 'date' : BuiltinRule('[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []), - 'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []), - 'date-time' : BuiltinRule('date "T" time', ['date', 'time']), - 'date-string' : BuiltinRule('"\\"" date "\\"" space', ['date']), - 'time-string' : BuiltinRule('"\\"" time "\\"" space', ['time']), - 'date-time-string': BuiltinRule('"\\"" date-time "\\"" space', ['date-time']), + "date": BuiltinRule( + '[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [0-1] )', + [], + ), + "time": BuiltinRule( + '([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', + [], + ), + "date-time": BuiltinRule('date "T" time', ["date", "time"]), + "date-string": BuiltinRule('"\\"" date "\\"" space', ["date"]), + "time-string": BuiltinRule('"\\"" time "\\"" space', ["time"]), + "date-time-string": BuiltinRule('"\\"" date-time "\\"" space', ["date-time"]), } -DOTALL = '[\\U00000000-\\U0010FFFF]' -DOT = '[^\\x0A\\x0D]' - -RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()]) +DOTALL = "[\\U00000000-\\U0010FFFF]" +DOT = "[^\\x0A\\x0D]" - -NON_LITERAL_SET = set('|.()[]{}*+?') -ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('[]()|{}*+?') +RESERVED_NAMES = set( + ["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()] +) +NON_LITERAL_SET = set("|.()[]{}*+?") +ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set("[]()|{}*+?") class SchemaConverter: @@ -1496,7 +1525,7 @@ def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern): self._dotall = dotall self._raw_pattern = raw_pattern self._rules = { - 'space': SPACE_RULE, + "space": SPACE_RULE, } self._refs = {} self._refs_being_resolved = set() @@ -1507,76 +1536,89 @@ def _format_literal(self, literal): ) return f'"{escaped}"' - def not_literal(self, literal: str, dotall: bool = True, maybe_escaped_underscores = False) -> str: - ''' - not_literal('a') -> '[^a]' - not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?' - ''' - assert len(literal) > 0, 'Empty literal not supported' + def not_literal( + self, literal: str, dotall: bool = True, maybe_escaped_underscores=False + ) -> str: + """ + not_literal('a') -> '[^a]' + not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?' + """ + assert len(literal) > 0, "Empty literal not supported" + def recurse(i: int): c = literal[i] - if maybe_escaped_underscores and c == '_': - yield f'[^{c}\\\\]' - yield ' | ' + if maybe_escaped_underscores and c == "_": + yield f"[^{c}\\\\]" + yield " | " yield f'"\\\\"? "{c}"' else: - yield f'[^{c}]' + yield f"[^{c}]" if i < len(literal) - 1: - yield ' | ' + yield " | " yield self._format_literal(c) - yield ' (' + yield " (" yield from recurse(i + 1) - yield ')?' + yield ")?" - return ''.join(('(', *recurse(0), ')')) + return "".join(("(", *recurse(0), ")")) def _add_rule(self, name, rule): - esc_name = INVALID_RULE_CHARS_RE.sub('-', name) + esc_name = INVALID_RULE_CHARS_RE.sub("-", name) if esc_name not in self._rules or self._rules[esc_name] == rule: key = esc_name else: i = 0 - while f'{esc_name}{i}' in self._rules and self._rules[f'{esc_name}{i}'] != rule: + while ( + f"{esc_name}{i}" in self._rules + and self._rules[f"{esc_name}{i}"] != rule + ): i += 1 - key = f'{esc_name}{i}' + key = f"{esc_name}{i}" self._rules[key] = rule return key def resolve_refs(self, schema: dict, url: str): - ''' - Resolves all $ref fields in the given schema, fetching any remote schemas, - replacing $ref with absolute reference URL and populating self._refs with the - respective referenced (sub)schema dictionaries. - ''' + """ + Resolves all $ref fields in the given schema, fetching any remote schemas, + replacing $ref with absolute reference URL and populating self._refs with the + respective referenced (sub)schema dictionaries. + """ + def visit(n: dict): if isinstance(n, list): return [visit(x) for x in n] elif isinstance(n, dict): - ref = n.get('$ref') + ref = n.get("$ref") if ref is not None and ref not in self._refs: - if ref.startswith('https://'): - assert self._allow_fetch, 'Fetching remote schemas is not allowed (use --allow-fetch for force)' + if ref.startswith("https://"): + assert ( + self._allow_fetch + ), "Fetching remote schemas is not allowed (use --allow-fetch for force)" import requests - frag_split = ref.split('#') + frag_split = ref.split("#") base_url = frag_split[0] target = self._refs.get(base_url) if target is None: - target = self.resolve_refs(requests.get(ref).json(), base_url) + target = self.resolve_refs( + requests.get(ref).json(), base_url + ) self._refs[base_url] = target - if len(frag_split) == 1 or frag_split[-1] == '': + if len(frag_split) == 1 or frag_split[-1] == "": return target - elif ref.startswith('#/'): + elif ref.startswith("#/"): target = schema - ref = f'{url}{ref}' - n['$ref'] = ref + ref = f"{url}{ref}" + n["$ref"] = ref else: - raise ValueError(f'Unsupported ref {ref}') + raise ValueError(f"Unsupported ref {ref}") - for sel in ref.split('#')[-1].split('/')[1:]: - assert target is not None and sel in target, f'Error resolving ref {ref}: {sel} not in {target}' + for sel in ref.split("#")[-1].split("/")[1:]: + assert ( + target is not None and sel in target + ), f"Error resolving ref {ref}: {sel} not in {target}" target = target[sel] self._refs[ref] = target @@ -1585,28 +1627,33 @@ def visit(n: dict): visit(v) return n + return visit(schema) def _generate_union_rule(self, name, alt_schemas): - return ' | '.join(( - self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') - for i, alt_schema in enumerate(alt_schemas) - )) + return " | ".join( + ( + self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') + for i, alt_schema in enumerate(alt_schemas) + ) + ) def _visit_pattern(self, pattern, name): - ''' - Transforms a regular expression pattern into a GBNF rule. + """ + Transforms a regular expression pattern into a GBNF rule. - Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions - Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md + Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions + Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md - Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. + Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. - Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which - we define sub-rules to keep the output lean. - ''' + Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which + we define sub-rules to keep the output lean. + """ - assert pattern.startswith('^') and pattern.endswith('$'), 'Pattern must start with "^" and end with "$"' + assert pattern.startswith("^") and pattern.endswith( + "$" + ), 'Pattern must start with "^" and end with "$"' pattern = pattern[1:-1] sub_rule_ids = {} @@ -1615,12 +1662,12 @@ def _visit_pattern(self, pattern, name): def to_rule(s: Tuple[str, bool]) -> str: (txt, is_literal) = s - return "\"" + txt + "\"" if is_literal else txt + return '"' + txt + '"' if is_literal else txt def transform() -> Tuple[str, bool]: - ''' - Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. - ''' + """ + Parse a unit at index i (advancing it), and return its string representation + whether it's a literal. + """ nonlocal i nonlocal pattern nonlocal sub_rule_ids @@ -1638,64 +1685,72 @@ def get_dot(): else: # Accept any character... except \n and \r line break chars (\x0A and \xOD) rule = DOT - return self._add_rule(f'dot', rule) + return self._add_rule(f"dot", rule) def join_seq(): nonlocal seq ret = [] for is_literal, g in groupby(seq, lambda x: x[1]): if is_literal: - ret.append((''.join(x[0] for x in g), True)) + ret.append(("".join(x[0] for x in g), True)) else: ret.extend(g) if len(ret) == 1: return ret[0] - return (' '.join(to_rule(x) for x in seq), False) + return (" ".join(to_rule(x) for x in seq), False) while i < length: c = pattern[i] - if c == '.': + if c == ".": seq.append((get_dot(), False)) i += 1 - elif c == '(': + elif c == "(": i += 1 if i < length: - assert pattern[i] != '?', f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' - seq.append((f'({to_rule(transform())})', False)) - elif c == ')': + assert ( + pattern[i] != "?" + ), f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' + seq.append((f"({to_rule(transform())})", False)) + elif c == ")": i += 1 - assert start > 0 and pattern[start-1] == '(', f'Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}' + assert ( + start > 0 and pattern[start - 1] == "(" + ), f"Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}" return join_seq() - elif c == '[': + elif c == "[": square_brackets = c i += 1 - while i < length and pattern[i] != ']': - if pattern[i] == '\\': - square_brackets += pattern[i:i+2] + while i < length and pattern[i] != "]": + if pattern[i] == "\\": + square_brackets += pattern[i : i + 2] i += 2 else: square_brackets += pattern[i] i += 1 - assert i < length, f'Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}' - square_brackets += ']' + assert ( + i < length + ), f"Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}" + square_brackets += "]" i += 1 seq.append((square_brackets, False)) - elif c == '|': - seq.append(('|', False)) + elif c == "|": + seq.append(("|", False)) i += 1 - elif c in ('*', '+', '?'): + elif c in ("*", "+", "?"): seq[-1] = (to_rule(seq[-1]) + c, False) i += 1 - elif c == '{': + elif c == "{": curly_brackets = c i += 1 - while i < length and pattern[i] != '}': + while i < length and pattern[i] != "}": curly_brackets += pattern[i] i += 1 - assert i < length, f'Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}' - curly_brackets += '}' + assert ( + i < length + ), f"Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}" + curly_brackets += "}" i += 1 - nums = [s.strip() for s in curly_brackets[1:-1].split(',')] + nums = [s.strip() for s in curly_brackets[1:-1].split(",")] min_times = 0 max_times = None try: @@ -1707,35 +1762,49 @@ def join_seq(): min_times = int(nums[0]) if nums[0] else 0 max_times = int(nums[1]) if nums[1] else None except ValueError: - raise ValueError(f'Invalid quantifier {curly_brackets} in /{pattern}/') + raise ValueError( + f"Invalid quantifier {curly_brackets} in /{pattern}/" + ) (sub, sub_is_literal) = seq[-1] if not sub_is_literal: id = sub_rule_ids.get(sub) if id is None: - id = self._add_rule(f'{name}-{len(sub_rule_ids) + 1}', sub) + id = self._add_rule(f"{name}-{len(sub_rule_ids) + 1}", sub) sub_rule_ids[sub] = id sub = id - seq[-1] = (_build_repetition(f'"{sub}"' if sub_is_literal else sub, min_times, max_times, item_rule_is_literal=sub_is_literal), False) + seq[-1] = ( + _build_repetition( + f'"{sub}"' if sub_is_literal else sub, + min_times, + max_times, + item_rule_is_literal=sub_is_literal, + ), + False, + ) else: - literal = '' + literal = "" while i < length: - if pattern[i] == '\\' and i < length - 1: + if pattern[i] == "\\" and i < length - 1: next = pattern[i + 1] if next in ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS: i += 1 literal += pattern[i] i += 1 else: - literal += pattern[i:i+2] + literal += pattern[i : i + 2] i += 2 elif pattern[i] == '"' and not self._raw_pattern: literal += '\\"' i += 1 - elif pattern[i] not in NON_LITERAL_SET and \ - (i == length - 1 or literal == '' or pattern[i+1] == '.' or pattern[i+1] not in NON_LITERAL_SET): + elif pattern[i] not in NON_LITERAL_SET and ( + i == length - 1 + or literal == "" + or pattern[i + 1] == "." + or pattern[i + 1] not in NON_LITERAL_SET + ): literal += pattern[i] i += 1 else: @@ -1747,12 +1816,15 @@ def join_seq(): return self._add_rule( name, - to_rule(transform()) if self._raw_pattern \ - else "\"\\\"\" " + to_rule(transform()) + " \"\\\"\" space") - + ( + to_rule(transform()) + if self._raw_pattern + else '"\\"" ' + to_rule(transform()) + ' "\\"" space' + ), + ) def _resolve_ref(self, ref): - ref_name = ref.split('/')[-1] + ref_name = ref.split("/")[-1] if ref_name not in self._rules and ref not in self._refs_being_resolved: self._refs_being_resolved.add(ref) resolved = self._refs[ref] @@ -1764,131 +1836,203 @@ def _generate_constant_rule(self, value): return self._format_literal(json.dumps(value)) def visit(self, schema, name): - schema_type = schema.get('type') - schema_format = schema.get('format') - rule_name = name + '-' if name in RESERVED_NAMES else name or 'root' + schema_type = schema.get("type") + schema_format = schema.get("format") + rule_name = name + "-" if name in RESERVED_NAMES else name or "root" - if (ref := schema.get('$ref')) is not None: + if (ref := schema.get("$ref")) is not None: return self._add_rule(rule_name, self._resolve_ref(ref)) - elif 'oneOf' in schema or 'anyOf' in schema: - return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf'])) + elif "oneOf" in schema or "anyOf" in schema: + return self._add_rule( + rule_name, + self._generate_union_rule(name, schema.get("oneOf") or schema["anyOf"]), + ) elif isinstance(schema_type, list): - return self._add_rule(rule_name, self._generate_union_rule(name, [{'type': t} for t in schema_type])) + return self._add_rule( + rule_name, + self._generate_union_rule(name, [{"type": t} for t in schema_type]), + ) - elif 'const' in schema: - return self._add_rule(rule_name, self._generate_constant_rule(schema['const'])) + elif "const" in schema: + return self._add_rule( + rule_name, self._generate_constant_rule(schema["const"]) + ) - elif 'enum' in schema: - rule = ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + elif "enum" in schema: + rule = " | ".join((self._generate_constant_rule(v) for v in schema["enum"])) return self._add_rule(rule_name, rule) - elif schema_type in (None, 'object') and \ - ('properties' in schema or \ - ('additionalProperties' in schema and schema['additionalProperties'] is not True)): - required = set(schema.get('required', [])) - properties = list(schema.get('properties', {}).items()) - return self._add_rule(rule_name, self._build_object_rule(properties, required, name, schema.get('additionalProperties'))) + elif schema_type in (None, "object") and ( + "properties" in schema + or ( + "additionalProperties" in schema + and schema["additionalProperties"] is not True + ) + ): + required = set(schema.get("required", [])) + properties = list(schema.get("properties", {}).items()) + return self._add_rule( + rule_name, + self._build_object_rule( + properties, required, name, schema.get("additionalProperties") + ), + ) - elif schema_type in (None, 'object') and 'allOf' in schema: + elif schema_type in (None, "object") and "allOf" in schema: required = set() properties = [] hybrid_name = name + def add_component(comp_schema, is_required): - if (ref := comp_schema.get('$ref')) is not None: + if (ref := comp_schema.get("$ref")) is not None: comp_schema = self._refs[ref] - if 'properties' in comp_schema: - for prop_name, prop_schema in comp_schema['properties'].items(): + if "properties" in comp_schema: + for prop_name, prop_schema in comp_schema["properties"].items(): properties.append((prop_name, prop_schema)) if is_required: required.add(prop_name) - for t in schema['allOf']: - if 'anyOf' in t: - for tt in t['anyOf']: + for t in schema["allOf"]: + if "anyOf" in t: + for tt in t["anyOf"]: add_component(tt, is_required=False) else: add_component(t, is_required=True) - return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=[])) + return self._add_rule( + rule_name, + self._build_object_rule( + properties, required, hybrid_name, additional_properties=[] + ), + ) - elif schema_type in (None, 'array') and ('items' in schema or 'prefixItems' in schema): - items = schema.get('items') or schema['prefixItems'] + elif schema_type in (None, "array") and ( + "items" in schema or "prefixItems" in schema + ): + items = schema.get("items") or schema["prefixItems"] if isinstance(items, list): return self._add_rule( rule_name, - '"[" space ' + - ' "," space '.join( + '"[" space ' + + ' "," space '.join( self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') - for i, item in enumerate(items)) + - ' "]" space') + for i, item in enumerate(items) + ) + + ' "]" space', + ) else: item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') min_items = schema.get("minItems", 0) max_items = schema.get("maxItems") - return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' "]" space') + return self._add_rule( + rule_name, + '"[" space ' + + _build_repetition( + item_rule_name, min_items, max_items, separator_rule='"," space' + ) + + ' "]" space', + ) - elif schema_type in (None, 'string') and 'pattern' in schema: - return self._visit_pattern(schema['pattern'], rule_name) + elif schema_type in (None, "string") and "pattern" in schema: + return self._visit_pattern(schema["pattern"], rule_name) - elif schema_type in (None, 'string') and re.match(r'^uuid[1-5]?$', schema_format or ''): + elif schema_type in (None, "string") and re.match( + r"^uuid[1-5]?$", schema_format or "" + ): return self._add_primitive( - 'root' if rule_name == 'root' else schema_format, - PRIMITIVE_RULES['uuid'] + "root" if rule_name == "root" else schema_format, + PRIMITIVE_RULES["uuid"], ) - elif schema_type in (None, 'string') and f'{schema_format}-string' in STRING_FORMAT_RULES: - prim_name = f'{schema_format}-string' - return self._add_rule(rule_name, self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name])) - - elif schema_type == 'string' and ('minLength' in schema or 'maxLength' in schema): - char_rule = self._add_primitive('char', PRIMITIVE_RULES['char']) - min_len = schema.get('minLength', 0) - max_len = schema.get('maxLength') + elif ( + schema_type in (None, "string") + and f"{schema_format}-string" in STRING_FORMAT_RULES + ): + prim_name = f"{schema_format}-string" + return self._add_rule( + rule_name, + self._add_primitive(prim_name, STRING_FORMAT_RULES[prim_name]), + ) - return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\"" space') + elif schema_type == "string" and ( + "minLength" in schema or "maxLength" in schema + ): + char_rule = self._add_primitive("char", PRIMITIVE_RULES["char"]) + min_len = schema.get("minLength", 0) + max_len = schema.get("maxLength") + + return self._add_rule( + rule_name, + r'"\"" ' + + _build_repetition(char_rule, min_len, max_len) + + r' "\"" space', + ) - elif (schema_type == 'object') or (len(schema) == 0): - return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object'])) + elif (schema_type == "object") or (len(schema) == 0): + return self._add_rule( + rule_name, self._add_primitive("object", PRIMITIVE_RULES["object"]) + ) else: - assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}' + assert schema_type in PRIMITIVE_RULES, f"Unrecognized schema: {schema}" # TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero - return self._add_primitive('root' if rule_name == 'root' else schema_type, PRIMITIVE_RULES[schema_type]) + return self._add_primitive( + "root" if rule_name == "root" else schema_type, + PRIMITIVE_RULES[schema_type], + ) def _add_primitive(self, name: str, rule: BuiltinRule): n = self._add_rule(name, rule.content) for dep in rule.deps: dep_rule = PRIMITIVE_RULES.get(dep) or STRING_FORMAT_RULES.get(dep) - assert dep_rule, f'Rule {dep} not known' + assert dep_rule, f"Rule {dep} not known" if dep not in self._rules: self._add_primitive(dep, dep_rule) return n - def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[str], name: str, additional_properties: Union[bool, Any]): + def _build_object_rule( + self, + properties: List[Tuple[str, Any]], + required: Set[str], + name: str, + additional_properties: Union[bool, Any], + ): prop_order = self._prop_order # sort by position in prop_order (if specified) then by original order - sorted_props = [kv[0] for _, kv in sorted(enumerate(properties), key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]))] + sorted_props = [ + kv[0] + for _, kv in sorted( + enumerate(properties), + key=lambda ikv: (prop_order.get(ikv[1][0], len(prop_order)), ikv[0]), + ) + ] prop_kv_rule_names = {} for prop_name, prop_schema in properties: - prop_rule_name = self.visit(prop_schema, f'{name}{"-" if name else ""}{prop_name}') + prop_rule_name = self.visit( + prop_schema, f'{name}{"-" if name else ""}{prop_name}' + ) prop_kv_rule_names[prop_name] = self._add_rule( f'{name}{"-" if name else ""}{prop_name}-kv', - fr'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}' + rf'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}', ) required_props = [k for k in sorted_props if k in required] optional_props = [k for k in sorted_props if k not in required] if additional_properties == True or isinstance(additional_properties, dict): sub_name = f'{name}{"-" if name else ""}additional' - value_rule = self.visit({} if additional_properties == True else additional_properties, f'{sub_name}-value') + value_rule = self.visit( + {} if additional_properties == True else additional_properties, + f"{sub_name}-value", + ) prop_kv_rule_names["*"] = self._add_rule( - f'{sub_name}-kv', - self._add_primitive('string', PRIMITIVE_RULES['string']) + f' ":" space {value_rule}' + f"{sub_name}-kv", + self._add_primitive("string", PRIMITIVE_RULES["string"]) + + f' ":" space {value_rule}', ) optional_props.append("*") @@ -1896,51 +2040,55 @@ def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[st rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props) if optional_props: - rule += ' (' + rule += " (" if required_props: rule += ' "," space ( ' def get_recursive_refs(ks, first_is_optional): [k, *rest] = ks kv_rule_name = prop_kv_rule_names[k] - if k == '*': + if k == "*": res = self._add_rule( f'{name}{"-" if name else ""}additional-kvs', - f'{kv_rule_name} ( "," space ' + kv_rule_name + ' )*' + f'{kv_rule_name} ( "," space ' + kv_rule_name + " )*", ) elif first_is_optional: res = f'( "," space {kv_rule_name} )?' else: res = kv_rule_name if len(rest) > 0: - res += ' ' + self._add_rule( + res += " " + self._add_rule( f'{name}{"-" if name else ""}{k}-rest', - get_recursive_refs(rest, first_is_optional=True) + get_recursive_refs(rest, first_is_optional=True), ) return res - rule += ' | '.join( + rule += " | ".join( get_recursive_refs(optional_props[i:], first_is_optional=False) for i in range(len(optional_props)) ) if required_props: - rule += ' )' - rule += ' )?' + rule += " )" + rule += " )?" rule += ' "}" space' return rule def format_grammar(self): - return '\n'.join( - f'{name} ::= {rule}' + return "\n".join( + f"{name} ::= {rule}" for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0]) ) + + def json_schema_to_gbnf(schema: str, prop_order: Optional[List[str]] = None): prop_order = prop_order or [] schema = json.loads(schema) prop_order = {name: idx for idx, name in enumerate(prop_order)} - converter = SchemaConverter(prop_order=prop_order, allow_fetch=False, dotall=False, raw_pattern=False) + converter = SchemaConverter( + prop_order=prop_order, allow_fetch=False, dotall=False, raw_pattern=False + ) schema = converter.resolve_refs(schema, "stdin") converter.visit(schema, "") return converter.format_grammar() diff --git a/llama_cpp/llama_tokenizer.py b/llama_cpp/llama_tokenizer.py index 5a8b13b7a5..029bf2acc1 100644 --- a/llama_cpp/llama_tokenizer.py +++ b/llama_cpp/llama_tokenizer.py @@ -17,7 +17,7 @@ def tokenize( self, text: bytes, add_bos: bool = True, special: bool = True ) -> List[int]: """Tokenize the text into tokens. - + Args: text: The text to tokenize. add_bos: Whether to add a beginning of sequence token. @@ -29,10 +29,11 @@ def detokenize( self, tokens: List[int], prev_tokens: Optional[List[int]] = None ) -> bytes: """Detokenize the tokens into text. - + Args: tokens: The tokens to detokenize. - prev_tokens: If tokens is a continuation of a previous sequence, the previous tokens.""" + prev_tokens: If tokens is a continuation of a previous sequence, the previous tokens. + """ raise NotImplementedError @@ -80,7 +81,9 @@ def detokenize( self, tokens: List[int], prev_tokens: Optional[List[int]] = None ) -> bytes: if prev_tokens is not None: - text = self.hf_tokenizer.decode(prev_tokens + tokens).encode("utf-8", errors="ignore") + text = self.hf_tokenizer.decode(prev_tokens + tokens).encode( + "utf-8", errors="ignore" + ) prev_text = self.hf_tokenizer.decode(prev_tokens).encode( "utf-8", errors="ignore" ) diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 4677785ae1..573d7d651f 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -6,6 +6,7 @@ https://github.com/openai/openai-openapi/blob/master/openapi.yaml """ + from typing import Any, List, Optional, Dict, Union from typing_extensions import TypedDict, NotRequired, Literal @@ -156,7 +157,9 @@ class ChatCompletionFunctionCallOption(TypedDict): class ChatCompletionRequestResponseFormat(TypedDict): type: Literal["text", "json_object"] - schema: NotRequired[JsonType] # https://docs.endpoints.anyscale.com/guides/json_mode/ + schema: NotRequired[ + JsonType + ] # https://docs.endpoints.anyscale.com/guides/json_mode/ class ChatCompletionRequestMessageContentPartText(TypedDict): diff --git a/llama_cpp/server/__main__.py b/llama_cpp/server/__main__.py index a6f1f4e9ca..bbac4957e9 100644 --- a/llama_cpp/server/__main__.py +++ b/llama_cpp/server/__main__.py @@ -21,6 +21,7 @@ Then visit http://localhost:8000/docs to see the interactive API docs. """ + from __future__ import annotations import os @@ -68,7 +69,9 @@ def main(): json.dumps(yaml.safe_load(f)) ) else: - config_file_settings = ConfigFileSettings.model_validate_json(f.read()) + config_file_settings = ConfigFileSettings.model_validate_json( + f.read() + ) server_settings = ServerSettings.model_validate(config_file_settings) model_settings = config_file_settings.models else: diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 8fd41a3fc7..cd3255176f 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -162,7 +162,9 @@ async def get_event_publisher( on_complete: typing.Optional[typing.Callable[[], None]] = None, ): server_settings = next(get_server_settings()) - interrupt_requests = server_settings.interrupt_requests if server_settings else False + interrupt_requests = ( + server_settings.interrupt_requests if server_settings else False + ) async with inner_send_chan: try: async for chunk in iterate_in_threadpool(iterator): diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 9d9b533d52..b206558133 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -182,15 +182,17 @@ class ModelSettings(BaseSettings): default=True, description="Whether to print debug information." ) - @model_validator(mode="before") # pre=True to ensure this runs before any other validation + @model_validator( + mode="before" + ) # pre=True to ensure this runs before any other validation def set_dynamic_defaults(self) -> Self: # If n_threads or n_threads_batch is -1, set it to multiprocessing.cpu_count() cpu_count = multiprocessing.cpu_count() values = cast(Dict[str, int], self) - if values.get('n_threads', 0) == -1: - values['n_threads'] = cpu_count - if values.get('n_threads_batch', 0) == -1: - values['n_threads_batch'] = cpu_count + if values.get("n_threads", 0) == -1: + values["n_threads"] = cpu_count + if values.get("n_threads_batch", 0) == -1: + values["n_threads_batch"] = cpu_count return self diff --git a/llama_cpp/server/types.py b/llama_cpp/server/types.py index a75f9e55b9..fdd1644568 100644 --- a/llama_cpp/server/types.py +++ b/llama_cpp/server/types.py @@ -216,7 +216,7 @@ class CreateChatCompletionRequest(BaseModel): min_tokens: int = min_tokens_field logprobs: Optional[bool] = Field( default=False, - description="Whether to output the logprobs or not. Default is True" + description="Whether to output the logprobs or not. Default is True", ) top_logprobs: Optional[int] = Field( default=None, From 08f2bb36fefdc63559da12707ee830097447f29a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 9 Jul 2024 14:06:46 -0400 Subject: [PATCH 248/561] fix(minor): Minor ruff fixes --- llama_cpp/_internals.py | 1 + llama_cpp/llama.py | 8 +++----- llama_cpp/llama_chat_format.py | 9 +++++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index ff378739f6..d5d3b2179e 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -4,6 +4,7 @@ import ctypes from typing import ( + Dict, List, Optional, Sequence, diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 3669efdec6..d8c2e0cddc 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -11,10 +11,11 @@ import warnings import contextlib import multiprocessing -from types import TracebackType from typing import ( + Any, List, + Literal, Optional, Union, Generator, @@ -23,14 +24,11 @@ Deque, Callable, Dict, - Type, ) from collections import deque from pathlib import Path -from llama_cpp.llama_types import List - from .llama_types import * from .llama_grammar import LlamaGrammar from .llama_cache import ( @@ -901,7 +899,7 @@ def embed( pooling_type = self.pooling_type() logits_all = pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE - if self.context_params.embeddings == False: + if self.context_params.embeddings is False: raise RuntimeError( "Llama model must be created with embedding=True to call this method" ) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 78d83ba866..ea8d07febe 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import sys import json import ctypes import dataclasses @@ -627,6 +628,8 @@ def chat_completion_handler( json.dumps(schema), verbose=llama.verbose ) except Exception as e: + if llama.verbose: + print(str(e), file=sys.stderr) grammar = llama_grammar.LlamaGrammar.from_string( llama_grammar.JSON_GBNF, verbose=llama.verbose ) @@ -1611,12 +1614,12 @@ def message_to_str(msg: llama_types.ChatCompletionRequestMessage): function_call = completion_text.split(".")[-1][:-1] new_prompt = prompt + completion_text + stop elif isinstance(function_call, str) and function_call != "none": - new_prompt = prompt + f":\n" + new_prompt = prompt + ":\n" elif isinstance(function_call, dict): new_prompt = prompt + f" to=functions.{function_call['name']}:\n" function_call = function_call["name"] else: - new_prompt = prompt + f":\n" + new_prompt = prompt + ":\n" function_body = None for function in functions or []: @@ -2871,6 +2874,8 @@ def embed_image_bytes(image_bytes: bytes): json.dumps(schema), verbose=llama.verbose ) except Exception as e: + if llama.verbose: + print(str(e), file=sys.stderr) grammar = llama_grammar.LlamaGrammar.from_string( llama_grammar.JSON_GBNF, verbose=llama.verbose ) From f7f4fa8782ddb965ba394bfa7b7b42d1fabc7482 Mon Sep 17 00:00:00 2001 From: yentur <63314411+yentur@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:24:06 +0300 Subject: [PATCH 249/561] feat(ci): Update simple Dockerfile (#1459) Co-authored-by: omer.yentur Co-authored-by: Andrei --- docker/simple/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/simple/Dockerfile b/docker/simple/Dockerfile index 97a6c19df9..2838fd1ff7 100644 --- a/docker/simple/Dockerfile +++ b/docker/simple/Dockerfile @@ -23,7 +23,9 @@ COPY . /app RUN python3 -m pip install --upgrade pip -RUN make deps && make build && make clean +RUN python3 -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context + +RUN pip install llama-cpp-python --verbose; # Set environment variable for the host ENV HOST=0.0.0.0 @@ -33,4 +35,4 @@ ENV PORT=8000 EXPOSE 8000 # Run the server start script -CMD ["/bin/sh", "/app/docker/simple/run.sh"] +CMD ["/bin/sh", "/app/run.sh"] From 7613d23db7bc3cfce7d4123f220dcfa6657236ee Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 17 Jul 2024 18:06:58 -0400 Subject: [PATCH 250/561] feat: Update llama.cpp --- llama_cpp/llama.py | 23 ++++++--- llama_cpp/llama_cpp.py | 114 ++++++++++++++++++++++++++++------------- vendor/llama.cpp | 2 +- 3 files changed, 95 insertions(+), 44 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index d8c2e0cddc..4b6f6b8f4a 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -408,15 +408,24 @@ def __init__( ) ) + self._lora_adapter: Optional[llama_cpp.llama_lora_adapter_p] = None + if self.lora_path: - if self._model.apply_lora_from_file( - self.lora_path, - self.lora_scale, - self.lora_base, - self.n_threads, + assert self._model.model is not None + self._lora_adapter = llama_cpp.llama_lora_adapter_init( + self._model.model, + self.lora_path.encode("utf-8"), + ) + if self._lora_adapter is None: + raise RuntimeError( + f"Failed to initialize LoRA adapter from lora path: {self.lora_path}" + ) + assert self._ctx.ctx is not None + if llama_cpp.llama_lora_adapter_set( + self._ctx.ctx, self._lora_adapter, self.lora_scale ): raise RuntimeError( - f"Failed to apply LoRA from lora path: {self.lora_path} to base path: {self.lora_base}" + f"Failed to set LoRA adapter from lora path: {self.lora_path}" ) if self.verbose: @@ -2077,6 +2086,8 @@ def close(self) -> None: self._stack.close() def __del__(self) -> None: + if self._lora_adapter is not None: + llama_cpp.llama_lora_adapter_free(self._lora_adapter) self.close() @staticmethod diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index decc08e657..35dc030b17 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -401,7 +401,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors -# LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 +# // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 # // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed # // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed # LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors @@ -430,14 +430,16 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors # LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors # LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors - +# LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // except 1d tensors +# # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; LLAMA_FTYPE_ALL_F32 = 0 LLAMA_FTYPE_MOSTLY_F16 = 1 LLAMA_FTYPE_MOSTLY_Q4_0 = 2 LLAMA_FTYPE_MOSTLY_Q4_1 = 3 -LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4 LLAMA_FTYPE_MOSTLY_Q8_0 = 7 LLAMA_FTYPE_MOSTLY_Q5_0 = 8 LLAMA_FTYPE_MOSTLY_Q5_1 = 9 @@ -464,6 +466,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_FTYPE_MOSTLY_IQ4_XS = 30 LLAMA_FTYPE_MOSTLY_IQ1_M = 31 LLAMA_FTYPE_MOSTLY_BF16 = 32 +LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 +LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 +LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { @@ -1100,6 +1105,12 @@ class llama_chat_message(ctypes.Structure): ] +# // lora adapter +# struct llama_lora_adapter; +llama_lora_adapter_p = ctypes.c_void_p +llama_lora_adapter_p_ctypes = ctypes.POINTER(ctypes.c_void_p) + + # // Helpers for getting default parameters # LLAMA_API struct llama_model_params llama_model_default_params(void); @ctypes_function( @@ -1507,43 +1518,72 @@ def llama_model_quantize( ... -# // Apply a LoRA adapter to a loaded model -# // path_base_model is the path to a higher quality model to use as a base for -# // the layers modified by the adapter. Can be NULL to use the current loaded model. -# // The model needs to be reloaded before applying a new adapter, otherwise the adapter -# // will be applied on top of the previous one -# // Returns 0 on success -# LLAMA_API int32_t llama_model_apply_lora_from_file( -# const struct llama_model * model, -# const char * path_lora, -# float scale, -# const char * path_base_model, -# int32_t n_threads); -@ctypes_function( - "llama_model_apply_lora_from_file", - [ - llama_model_p_ctypes, - ctypes.c_char_p, - ctypes.c_float, - ctypes.c_char_p, - ctypes.c_int32, - ], +# // Load a LoRA adapter from file +# // The loaded adapter will be associated to the given model, and will be free when the model is deleted +# LLAMA_API struct llama_lora_adapter * llama_lora_adapter_init( +# struct llama_model * model, +# const char * path_lora); +@ctypes_function( + "llama_lora_adapter_init", + [llama_model_p_ctypes, ctypes.c_char_p], + llama_lora_adapter_p_ctypes, +) +def llama_lora_adapter_init( + model: llama_model_p, path_lora: bytes, / +) -> Optional[llama_lora_adapter_p]: + """Load a LoRA adapter from file + The loaded adapter will be associated to the given model, and will be free when the model is deleted""" + ... + + +# // Add a loaded LoRA adapter to given context +# // This will not modify model's weight +# LLAMA_API int32_t llama_lora_adapter_set( +# struct llama_context * ctx, +# struct llama_lora_adapter * adapter, +# float scale); +@ctypes_function( + "llama_lora_adapter_set", + [llama_context_p_ctypes, llama_lora_adapter_p_ctypes, ctypes.c_float], ctypes.c_int32, ) -def llama_model_apply_lora_from_file( - model: llama_model_p, - path_lora: Union[ctypes.c_char_p, bytes], - scale: Union[ctypes.c_float, float], - path_base_model: Union[ctypes.c_char_p, bytes, None], - n_threads: Union[ctypes.c_int32, int], - /, +def llama_lora_adapter_set( + ctx: llama_context_p, adapter: llama_lora_adapter_p, scale: float, / +) -> int: + """Add a loaded LoRA adapter to given context + This will not modify model's weight""" + ... + + +# // Remove a LoRA adapter from given context +# // Return -1 if the adapter is not present in the context +# LLAMA_API int32_t llama_lora_adapter_remove( +# struct llama_context * ctx, +# struct llama_lora_adapter * adapter); +@ctypes_function( + "llama_lora_adapter_remove", + [llama_context_p_ctypes, llama_lora_adapter_p_ctypes], + ctypes.c_int32, +) +def llama_lora_adapter_remove( + ctx: llama_context_p, adapter: llama_lora_adapter_p, / ) -> int: - """Apply a LoRA adapter to a loaded model - path_base_model is the path to a higher quality model to use as a base for - the layers modified by the adapter. Can be NULL to use the current loaded model. - The model needs to be reloaded before applying a new adapter, otherwise the adapter - will be applied on top of the previous one - Returns 0 on success""" + """Remove a LoRA adapter from given context + Return -1 if the adapter is not present in the context""" + ... + + +# // Manually free a LoRA adapter +# // Note: loaded adapters will be free when the associated model is deleted +# LLAMA_API void llama_lora_adapter_free(struct llama_lora_adapter * adapter); +@ctypes_function( + "llama_lora_adapter_free", + [llama_lora_adapter_p_ctypes], + None, +) +def llama_lora_adapter_free(adapter: llama_lora_adapter_p, /): + """Manually free a LoRA adapter + Note: loaded adapters will be free when the associated model is deleted""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7fdb6f73e3..30f80ca0bc 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7fdb6f73e35605c8dbc39e9f19cd9ed84dbc87f2 +Subproject commit 30f80ca0bcee58669ada7a94244eeccc8c4807cc From 66d5cddbc31976636cf560e5b0b86e4333e85f2b Mon Sep 17 00:00:00 2001 From: Grider Date: Wed, 17 Jul 2024 23:09:28 +0100 Subject: [PATCH 251/561] fix(server): Use split_mode from model settings (#1594) Co-authored-by: Andrei --- llama_cpp/server/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index ad39c1004b..c486f88852 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -223,6 +223,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: **kwargs, # Model Params n_gpu_layers=settings.n_gpu_layers, + split_mode=settings.split_mode, main_gpu=settings.main_gpu, tensor_split=settings.tensor_split, vocab_only=settings.vocab_only, From 797f54c3171a8aa56b83459651ca8ecc7b3bdf26 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Wed, 17 Jul 2024 23:09:55 +0100 Subject: [PATCH 252/561] fix(docs): Update README.md typo (#1589) Missing ll --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5641ccaa8f..b0dfdd5b5c 100644 --- a/README.md +++ b/README.md @@ -755,7 +755,7 @@ pip install -e .[all] make clean ``` -You can also test out specific commits of `lama.cpp` by checking out the desired commit in the `vendor/llama.cpp` submodule and then running `make clean` and `pip install -e .` again. Any changes in the `llama.h` API will require +You can also test out specific commits of `llama.cpp` by checking out the desired commit in the `vendor/llama.cpp` submodule and then running `make clean` and `pip install -e .` again. Any changes in the `llama.h` API will require changes to the `llama_cpp/llama_cpp.py` file to match the new API (additional changes may be required elsewhere). ## FAQ From 070047662fe6261a32eaae0b380b52119d9af54e Mon Sep 17 00:00:00 2001 From: ddh0 Date: Wed, 17 Jul 2024 20:51:16 -0500 Subject: [PATCH 253/561] fix: Change repeat_penalty to 1.0 to match llama.cpp defaults (#1590) * reapply changes after sync with main branch * change repeat_penalty to 1.0 to match llama.cpp defaults --------- Co-authored-by: Andrei --- llama_cpp/_internals.py | 2 +- llama_cpp/llama.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index d5d3b2179e..dcd4e17ff7 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -721,7 +721,7 @@ class _LlamaSamplingParams: typical_p: float = 1.00 temp: float = 0.80 penalty_last_n: int = 64 - penalty_repeat: float = 1.10 + penalty_repeat: float = 1.0 penalty_freq: float = 0.00 penalty_present: float = 0.00 mirostat: int = 0 diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 4b6f6b8f4a..005045f5c4 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -658,7 +658,7 @@ def sample( min_p: float = 0.05, typical_p: float = 1.0, temp: float = 0.80, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, frequency_penalty: float = 0.0, presence_penalty: float = 0.0, tfs_z: float = 1.0, @@ -733,7 +733,7 @@ def generate( min_p: float = 0.05, typical_p: float = 1.0, temp: float = 0.80, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, reset: bool = True, frequency_penalty: float = 0.0, presence_penalty: float = 0.0, @@ -751,7 +751,7 @@ def generate( Examples: >>> llama = Llama("models/ggml-7b.bin") >>> tokens = llama.tokenize(b"Hello, world!") - >>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.1): + >>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.0): ... print(llama.detokenize([token])) Args: @@ -1020,7 +1020,7 @@ def _create_completion( stop: Optional[Union[str, List[str]]] = [], frequency_penalty: float = 0.0, presence_penalty: float = 0.0, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, top_k: int = 40, stream: bool = False, seed: Optional[int] = None, @@ -1639,7 +1639,7 @@ def create_completion( stop: Optional[Union[str, List[str]]] = [], frequency_penalty: float = 0.0, presence_penalty: float = 0.0, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, top_k: int = 40, stream: bool = False, seed: Optional[int] = None, @@ -1736,7 +1736,7 @@ def __call__( stop: Optional[Union[str, List[str]]] = [], frequency_penalty: float = 0.0, presence_penalty: float = 0.0, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, top_k: int = 40, stream: bool = False, seed: Optional[int] = None, @@ -1833,7 +1833,7 @@ def create_chat_completion( max_tokens: Optional[int] = None, presence_penalty: float = 0.0, frequency_penalty: float = 0.0, - repeat_penalty: float = 1.1, + repeat_penalty: float = 1.0, tfs_z: float = 1.0, mirostat_mode: int = 0, mirostat_tau: float = 5.0, From 3638f7333b8ece31c61f83b0cbbfc6c1dfe06814 Mon Sep 17 00:00:00 2001 From: Michael Schock Date: Wed, 17 Jul 2024 18:52:04 -0700 Subject: [PATCH 254/561] feat: Add 'required' literal to ChatCompletionToolChoiceOption (#1597) --- llama_cpp/llama_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 573d7d651f..bbb58afc35 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -275,7 +275,7 @@ class ChatCompletionNamedToolChoice(TypedDict): ChatCompletionToolChoiceOption = Union[ - Literal["none", "auto"], ChatCompletionNamedToolChoice + Literal["none", "auto", "required"], ChatCompletionNamedToolChoice ] From f95057a4750a14de28bcf05b7dd380475f9f9c20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jul 2024 17:16:13 -0400 Subject: [PATCH 255/561] chore(deps): bump microsoft/setup-msbuild from 1.3 to 2 (#1585) Bumps [microsoft/setup-msbuild](https://github.com/microsoft/setup-msbuild) from 1.3 to 2. - [Release notes](https://github.com/microsoft/setup-msbuild/releases) - [Changelog](https://github.com/microsoft/setup-msbuild/blob/main/building-release.md) - [Commits](https://github.com/microsoft/setup-msbuild/compare/v1.3...v2) --- updated-dependencies: - dependency-name: microsoft/setup-msbuild dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 3c02670958..0733a68c5b 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -45,7 +45,7 @@ jobs: steps: - name: Add MSBuild to PATH if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v1.3 + uses: microsoft/setup-msbuild@v2 with: vs-version: '[16.11,16.12)' From 5105f400f32d48b2eb4d8bac563072fb38912115 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 22 Jul 2024 15:20:12 -0400 Subject: [PATCH 256/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 10 ++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 35dc030b17..881b41b51f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -247,8 +247,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 6 -LLAMA_SESSION_VERSION = 6 +# define LLAMA_SESSION_VERSION 7 +LLAMA_SESSION_VERSION = 7 # define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ @@ -314,6 +314,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, # LLAMA_VOCAB_PRE_TYPE_VIKING = 18, # LLAMA_VOCAB_PRE_TYPE_JAIS = 19, +# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, +# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, +# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -335,6 +338,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17 LLAMA_VOCAB_PRE_TYPE_VIKING = 18 LLAMA_VOCAB_PRE_TYPE_JAIS = 19 +LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20 +LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21 +LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 30f80ca0bc..081fe431aa 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 30f80ca0bcee58669ada7a94244eeccc8c4807cc +Subproject commit 081fe431aa8fb6307145c4feb3eed4f48cab19f8 From 816d4912d9d2971198d2300a840ce4c100152502 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 22 Jul 2024 15:26:00 -0400 Subject: [PATCH 257/561] chore: Bump version --- CHANGELOG.md | 9 +++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47077d8af8..d71e1609e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.83] + +- feat: Update llama.cpp to ggerganov/llama.cpp@081fe431aa8fb6307145c4feb3eed4f48cab19f8 +- feat: Add 'required' literal to ChatCompletionToolChoiceOption by @mjschock in #1597 +- fix: Change repeat_penalty to 1.0 to match llama.cpp defaults by @ddh0 in #1590 +- fix(docs): Update README.md typo by @ericcurtin in #1589 +- fix(server): Use split_mode from model settings by @grider-withourai in #1594 +- feat(ci): Dockerfile update base images and post-install cleanup by @Smartappli in #1530 + ## [0.2.82] - feat: Update llama.cpp to ggerganov/llama.cpp@7fdb6f73e35605c8dbc39e9f19cd9ed84dbc87f2 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 09ffb594e4..65364d8782 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.82" +__version__ = "0.2.83" From a14b49d64e760183504e1a5b37f4c9beb36a20ff Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 23 Jul 2024 21:26:20 -0400 Subject: [PATCH 258/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 234 ++++++++++++++++++++++++++--------------- vendor/llama.cpp | 2 +- 2 files changed, 153 insertions(+), 83 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 881b41b51f..e70f2832ea 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1192,7 +1192,8 @@ def llama_backend_init(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): ... +def llama_numa_init(numa: int, /): + ... # // Call once at the end of the program - currently only used for MPI @@ -1217,7 +1218,8 @@ def llama_backend_free(): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: ... +) -> Optional[llama_model_p]: + ... # LLAMA_API void llama_free_model(struct llama_model * model); @@ -1226,7 +1228,8 @@ def llama_load_model_from_file( [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): ... +def llama_free_model(model: llama_model_p, /): + ... # LLAMA_API struct llama_context * llama_new_context_with_model( @@ -1239,7 +1242,8 @@ def llama_free_model(model: llama_model_p, /): ... ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: ... +) -> Optional[llama_context_p]: + ... # // Frees all allocated memory @@ -1260,87 +1264,104 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: ... +def llama_time_us() -> int: + ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: ... +def llama_max_devices() -> int: + ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: ... +def llama_supports_mmap() -> bool: + ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: ... +def llama_supports_mlock() -> bool: + ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: ... +def llama_supports_gpu_offload() -> bool: + ... # LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: + ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: ... +def llama_n_batch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); @ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... +def llama_n_ubatch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); @ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... +def llama_n_seq_max(ctx: llama_context_p, /) -> int: + ... # LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) -def llama_pooling_type(ctx: llama_context_p, /) -> int: ... +def llama_pooling_type(ctx: llama_context_p, /) -> int: + ... # LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: ... +def llama_vocab_type(model: llama_model_p, /) -> int: + ... # LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: ... +def llama_rope_type(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: ... +def llama_n_vocab(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_embd (const struct llama_model * model); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: ... +def llama_n_embd(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_layer (const struct llama_model * model); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: ... +def llama_n_layer(model: llama_model_p, /) -> int: + ... # // Get the model's RoPE frequency scaling factor @@ -1538,7 +1559,8 @@ def llama_lora_adapter_init( model: llama_model_p, path_lora: bytes, / ) -> Optional[llama_lora_adapter_p]: """Load a LoRA adapter from file - The loaded adapter will be associated to the given model, and will be free when the model is deleted""" + The loaded adapter will be associated to the given model, and will be free when the model is deleted + """ ... @@ -2092,7 +2114,8 @@ def llama_state_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_load_session_file( @@ -2120,7 +2143,8 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: ... +) -> int: + ... # LLAMA_API bool llama_state_save_file( @@ -2144,7 +2168,8 @@ def llama_state_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_save_session_file( @@ -2169,7 +2194,8 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: ... +) -> int: + ... # // Get the exact size needed to copy the KV cache of a single sequence @@ -2247,7 +2273,8 @@ def llama_state_seq_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: ... +) -> int: + ... # LLAMA_API size_t llama_state_seq_load_file( @@ -2277,7 +2304,8 @@ def llama_state_seq_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: ... +) -> int: + ... # // @@ -2582,7 +2610,8 @@ def llama_get_embeddings_seq( ) def llama_token_get_text( model: llama_model_p, token: Union[llama_token, int], / -) -> bytes: ... +) -> bytes: + ... # LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); @@ -2591,7 +2620,8 @@ def llama_token_get_text( ) def llama_token_get_score( model: llama_model_p, token: Union[llama_token, int], / -) -> float: ... +) -> float: + ... # LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token); @@ -2600,7 +2630,8 @@ def llama_token_get_score( ) def llama_token_get_attr( model: llama_model_p, token: Union[llama_token, int], / -) -> int: ... +) -> int: + ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) @@ -2664,7 +2695,7 @@ def llama_token_nl(model: llama_model_p, /) -> int: # // Returns -1 if unknown, 1 for true or 0 for false. -# LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model); +# LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model); @ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_int32) def llama_add_bos_token(model: llama_model_p, /) -> int: """Returns -1 if unknown, 1 for true or 0 for false.""" @@ -2672,7 +2703,7 @@ def llama_add_bos_token(model: llama_model_p, /) -> int: # // Returns -1 if unknown, 1 for true or 0 for false. -# LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model); +# LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model); @ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_int32) def llama_add_eos_token(model: llama_model_p, /) -> int: """Returns -1 if unknown, 1 for true or 0 for false.""" @@ -2689,17 +2720,20 @@ def llama_token_prefix(model: llama_model_p) -> int: # LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: ... +def llama_token_middle(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: ... +def llama_token_suffix(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle @ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: ... +def llama_token_eot(model: llama_model_p, /) -> int: + ... # // @@ -2861,6 +2895,11 @@ def llama_detokenize( ... +# // +# // Chat templates +# // + + # /// Apply chat template. Inspired by hf apply_chat_template() on python. # /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model" # /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template @@ -2895,7 +2934,8 @@ def llama_chat_apply_template( chat: CtypesArray[llama_chat_message], n_msg: int, /, -) -> int: ... +) -> int: + ... # // @@ -2950,6 +2990,79 @@ def llama_grammar_copy(grammar: llama_grammar_p, /) -> llama_grammar_p: ... +# /// @details Apply constraints from grammar +# LLAMA_API void llama_grammar_sample( +# const struct llama_grammar * grammar, +# const struct llama_context * ctx, +# llama_token_data_array * candidates); +@ctypes_function( + "llama_grammar_sample", + [ + llama_grammar_p, + llama_context_p_ctypes, + llama_token_data_array_p, + ], + None, +) +def llama_grammar_sample( + grammar: llama_grammar_p, + ctx: llama_context_p, + candidates: Union[ + CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] + ], + /, +): + """Apply constraints from grammar""" + ... + + +# LLAMA_API DEPRECATED(void llama_sample_grammar( +# struct llama_context * ctx, +# llama_token_data_array * candidates, +# const struct llama_grammar * grammar), +# "use llama_grammar_sample instead"); +@ctypes_function( + "llama_sample_grammar", + [llama_context_p_ctypes, llama_token_data_array_p, llama_grammar_p], + None, +) +def llama_sample_grammar( + ctx: llama_context_p, + candidates: Union[ + CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] + ], + grammar, # type: llama_grammar_p + /, +): + """Apply constraints from grammar + + Parameters: + candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + grammar: A grammar object containing the rules and constraints to apply to the generated text. + """ + ... + + +# /// @details Accepts the sampled token into the grammar +# LLAMA_API void llama_grammar_accept_token( +# struct llama_grammar * grammar, +# struct llama_context * ctx, +# llama_token token); +@ctypes_function( + "llama_grammar_accept_token", + [llama_grammar_p, llama_context_p_ctypes, llama_token], + None, +) +def llama_grammar_accept_token( + grammar: llama_grammar_p, + ctx: llama_context_p, + token: Union[llama_token, int], + /, +): + """Accepts the sampled token into the grammar""" + ... + + # // # // Sampling functions # // @@ -3236,33 +3349,6 @@ def llama_sample_temp( ... -# /// @details Apply constraints from grammar -# LLAMA_API void llama_sample_grammar( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# const struct llama_grammar * grammar); -@ctypes_function( - "llama_sample_grammar", - [llama_context_p_ctypes, llama_token_data_array_p, llama_grammar_p], - None, -) -def llama_sample_grammar( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - grammar, # type: llama_grammar_p - /, -): - """Apply constraints from grammar - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - grammar: A grammar object containing the rules and constraints to apply to the generated text. - """ - ... - - # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. # /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. # /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. @@ -3395,23 +3481,6 @@ def llama_sample_token( ... -# /// @details Accepts the sampled token into the grammar -# LLAMA_API void llama_grammar_accept_token( -# struct llama_context * ctx, -# struct llama_grammar * grammar, -# llama_token token); -@ctypes_function( - "llama_grammar_accept_token", - [llama_context_p_ctypes, llama_grammar_p, llama_token], - None, -) -def llama_grammar_accept_token( - ctx: llama_context_p, grammar: llama_grammar_p, token: Union[llama_token, int], / -) -> None: - """Accepts the sampled token into the grammar""" - ... - - # // # // Model split # // @@ -3533,4 +3602,5 @@ def llama_log_set( [ctypes.c_void_p, llama_context_p_ctypes], None, ) -def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): ... +def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): + ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 081fe431aa..b841d07408 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 081fe431aa8fb6307145c4feb3eed4f48cab19f8 +Subproject commit b841d0740855c5af1344a81f261139a45a2b39ee From dccb148580e4794879cd66ff6c5bea6af95a66e2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 28 Jul 2024 11:58:22 -0400 Subject: [PATCH 259/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 91 ++++++++++++++++++++++++++++++------------ vendor/llama.cpp | 2 +- 2 files changed, 66 insertions(+), 27 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index e70f2832ea..d563d77bc0 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -233,9 +233,6 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_DEFAULT_SEED 0xFFFFFFFF LLAMA_DEFAULT_SEED = 0xFFFFFFFF -# define LLAMA_MAX_RNG_STATE (64*1024) -LLAMA_MAX_RNG_STATE = 64 * 1024 - # define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' LLAMA_FILE_MAGIC_GGLA = 0x67676C61 @@ -247,13 +244,13 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 7 -LLAMA_SESSION_VERSION = 7 +# define LLAMA_SESSION_VERSION 8 +LLAMA_SESSION_VERSION = 8 # define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ -# define LLAMA_STATE_SEQ_VERSION 1 -LLAMA_STATE_SEQ_VERSION = 1 +# define LLAMA_STATE_SEQ_VERSION 2 +LLAMA_STATE_SEQ_VERSION = 2 # struct llama_model; llama_model_p = NewType("llama_model_p", int) @@ -1583,7 +1580,7 @@ def llama_lora_adapter_set( ... -# // Remove a LoRA adapter from given context +# // Remove a specific LoRA adapter from given context # // Return -1 if the adapter is not present in the context # LLAMA_API int32_t llama_lora_adapter_remove( # struct llama_context * ctx, @@ -1601,6 +1598,19 @@ def llama_lora_adapter_remove( ... +# // Remove all LoRA adapters from given context +# LLAMA_API void llama_lora_adapter_clear( +# struct llama_context * ctx); +@ctypes_function( + "llama_lora_adapter_clear", + [llama_context_p_ctypes], + None, +) +def llama_lora_adapter_clear(ctx: llama_context_p, /): + """Remove all LoRA adapters from given context""" + ... + + # // Manually free a LoRA adapter # // Note: loaded adapters will be free when the associated model is deleted # LLAMA_API void llama_lora_adapter_free(struct llama_lora_adapter * adapter); @@ -1992,17 +2002,17 @@ def llama_kv_cache_update(ctx: llama_context_p, /): # // -# Returns the maximum size in bytes of the state (rng, logits, embedding -# and kv_cache) - will often be smaller after compacting tokens -# LLAMA_API size_t llama_state_get_size(const struct llama_context * ctx); +# // Returns the *actual* size in bytes of the state +# // (rng, logits, embedding and kv_cache) +# // Only use when saving the state, not when restoring it, otherwise the size may be too small. +# LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); @ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_state_get_size(ctx: llama_context_p, /) -> int: - """Returns the maximum size in bytes of the state (rng, logits, embedding - and kv_cache) - will often be smaller after compacting tokens""" + """Returns the *actual* size in bytes of the state (rng, logits, embedding and kv_cache) - will often be smaller after compacting tokens""" ... -# LLAMA_API DEPRECATED(size_t llama_get_state_size(const struct llama_context * ctx), +# LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx), # "use llama_state_get_size instead"); @ctypes_function("llama_get_state_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_get_state_size(ctx: llama_context_p, /) -> int: @@ -2011,22 +2021,27 @@ def llama_get_state_size(ctx: llama_context_p, /) -> int: ... -# Copies the state to the specified destination address. -# Destination needs to have allocated enough memory. -# Returns the number of bytes copied +# // Copies the state to the specified destination address. +# // Destination needs to have allocated enough memory. +# // Returns the number of bytes copied # LLAMA_API size_t llama_state_get_data( # struct llama_context * ctx, -# uint8_t * dst); +# uint8_t * dst, +# size_t size); @ctypes_function( "llama_state_get_data", [ llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, ], ctypes.c_size_t, ) def llama_state_get_data( - ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], / + ctx: llama_context_p, + dst: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + /, ) -> int: """Copies the state to the specified destination address. Destination needs to have allocated enough memory. @@ -2059,14 +2074,18 @@ def llama_copy_state_data( # // Returns the number of bytes read # LLAMA_API size_t llama_state_set_data( # struct llama_context * ctx, -# const uint8_t * src); +# const uint8_t * src, +# size_t size); @ctypes_function( "llama_state_set_data", - [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8)], + [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t], ctypes.c_size_t, ) def llama_state_set_data( - ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], / + ctx: llama_context_p, + src: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + /, ) -> int: """Set the state reading from the specified address Returns the number of bytes read""" @@ -2216,14 +2235,24 @@ def llama_state_seq_get_size(ctx: llama_context_p, seq_id: llama_seq_id, /) -> i # LLAMA_API size_t llama_state_seq_get_data( # struct llama_context * ctx, # uint8_t * dst, +# size_t size, # llama_seq_id seq_id); @ctypes_function( "llama_state_seq_get_data", - [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), llama_seq_id], + [ + llama_context_p_ctypes, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + llama_seq_id, + ], ctypes.c_size_t, ) def llama_state_seq_get_data( - ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], seq_id: llama_seq_id, / + ctx: llama_context_p, + dst: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + seq_id: llama_seq_id, + /, ) -> int: """Copy the KV cache of a single sequence into the specified buffer""" ... @@ -2236,14 +2265,24 @@ def llama_state_seq_get_data( # LLAMA_API size_t llama_state_seq_set_data( # struct llama_context * ctx, # const uint8_t * src, +# size_t size, # llama_seq_id dest_seq_id); @ctypes_function( "llama_state_seq_set_data", - [llama_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), llama_seq_id], + [ + llama_context_p_ctypes, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + llama_seq_id, + ], ctypes.c_size_t, ) def llama_state_seq_set_data( - ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], dest_seq_id: llama_seq_id, / + ctx: llama_context_p, + src: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + dest_seq_id: llama_seq_id, + /, ) -> int: """Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b841d07408..4730faca61 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b841d0740855c5af1344a81f261139a45a2b39ee +Subproject commit 4730faca618ff9cee0780580145e3cbe86f24876 From 9ed6b2727c0790a106d4be8c4d25b1d6eceff050 Mon Sep 17 00:00:00 2001 From: Abhay Nigam <36666689+mashuk999@users.noreply.github.com> Date: Sun, 28 Jul 2024 21:29:01 +0530 Subject: [PATCH 260/561] fix: Correcting run.sh filepath in Simple Docker implementation (#1626) --- docker/simple/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/simple/Dockerfile b/docker/simple/Dockerfile index 2838fd1ff7..3594df1a50 100644 --- a/docker/simple/Dockerfile +++ b/docker/simple/Dockerfile @@ -35,4 +35,4 @@ ENV PORT=8000 EXPOSE 8000 # Run the server start script -CMD ["/bin/sh", "/app/run.sh"] +CMD ["/bin/sh", "/app/docker/simple/run.sh"] From 4bf3b43df95e62ddf74413a22a1df3411d206acc Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 28 Jul 2024 12:06:36 -0400 Subject: [PATCH 261/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d71e1609e7..4d74c30328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.84] + +- feat: Update llama.cpp to ggerganov/llama.cpp@4730faca618ff9cee0780580145e3cbe86f24876 +- fix: fix: Correcting run.sh filepath in Simple Docker implementation by @mashuk999 in #1626 + ## [0.2.83] - feat: Update llama.cpp to ggerganov/llama.cpp@081fe431aa8fb6307145c4feb3eed4f48cab19f8 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 65364d8782..d94a23db58 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.83" +__version__ = "0.2.84" From cffb4ec8c40e556e10b4e1168a43ba7f18bd957d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 30 Jul 2024 22:11:49 -0400 Subject: [PATCH 262/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4730faca61..268c566006 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4730faca618ff9cee0780580145e3cbe86f24876 +Subproject commit 268c5660062270a2c19a36fc655168aa287aaec2 From 53c6f329595bc3d8d242f58e0f1341d57bedfd92 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 31 Jul 2024 12:10:19 -0400 Subject: [PATCH 263/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 268c566006..398ede5efe 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 268c5660062270a2c19a36fc655168aa287aaec2 +Subproject commit 398ede5efeb07b9adf9fbda7ea63f630d476a792 From 0b1a8d8eff24595554513f0586929cba3e583bc2 Mon Sep 17 00:00:00 2001 From: "yuri@FreeBSD" Date: Wed, 31 Jul 2024 09:12:19 -0700 Subject: [PATCH 264/561] feat: FreeBSD compatibility (#1635) --- llama_cpp/llama_cpp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d563d77bc0..727195cf51 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -28,7 +28,7 @@ def _load_shared_library(lib_base_name: str): # for llamacpp) and "llama" (default name for this repo) _lib_paths: List[pathlib.Path] = [] # Determine the file extension based on the platform - if sys.platform.startswith("linux"): + if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): _lib_paths += [ _base_path / f"lib{lib_base_name}.so", ] From 8297a0d32ea1b5d88451a1805323a0a4bde466b2 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 31 Jul 2024 18:13:00 +0200 Subject: [PATCH 265/561] fix(docker): Update Dockerfile build options from `LLAMA_` to `GGML_` (#1634) --- docker/openblas_simple/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/openblas_simple/Dockerfile b/docker/openblas_simple/Dockerfile index 804c1b9029..5ee667dc0d 100644 --- a/docker/openblas_simple/Dockerfile +++ b/docker/openblas_simple/Dockerfile @@ -12,7 +12,7 @@ RUN apt update && apt install -y libopenblas-dev ninja-build build-essential pkg RUN python -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context -RUN CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS" pip install llama_cpp_python --verbose +RUN CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install llama_cpp_python --verbose # Run the server CMD python3 -m llama_cpp.server From ac0217404c72a66bd47a84fa741a09cf712b8975 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 31 Jul 2024 18:13:24 +0200 Subject: [PATCH 266/561] fix(docker): Fix GGML_CUDA param (#1633) --- docker/cuda_simple/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/cuda_simple/Dockerfile b/docker/cuda_simple/Dockerfile index 79b6a5bac8..0bbf20ffe9 100644 --- a/docker/cuda_simple/Dockerfile +++ b/docker/cuda_simple/Dockerfile @@ -15,13 +15,13 @@ COPY . . # setting build related env vars ENV CUDA_DOCKER_ARCH=all -ENV LLAMA_CUBLAS=1 +ENV GGML_CUDA=1 # Install depencencies RUN python3 -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context # Install llama-cpp-python (build with cuda) -RUN CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python +RUN CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python # Run the server CMD python3 -m llama_cpp.server From 8a12c9f34680c4bd3e6c84557f386a7253b8fbbe Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Wed, 31 Jul 2024 18:13:53 +0200 Subject: [PATCH 267/561] fix(docker): Update Dockerfile BLAS options (#1632) --- docker/open_llama/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/open_llama/Dockerfile b/docker/open_llama/Dockerfile index 70b16dffce..f05e66ef2a 100644 --- a/docker/open_llama/Dockerfile +++ b/docker/open_llama/Dockerfile @@ -20,13 +20,13 @@ RUN python3 -m pip install --upgrade pip pytest cmake scikit-build setuptools fa # Perform the conditional installations based on the image RUN echo "Image: ${IMAGE}" && \ - if [ "${IMAGE}" = "python:3-slim-bullseye" ] ; then \ + if [ "${IMAGE}" = "python:3-slim-bookworm" ] ; then \ echo "OpenBLAS install:" && \ apt-get install -y --no-install-recommends libopenblas-dev && \ - LLAMA_OPENBLAS=1 pip install llama-cpp-python --verbose; \ + CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install llama-cpp-python --verbose; \ else \ echo "CuBLAS install:" && \ - LLAMA_CUBLAS=1 pip install llama-cpp-python --verbose; \ + CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python --verbose; \ fi # Clean up apt cache From 1f0b9a2872065a89c16a4567b561ddd10f2533fe Mon Sep 17 00:00:00 2001 From: Shamit Verma Date: Wed, 31 Jul 2024 21:45:51 +0530 Subject: [PATCH 268/561] fix : Missing LoRA adapter after API change (#1630) --- llama_cpp/llama.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 005045f5c4..0cb5ca2fc4 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2083,11 +2083,14 @@ def pooling_type(self) -> str: def close(self) -> None: """Explicitly free the model from memory.""" - self._stack.close() + if hasattr(self,'_stack'): + if self._stack is not None: + self._stack.close() def __del__(self) -> None: - if self._lora_adapter is not None: - llama_cpp.llama_lora_adapter_free(self._lora_adapter) + if hasattr(self,'_lora_adapter'): + if self._lora_adapter is not None: + llama_cpp.llama_lora_adapter_free(self._lora_adapter) self.close() @staticmethod From f7b9e6d42981bb75106699e0712a6378c39ef92c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 31 Jul 2024 12:24:21 -0400 Subject: [PATCH 269/561] chore: Bump version --- CHANGELOG.md | 9 +++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d74c30328..6dc25442b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.85] + +- feat: Update llama.cpp to ggerganov/llama.cpp@398ede5efeb07b9adf9fbda7ea63f630d476a792 +- fix: Missing LoRA adapter after API change by @shamitv in #1630 +- fix(docker): Update Dockerfile BLAS options by @olivierdebauche in #1632 +- fix(docker): Fix GGML_CUDA param by @olivierdebauche in #1633 +- fix(docker): Update Dockerfile build options from `LLAMA_` to `GGML_` by @olivierdebauche in #1634 +- feat: FreeBSD compatibility by @yurivict in #1635 + ## [0.2.84] - feat: Update llama.cpp to ggerganov/llama.cpp@4730faca618ff9cee0780580145e3cbe86f24876 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index d94a23db58..97c8d8174f 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.84" +__version__ = "0.2.85" From 5575fedd33141ffdeb87304da3b1225da58db76a Mon Sep 17 00:00:00 2001 From: tc-wolf <50339167+tc-wolf@users.noreply.github.com> Date: Sun, 4 Aug 2024 16:07:54 -0500 Subject: [PATCH 270/561] fix: llama_grammar_accept_token arg order (#1649) Old was: llama_grammar_accept_token(ctx, grammar, token) Now this is: llama_grammar_accept_token(grammar, ctx, token) --- llama_cpp/_internals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index dcd4e17ff7..357023e3c5 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -511,7 +511,7 @@ def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: def grammar_accept_token(self, grammar: LlamaGrammar, token: int): assert self.ctx is not None assert grammar.grammar is not None - llama_cpp.llama_grammar_accept_token(self.ctx, grammar.grammar, token) + llama_cpp.llama_grammar_accept_token(grammar.grammar, self.ctx, token) def reset_timings(self): assert self.ctx is not None From dff186cc5911a03c1c2de2505ab467c419aa6821 Mon Sep 17 00:00:00 2001 From: ExtReMLapin <3909752+ExtReMLapin@users.noreply.github.com> Date: Wed, 7 Aug 2024 02:19:36 +0200 Subject: [PATCH 271/561] feat: Ported back new grammar changes from C++ to Python implementation (#1637) * Backported . (any chat) from llama.cpp * unfinished {count,optionalmax) * implemented slice function in std:vector * fixed mistake done while reading * ported https://github.com/ggerganov/llama.cpp/pull/7194 * multiple fixes, var copy * Rewrite LlamaGrammar internals in python style * bugfix --------- Co-authored-by: Andrei --- llama_cpp/llama_cpp.py | 2 +- llama_cpp/llama_grammar.py | 1652 +++++++++++++++--------------------- 2 files changed, 682 insertions(+), 972 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 727195cf51..d598cf9f16 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -3002,7 +3002,7 @@ def llama_grammar_init( n_rules: Union[ctypes.c_size_t, int], start_rule_index: Union[ctypes.c_size_t, int], /, -) -> llama_grammar_p: +) -> Optional[llama_grammar_p]: """Initialize a grammar from a set of rules.""" ... diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 0ac7354bba..e60817e1d7 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -1,520 +1,101 @@ """Python implementation of llama grammar parser directly translated from C++ source file in vendor/llama.cpp/common/grammar-parser.cpp.""" # flake8: noqa -from pathlib import Path import sys -from ctypes import * # type: ignore -from enum import Enum -from itertools import islice, groupby +import ctypes +import enum +import typing +import dataclasses + +from itertools import groupby from typing import ( Any, - Callable, - Dict, Set, - Generic, List, Optional, - OrderedDict, - TextIO, Tuple, - TypeVar, Union, - overload, ) import llama_cpp.llama_cpp as llama_cpp -# Type aliases -llama_grammar_element = llama_cpp.llama_grammar_element -llama_grammar_element_p = llama_cpp.llama_grammar_element_p -llama_grammar_p = llama_cpp.llama_grammar_p - -# Type variables -Ptr = TypeVar("Ptr", bound="const_char_p") -T = TypeVar("T") -U = TypeVar("U") -V = TypeVar("V") -W = TypeVar("W") - - -class Sentinel: - """Used to mark the end of a iterator of std::vector & std::map.""" - - -class LlamaGrammar: - """Keeps reference counts of all the arguments, so that they are not - garbage collected by Python.""" - - def __del__(self) -> None: - """Free the grammar pointer when the object is deleted.""" - if self.grammar is not None: - llama_cpp.llama_grammar_free(self.grammar) - self.grammar = None - - def __init__( - self, - parsed_grammar: "parse_state", - ) -> None: - """Initialize the grammar pointer from the parsed state.""" - self._grammar_rules = ( - parsed_grammar.c_rules() - ) # type: std.vector[std.vector[LlamaGrammarElement]] - self._n_rules = self._grammar_rules.size() # type: int - self._start_rule_index = parsed_grammar.symbol_ids.at("root") # type: int - self.init() - - @classmethod - def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": - """Convert a GBNF grammar to a Llama grammar.""" - parsed_grammar = parse(const_char_p(grammar)) # type: parse_state - if parsed_grammar.rules.empty(): - raise ValueError( - f"{cls.from_string.__name__}: error parsing grammar file: parsed_grammar.rules is empty" - ) - if verbose: - print(f"{cls.from_string.__name__} grammar:", file=sys.stderr) - print_grammar(sys.stderr, parsed_grammar) - print(file=sys.stderr) - return cls(parsed_grammar) - - @classmethod - def from_json_schema( - cls, - json_schema: str, - verbose: bool = True, - ) -> "LlamaGrammar": - """Convert a JSON schema to a Llama grammar.""" - return cls.from_string(json_schema_to_gbnf(json_schema), verbose=verbose) - - @classmethod - def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGrammar": - try: - with open(file) as f: - grammar = f.read() - except Exception as err: - raise Exception( - f"{cls.from_file.__name__}: error reading grammar file: {err}" - ) - - if grammar: - return cls.from_string(grammar, verbose=verbose) - - raise ValueError( - f"{cls.from_file.__name__}: error parsing grammar file: params_grammer is empty" - ) - - def init(self) -> None: - # Step 1: Convert LlamaGrammarElement to llama_grammar_element - self._element_lists = [ - [ - llama_grammar_element(c_int(elem.type.value), c_uint32(elem.value)) - for elem in subvector - ] - for subvector in self._grammar_rules - ] # type: List[List[llama_grammar_element]] - - # Step 2: Convert each list to llama_grammar_element array and get pointer - self._element_arrays = [ - (llama_grammar_element * len(sublist))(*sublist) - for sublist in self._element_lists - ] # type: List[Array[llama_grammar_element]] - - # Step 3: Get pointer of each array - self._element_array_pointers = [ - cast(subarray, llama_grammar_element_p) for subarray in self._element_arrays - ] # type: List[llama_grammar_element_p] - - # Step 4: Make array of these pointers and get its pointer - self._rules = (llama_grammar_element_p * len(self._element_array_pointers))( - *self._element_array_pointers - ) - self.grammar = llama_cpp.llama_grammar_init( - self._rules, c_size_t(self._n_rules), c_size_t(self._start_rule_index) - ) +class GrammarElementType(enum.IntEnum): + END = llama_cpp.LLAMA_GRETYPE_END + ALT = llama_cpp.LLAMA_GRETYPE_ALT + RULE_REF = llama_cpp.LLAMA_GRETYPE_RULE_REF + CHAR = llama_cpp.LLAMA_GRETYPE_CHAR + CHAR_NOT = llama_cpp.LLAMA_GRETYPE_CHAR_NOT + CHAR_RNG_UPPER = llama_cpp.LLAMA_GRETYPE_CHAR_RNG_UPPER + CHAR_ALT = llama_cpp.LLAMA_GRETYPE_CHAR_ALT + CHAR_ANY = llama_cpp.LLAMA_GRETYPE_CHAR_ANY - def reset(self) -> None: - if self.grammar is not None: - llama_cpp.llama_grammar_free(self.grammar) - self.init() +@dataclasses.dataclass +class GrammarElement: + type: GrammarElementType + value: int -class LlamaGrammarElement: - def __init__(self, type: "llama_gretype", value: int): - self.type = type - self.value = value # Unicode code point or rule ID +@dataclasses.dataclass +class ParseState: + symbol_ids: typing.Dict[str, int] = dataclasses.field(default_factory=dict) + rules: typing.List[typing.List[GrammarElement]] = dataclasses.field(default_factory=list) -class const_char_p: - """C++ implementation of const char *.""" - - def __init__(self, value: Union[str, Ptr], move: Optional[int] = None): - if isinstance(value, const_char_p): - # We're copying an existing const_char_p - self.value = value.value - self.pos = value.pos + (move or 0) - return - - # We're creating a new const_char_p - self.value = value - self.pos = move or 0 - - def __str__(self) -> str: - assert self.value is not None, "null pointer" - return self.value[self.pos :] - - def __getitem__(self, index: int) -> str: - value = str(self) - return value[index] if index < len(value) else "" - - @overload - def __add__(self: Ptr, other: int) -> Ptr: ... - - @overload - def __add__(self: Ptr, other: Ptr) -> int: ... - - def __add__(self: Ptr, other: Union[int, Ptr]) -> Union[int, Ptr]: - return ( - self.__class__(self.value, self.pos + other) - if isinstance(other, int) - else self.pos + other.pos - ) - - @overload - def __sub__(self: Ptr, other: int) -> Ptr: ... - - @overload - def __sub__(self: Ptr, other: Ptr) -> int: ... - - def __sub__(self: Ptr, other: Union[int, Ptr]) -> Union[int, Ptr]: - return ( - self.__class__(self.value, self.pos - other) - if isinstance(other, int) - else self.pos - other.pos - ) - - def __eq__(self: Ptr, other: Ptr) -> bool: - assert self.value == other.value, "comparing pointers from different strings" - return self.pos == other.pos - - def __lt__(self: Ptr, other: Ptr) -> bool: - assert self.value == other.value, "comparing pointers from different strings" - return self.pos < other.pos - - def __gt__(self: Ptr, other: Ptr) -> bool: - assert self.value == other.value, "comparing pointers from different strings" - return self.pos > other.pos - - -class std: - @staticmethod - def string(ptr: const_char_p, length: Optional[int] = None) -> str: - """C++ implementation of std::string constructor.""" - value = str(ptr) - if length is not None: - value = value[:length] - return value - - class vector(Generic[T], List[T]): - """C++ implementation of std::vector.""" - - class iterator: - def __init__(self, vector: "std.vector[T]", index: int): - self._vector = vector - self._index = index - self._version = vector._version - - def _check_version(self): - if self._version != self._vector._version: - raise RuntimeError("Iterator used after vector was modified.") - - def __iter__(self): - return self - - def __next__(self) -> T: - self._check_version() - if self._index >= self._vector.size(): - raise StopIteration - value = self._vector[self._index] - self._index += 1 - return value - - def __add__(self, value: int) -> "std.vector[T].iterator": - return self.__class__(self._vector, self._index + value) - - def __sub__(self, value: int) -> "std.vector[T].iterator": - return self.__class__(self._vector, self._index - value) - - def __init__(self): - self._version = 0 - - def modify(self): - # This is a bit of a hack to make sure iterators are invalidated - self._version += 1 - - def push_back(self, value: T) -> None: - self.modify() - self.append(value) - - def pop_back(self) -> None: - self.modify() - if not self.empty(): - self.pop() - - def back(self) -> T: - return self[-1] - - def size(self) -> int: - return len(self) - - def clear(self) -> None: - self.modify() - super().clear() - - def empty(self) -> bool: - return self.size() == 0 - - def data(self) -> "std.vector[T]": - return self - - def resize( - self, - new_size: int, - fill_value_factory: Optional[Callable[[], T]] = None, - ) -> None: - if new_size > self.size(): - if fill_value_factory is None: - raise ValueError("A fill value factory function must be provided.") - self.reserve(new_size, fill_value_factory) - elif new_size < self.size(): - self[:] = self[:new_size] - - def reserve(self, capacity: int, fill_value_factory: Callable[[], T]) -> None: - if capacity > self.size(): - fill_value = fill_value_factory() - self.extend([fill_value] * (capacity - self.size())) - - def front(self) -> T: - if not self.empty(): - return self[0] - else: - raise IndexError("Vector is empty.") - - def assign(self, count: int, value: T) -> None: - self.clear() - self.extend([value] * count) - - def insert( - self, - pos: "std.vector[T].iterator", - first: "std.vector[T].iterator", - last: "std.vector[T].iterator", - ) -> None: - self[pos._index : pos._index] = list( - islice(first._vector, first._index, last._index) - ) - - def begin(self) -> "std.vector[T].iterator": - return self.iterator(self, 0) - - def end(self) -> "std.vector[T].iterator": - return self.iterator(self, self.size()) - - class map(Generic[T, U], OrderedDict[T, U]): - """C++ implementation of std::map.""" - - class iterator(Generic[V, W]): - def __init__(self, _map: "std.map[T, U]", key: Union[T, Sentinel]): - self._map = _map - self.iter = iter(_map) - self.key = key - self._advance() - - def _sanitize_key(self) -> T: - if isinstance(self.key, Sentinel): - raise StopIteration - return self.key - - def _advance(self) -> None: - try: - while next(self.iter) != self.key: - pass - except StopIteration: - self.key = Sentinel() - - def __next__(self) -> Tuple[T, U]: - key = self._sanitize_key() - if key in self._map: - value = self._map[key] - self._advance() - return key, value - else: - raise StopIteration - - def get(self) -> Tuple[T, U]: - key = self._sanitize_key() - return key, self._map[key] - - @property - def first(self) -> T: - return self._sanitize_key() - - @property - def second(self) -> U: - return self._map[self._sanitize_key()] - - def insert( - self, key: T, value: U - ) -> Tuple["std.map[T, U].iterator[T, U]", bool]: - if key in self: - return self.iterator(self, key), False - else: - self[key] = value - return self.iterator(self, key), True - - def find(self, key: T) -> "std.map[T, U].iterator[T, U]": - if key in self: - return self.iterator(self, key) - else: - return self.end() - - def at(self, key: T) -> U: - if key in self: - return self[key] - else: - raise KeyError("The provided key is not found in the map.") - - def erase(self, iterator: "std.map[T, U].iterator[T, U]") -> None: - key = iterator.first - if key in self: - del self[key] - - def size(self) -> int: - return len(self) - - def empty(self) -> bool: - return self.size() == 0 - - def lower_bound(self, key: T) -> "std.map[T, U].iterator[T, U]": - try: - keys = sorted(list(self.keys())) # type: ignore - for k in keys: - if k >= key: - return self.iterator(self, k) - raise ValueError("No key found that is not less than the input key") - except TypeError: - raise TypeError("Keys of type T cannot be sorted.") - - def begin(self) -> "std.map[T, U].iterator[T, U]": - return self.iterator(self, next(iter(self))) - - def end(self) -> "std.map[T, U].iterator[T, U]": - return self.iterator(self, Sentinel()) - - -# // grammar element type -# enum llama_gretype { -# // end of rule definition -# LLAMA_GRETYPE_END = 0, - -# // start of alternate definition for rule -# LLAMA_GRETYPE_ALT = 1, - -# // non-terminal element: reference to rule -# LLAMA_GRETYPE_RULE_REF = 2, - -# // terminal element: character (code point) -# LLAMA_GRETYPE_CHAR = 3, - -# // inverse char(s) ([^a], [^a-b] [^abc]) -# LLAMA_GRETYPE_CHAR_NOT = 4, - -# // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to -# // be an inclusive range ([a-z]) -# LLAMA_GRETYPE_CHAR_RNG_UPPER = 5, - - -# // modifies a preceding LLAMA_GRETYPE_CHAR or -# // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) -# LLAMA_GRETYPE_CHAR_ALT = 6, -# }; -class llama_gretype(Enum): - """grammar element type""" - - LLAMA_GRETYPE_END = 0 # end of rule definition - LLAMA_GRETYPE_ALT = 1 # start of alternate definition for rule - LLAMA_GRETYPE_RULE_REF = 2 # non-terminal element: reference to rule - LLAMA_GRETYPE_CHAR = 3 # terminal element: character (code point) - LLAMA_GRETYPE_CHAR_NOT = 4 # inverse char(s) ([^a], [^a-b] [^abc]) - LLAMA_GRETYPE_CHAR_RNG_UPPER = 5 # modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to be an inclusive range ([a-z]) - LLAMA_GRETYPE_CHAR_ALT = 6 # modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) - - -# struct parse_state { -# std::map symbol_ids; -# std::vector> rules; -# std::vector c_rules(); -# }; -class parse_state: - def __init__(self): - self.symbol_ids: std.map[str, int] = std.map() - self.rules: std.vector[std.vector[LlamaGrammarElement]] = std.vector() - - # std::vector parse_state::c_rules() { - # std::vector ret; - # for (const auto & rule : rules) { - # ret.push_back(rule.data()); - # } - # return ret; - # } - def c_rules(self) -> std.vector[std.vector[LlamaGrammarElement]]: - ret = std.vector() # type: std.vector[std.vector[LlamaGrammarElement]] - for rule in self.rules: - ret.push_back(rule.data()) - return ret - - def __repr__(self) -> str: - return ( - f"parse_state(symbol_ids={len(self.symbol_ids)}, rules={len(self.rules)})" - ) +# static std::pair decode_utf8(const char * src) { +# static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; +# uint8_t first_byte = static_cast(*src); +# uint8_t highbits = first_byte >> 4; +# int len = lookup[highbits]; +# uint8_t mask = (1 << (8 - len)) - 1; +# uint32_t value = first_byte & mask; +# const char * end = src + len; // may overrun! +# const char * pos = src + 1; +# for ( ; pos < end && *pos; pos++) { +# value = (value << 6) + (static_cast(*pos) & 0x3F); +# } +# return std::make_pair(value, pos); +# } +def decode_utf8(src: str) -> typing.Tuple[int, str]: + lookup: list[int] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4] + first_byte: int = ord(src[0]) + highbits: int = first_byte >> 4 + length: int = lookup[highbits] + mask: int = (1 << (8 - length)) - 1 + value: int = first_byte & mask + end: int = min(len(src), length) # Prevent overrun + + pos: int = 1 + for pos in range(1, end): + if not src[pos]: + break + value = (value << 6) + (ord(src[pos]) & 0x3F) -# struct llama_grammar { -# const std::vector> rules; -# std::vector> stacks; -# }; -# class llama_grammar: -# def __init__( -# self, -# rules: std.vector[std.vector[llama_grammar_element]], -# stacks: std.vector[std.vector[llama_grammar_element]], -# ): -# self.rules = rules -# self.stacks = stacks + return value, src[pos:] if pos < len(src) else "" -# uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) { +# static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) { # uint32_t next_id = static_cast(state.symbol_ids.size()); -# auto result = state.symbol_ids.insert(std::make_pair(std::string(src, len), next_id)); +# auto result = state.symbol_ids.emplace(std::string(src, len), next_id); # return result.first->second; # } -def get_symbol_id(state: parse_state, src: const_char_p, len: int) -> int: - next_id = state.symbol_ids.size() # type: int - result = state.symbol_ids.insert(std.string(src, len), next_id) - return result[0].second # type: ignore +def get_symbol_id(state: ParseState, name: str) -> int: + next_id = len(state.symbol_ids) + return state.symbol_ids.setdefault(name, next_id) -# uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) { +# static uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) { # uint32_t next_id = static_cast(state.symbol_ids.size()); # state.symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id; # return next_id; # } -def generate_symbol_id(state: parse_state, base_name: str) -> int: - next_id = state.symbol_ids.size() # type: int - state.symbol_ids[base_name + "_" + str(next_id)] = next_id +def generate_symbol_id(state: ParseState, base_name: str) -> int: + next_id = len(state.symbol_ids) + state.symbol_ids[f"{base_name}_{next_id}"] = next_id return next_id -# void add_rule( +# static void add_rule( # parse_state & state, # uint32_t rule_id, # const std::vector & rule) { @@ -523,51 +104,27 @@ def generate_symbol_id(state: parse_state, base_name: str) -> int: # } # state.rules[rule_id] = rule; # } -def add_rule( - state: parse_state, - rule_id: int, - rule: std.vector[LlamaGrammarElement], -) -> None: - if state.rules.size() <= rule_id: - state.rules.resize( - rule_id + 1, - fill_value_factory=std.vector[LlamaGrammarElement], - ) +def add_rule(state: ParseState, rule_id: int, rule: typing.List[GrammarElement]) -> None: + if len(state.rules) <= rule_id: + state.rules.extend([[]] * (rule_id + 1 - len(state.rules))) state.rules[rule_id] = rule -# std::pair decode_utf8(const char * src) { -# static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; -# uint8_t first_byte = static_cast(*src); -# uint8_t highbits = first_byte >> 4; -# int len = lookup[highbits]; -# uint8_t mask = (1 << (8 - len)) - 1; -# uint32_t value = first_byte & mask; -# const char * end = src + len; // may overrun! -# const char * pos = src + 1; -# for ( ; pos < end && *pos; pos++) { -# value = (value << 6) + (static_cast(*pos) & 0x3F); -# } -# return std::make_pair(value, pos); +# static bool is_digit_char(char c) { +# return '0' <= c && c <= '9'; # } -def decode_utf8(src: const_char_p) -> Tuple[int, const_char_p]: - """Decodes a UTF-8 character from the source string.""" - # Get the codepoint of the first character - value = ord(src[0]) - # Move the pointer ahead one character - pos = src + 1 - - return value, pos +def is_digit_char(c: str) -> bool: + return "0" <= c <= "9" -# bool is_word_char(char c) { -# return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || ('0' <= c && c <= '9'); +# static bool is_word_char(char c) { +# return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || is_digit_char(c); # } def is_word_char(c: str) -> bool: - return ("a" <= c <= "z") or ("A" <= c <= "Z") or c == "-" or ("0" <= c <= "9") + return ("a" <= c <= "z") or ("A" <= c <= "Z") or c == "-" or is_digit_char(c) -# std::pair parse_hex(const char * src, int size) { +# static std::pair parse_hex(const char * src, int size) { # const char * pos = src; # const char * end = src + size; # uint32_t value = 0; @@ -589,13 +146,12 @@ def is_word_char(c: str) -> bool: # } # return std::make_pair(value, pos); # } -def parse_hex(src: const_char_p, size: int) -> Tuple[int, const_char_p]: - pos = const_char_p(src) # type: const_char_p - end = src + size # type: const_char_p - value = 0 # type: int - while pos < end and pos[0]: +def parse_hex(src: str, size: int) -> typing.Tuple[int, str]: + pos = 0 + value = 0 + for _ in range(size): value <<= 4 - c = pos[0] # type: str + c = src[pos] if "a" <= c <= "f": value += ord(c) - ord("a") + 10 elif "A" <= c <= "F": @@ -605,12 +161,74 @@ def parse_hex(src: const_char_p, size: int) -> Tuple[int, const_char_p]: else: break pos += 1 - if pos != end: - raise RuntimeError("expecting " + str(size) + " hex chars at " + str(src)) - return (value, pos) + if pos != size: + raise ValueError(f"expecting {size} hex chars at {src}") + return value, src[pos:] + + +# static const char * parse_space(const char * src, bool newline_ok) { +# const char * pos = src; +# while (*pos == ' ' || *pos == '\t' || *pos == '#' || +# (newline_ok && (*pos == '\r' || *pos == '\n'))) { +# if (*pos == '#') { +# while (*pos && *pos != '\r' && *pos != '\n') { +# pos++; +# } +# } else { +# pos++; +# } +# } +# return pos; +# } +def parse_space(src: str, newline_ok: bool) -> str: + pos = src + while pos and (pos[0] in (' ', '\t', '#') or (newline_ok and pos[0] in ('\r', '\n'))): + if pos[0] == "#": + while pos and pos[0] not in ("\r", "\n"): + pos = pos[1:] + else: + pos = pos[1:] + return pos -# std::pair parse_char(const char * src) { +# static const char * parse_name(const char * src) { +# const char * pos = src; +# while (is_word_char(*pos)) { +# pos++; +# } +# if (pos == src) { +# throw std::runtime_error(std::string("expecting name at ") + src); +# } +# return pos; +# } +def parse_name(src: str) -> typing.Tuple[str, str]: + pos = src + while pos and is_word_char(pos[0]): + pos = pos[1:] + if pos == src: + raise ValueError(f"expecting name at {src}") + return src[:len(src) - len(pos)], pos + +# static const char * parse_int(const char * src) { +# const char * pos = src; +# while (is_digit_char(*pos)) { +# pos++; +# } +# if (pos == src) { +# throw std::runtime_error(std::string("expecting integer at ") + src); +# } +# return pos; +# } +def parse_int(src: str) -> typing.Tuple[int, str]: + pos = src + while pos and is_digit_char(pos[0]): + pos = pos[1:] + if pos == src: + raise ValueError(f"expecting integer at {src}") + return int(src[:len(src) - len(pos)]), pos + + +# static std::pair parse_char(const char * src) { # if (*src == '\\') { # switch (src[1]) { # case 'x': return parse_hex(src + 2, 2); @@ -632,273 +250,320 @@ def parse_hex(src: const_char_p, size: int) -> Tuple[int, const_char_p]: # } # throw std::runtime_error("unexpected end of input"); # } -def parse_char(src: const_char_p) -> Tuple[int, const_char_p]: +def parse_char(src: str) -> typing.Tuple[int, str]: + if not src: + raise ValueError("unexpected end of input") if src[0] == "\\": - case = src[1] # type: str - if case == "x": - return parse_hex(src + 2, 2) - elif case == "u": - return parse_hex(src + 2, 4) - elif case == "U": - return parse_hex(src + 2, 8) - elif case == "t": - return (ord("\t"), src + 2) # implicit cast - elif case == "r": - return (ord("\r"), src + 2) # implicit cast - elif case == "n": - return (ord("\n"), src + 2) # implicit cast - elif case in ("\\", '"', "[", "]"): - return (ord(case), src + 2) # implicit cast + if src[1] == "x": + return parse_hex(src[2:], 2) + elif src[1] == "u": + return parse_hex(src[2:], 4) + elif src[1] == "U": + return parse_hex(src[2:], 8) + elif src[1] == "t": + return ord("\t"), src[2:] + elif src[1] == "r": + return ord("\r"), src[2:] + elif src[1] == "n": + return ord("\n"), src[2:] + elif src[1] in ('\\', '"', '[', ']'): + return ord(src[1]), src[2:] else: - raise RuntimeError("unknown escape at " + str(src)) - elif src[0]: - return decode_utf8(src) - else: - raise RuntimeError("unexpected end of input") - - -# const char * parse_name(const char * src) { -# const char * pos = src; -# while (is_word_char(*pos)) { -# pos++; -# } -# if (pos == src) { -# throw std::runtime_error(std::string("expecting name at ") + src); -# } -# return pos; -# } -def parse_name(src: const_char_p) -> const_char_p: - pos = const_char_p(src) # type: const_char_p - while is_word_char(pos[0]): - pos += 1 - if pos == src: - raise RuntimeError("expecting name at " + str(src)) - return pos + raise ValueError(f"unknown escape at {src}") + return decode_utf8(src) - -# const char * parse_space(const char * src, bool newline_ok) { +# static const char * parse_sequence( +# parse_state & state, +# const char * src, +# const std::string & rule_name, +# std::vector & out_elements, +# bool is_nested) { +# size_t last_sym_start = out_elements.size(); # const char * pos = src; -# while (*pos == ' ' || *pos == '\t' || *pos == '#' || -# (newline_ok && (*pos == '\r' || *pos == '\n'))) { -# if (*pos == '#') { -# while (*pos && *pos != '\r' && *pos != '\n') { +# +# auto handle_repetitions = [&](int min_times, int max_times) { +# +# if (last_sym_start == out_elements.size()) { +# throw std::runtime_error(std::string("expecting preceding item to */+/?/{ at ") + pos); +# } +# +# // apply transformation to previous symbol (last_sym_start to end) according to +# // the following rewrite rules: +# // S{m,n} --> S S S (m times) S'(n-m) +# // S'(x) ::= S S'(x-1) | +# // (... n-m definitions of these S' rules ...) +# // S'(1) ::= S | +# // S{m,} --> S S S (m times) S' +# // S' ::= S S' | +# // S* --> S{0,} +# // --> S' ::= S S' | +# // S+ --> S{1,} +# // --> S S' +# // S' ::= S S' | +# // S? --> S{0,1} +# // --> S' +# // S' ::= S | +# +# std::vector previous_elements(out_elements.begin() + last_sym_start, out_elements.end()); +# if (min_times == 0) { +# out_elements.resize(last_sym_start); +# } else { +# // Repeat the previous elements (min_times - 1) times +# for (int i = 1; i < min_times; i++) { +# out_elements.insert(out_elements.end(), previous_elements.begin(), previous_elements.end()); +# } +# } +# +# uint32_t last_rec_rule_id = 0; +# auto n_opt = max_times < 0 ? 1 : max_times - min_times; +# +# std::vector rec_rule(previous_elements); +# for (int i = 0; i < n_opt; i++) { +# rec_rule.resize(previous_elements.size()); +# uint32_t rec_rule_id = generate_symbol_id(state, rule_name); +# if (i > 0 || max_times < 0) { +# rec_rule.push_back({LLAMA_GRETYPE_RULE_REF, max_times < 0 ? rec_rule_id : last_rec_rule_id}); +# } +# rec_rule.push_back({LLAMA_GRETYPE_ALT, 0}); +# rec_rule.push_back({LLAMA_GRETYPE_END, 0}); +# add_rule(state, rec_rule_id, rec_rule); +# last_rec_rule_id = rec_rule_id; +# } +# if (n_opt > 0) { +# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, last_rec_rule_id}); +# } +# }; +# +# while (*pos) { +# if (*pos == '"') { // literal string +# pos++; +# last_sym_start = out_elements.size(); +# while (*pos != '"') { +# if (!*pos) { +# throw std::runtime_error("unexpected end of input"); +# } +# auto char_pair = parse_char(pos); +# pos = char_pair.second; +# out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first}); +# } +# pos = parse_space(pos + 1, is_nested); +# } else if (*pos == '[') { // char range(s) +# pos++; +# enum llama_gretype start_type = LLAMA_GRETYPE_CHAR; +# if (*pos == '^') { # pos++; +# start_type = LLAMA_GRETYPE_CHAR_NOT; +# } +# last_sym_start = out_elements.size(); +# while (*pos != ']') { +# if (!*pos) { +# throw std::runtime_error("unexpected end of input"); +# } +# auto char_pair = parse_char(pos); +# pos = char_pair.second; +# enum llama_gretype type = last_sym_start < out_elements.size() +# ? LLAMA_GRETYPE_CHAR_ALT +# : start_type; +# +# out_elements.push_back({type, char_pair.first}); +# if (pos[0] == '-' && pos[1] != ']') { +# if (!pos[1]) { +# throw std::runtime_error("unexpected end of input"); +# } +# auto endchar_pair = parse_char(pos + 1); +# pos = endchar_pair.second; +# out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first}); +# } +# } +# pos = parse_space(pos + 1, is_nested); +# } else if (is_word_char(*pos)) { // rule reference +# const char * name_end = parse_name(pos); +# uint32_t ref_rule_id = get_symbol_id(state, pos, name_end - pos); +# pos = parse_space(name_end, is_nested); +# last_sym_start = out_elements.size(); +# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id}); +# } else if (*pos == '(') { // grouping +# // parse nested alternates into synthesized rule +# pos = parse_space(pos + 1, true); +# uint32_t sub_rule_id = generate_symbol_id(state, rule_name); +# pos = parse_alternates(state, pos, rule_name, sub_rule_id, true); +# last_sym_start = out_elements.size(); +# // output reference to synthesized rule +# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); +# if (*pos != ')') { +# throw std::runtime_error(std::string("expecting ')' at ") + pos); +# } +# pos = parse_space(pos + 1, is_nested); +# } else if (*pos == '.') { // any char +# last_sym_start = out_elements.size(); +# out_elements.push_back({LLAMA_GRETYPE_CHAR_ANY, 0}); +# pos = parse_space(pos + 1, is_nested); +# } else if (*pos == '*') { +# pos = parse_space(pos + 1, is_nested); +# handle_repetitions(0, -1); +# } else if (*pos == '+') { +# pos = parse_space(pos + 1, is_nested); +# handle_repetitions(1, -1); +# } else if (*pos == '?') { +# pos = parse_space(pos + 1, is_nested); +# handle_repetitions(0, 1); +# } else if (*pos == '{') { +# pos = parse_space(pos + 1, is_nested); +# +# if (!is_digit_char(*pos)) { +# throw std::runtime_error(std::string("expecting an int at ") + pos); # } +# const char * int_end = parse_int(pos); +# int min_times = std::stoul(std::string(pos, int_end - pos)); +# pos = parse_space(int_end, is_nested); +# +# int max_times = -1; +# +# if (*pos == '}') { +# max_times = min_times; +# pos = parse_space(pos + 1, is_nested); +# } else if (*pos == ',') { +# pos = parse_space(pos + 1, is_nested); +# +# if (is_digit_char(*pos)) { +# const char * int_end = parse_int(pos); +# max_times = std::stoul(std::string(pos, int_end - pos)); +# pos = parse_space(int_end, is_nested); +# } +# +# if (*pos != '}') { +# throw std::runtime_error(std::string("expecting '}' at ") + pos); +# } +# pos = parse_space(pos + 1, is_nested); +# } else { +# throw std::runtime_error(std::string("expecting ',' at ") + pos); +# } +# handle_repetitions(min_times, max_times); # } else { -# pos++; +# break; # } # } # return pos; # } -def parse_space(src: const_char_p, newline_ok: bool) -> const_char_p: - pos = const_char_p(src) # type: const_char_p - while pos[0] in (" ", "\t", "#") or (newline_ok and pos[0] in ("\r", "\n")): - if pos[0] == "#": - while pos[0] is not None and pos[0] not in ("\r", "\n"): - pos += 1 - else: - pos += 1 - return pos +def parse_sequence(state: ParseState, src: str, rule_name: str, out_elements: typing.List[GrammarElement], is_nested: bool) -> str: + last_sym_start = len(out_elements) + pos = src + def handle_repetitions(min_times: int, max_times: int) -> None: + nonlocal state, src, rule_name, out_elements, is_nested, last_sym_start, pos -# const char * parse_sequence( -# parse_state & state, -# const char * src, -# const std::string & rule_name, -# std::vector & out_elements, -# bool is_nested) { -def parse_sequence( - state: parse_state, - src: const_char_p, - rule_name: str, - out_elements: std.vector[LlamaGrammarElement], - is_nested: bool, -) -> const_char_p: - # size_t last_sym_start = out_elements.size(); - # const char * pos = src; - last_sym_start = out_elements.size() # type: int - pos = const_char_p(src) # type: const_char_p - # while (*pos) { - while pos[0]: - # if (*pos == '"') { // literal string - # pos++; - # last_sym_start = out_elements.size(); - # while (*pos != '"') { - # auto char_pair = parse_char(pos); - # pos = char_pair.second; - # out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first}); - # } - # pos = parse_space(pos + 1, is_nested); - if pos[0] == '"': # literal string - pos += 1 - last_sym_start = out_elements.size() - while pos[0] != '"': - char_pair = parse_char(pos) # type: Tuple[int, const_char_p] - pos = char_pair[1] - out_elements.push_back( - LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_CHAR, char_pair[0]) - ) - pos = parse_space(pos + 1, is_nested) - # } else if (*pos == '[') { // char range(s) - # pos++; - # enum llama_gretype start_type = LLAMA_GRETYPE_CHAR; - elif pos[0] == "[": # char range(s) - pos += 1 - start_type = llama_gretype.LLAMA_GRETYPE_CHAR # type: llama_gretype - # if (*pos == '^') { - # pos++; - # start_type = LLAMA_GRETYPE_CHAR_NOT; - # } - # last_sym_start = out_elements.size(); + if last_sym_start == len(out_elements): + raise ValueError(f"expecting preceding item to */+/?/{{ at {pos}") + + previous_elements = out_elements[last_sym_start:] + if min_times == 0: + del out_elements[last_sym_start:] + else: + for i in range(1, min_times): + out_elements.extend(previous_elements) + + last_rec_rule_id = 0 + n_opt = 1 if max_times < 0 else max_times - min_times + + rec_rule = previous_elements[:] + for i in range(n_opt): + rec_rule = rec_rule[:len(previous_elements)] + rec_rule_id = generate_symbol_id(state, rule_name) + if i > 0 or max_times < 0: + rec_rule.append(GrammarElement(GrammarElementType.RULE_REF, rec_rule_id if max_times < 0 else last_rec_rule_id)) + rec_rule.append(GrammarElement(GrammarElementType.ALT, 0)) + rec_rule.append(GrammarElement(GrammarElementType.END, 0)) + add_rule(state, rec_rule_id, rec_rule) + last_rec_rule_id = rec_rule_id + if n_opt > 0: + out_elements.append(GrammarElement(GrammarElementType.RULE_REF, last_rec_rule_id)) + + while pos: + if pos[0] == '"': + pos = pos[1:] + last_sym_start = len(out_elements) + while not pos.startswith('"'): + if not pos: + raise ValueError("unexpected end of input") + char, pos = parse_char(pos) + out_elements.append(GrammarElement(GrammarElementType.CHAR, char)) + pos = parse_space(pos[1:], is_nested) + elif pos[0] == "[": + pos = pos[1:] + start_type = GrammarElementType.CHAR if pos[0] == "^": - pos += 1 - start_type = llama_gretype.LLAMA_GRETYPE_CHAR_NOT - last_sym_start = out_elements.size() - # while (*pos != ']') { - # auto char_pair = parse_char(pos); - # pos = char_pair.second; - # enum llama_gretype type = last_sym_start < out_elements.size() - # ? LLAMA_GRETYPE_CHAR_ALT - # : start_type; - # out_elements.push_back({type, char_pair.first}); + pos = pos[1:] + start_type = GrammarElementType.CHAR_NOT + last_sym_start = len(out_elements) while pos[0] != "]": - char_pair = parse_char(pos) # type: Tuple[int, const_char_p] - pos = char_pair[1] - type = ( - llama_gretype.LLAMA_GRETYPE_CHAR_ALT - if last_sym_start < out_elements.size() - else start_type - ) # type: llama_gretype - out_elements.push_back(LlamaGrammarElement(type, char_pair[0])) - # if (pos[0] == '-' && pos[1] != ']') { - # auto endchar_pair = parse_char(pos + 1); - # pos = endchar_pair.second; - # out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first}); - # } - # } + if not pos: + raise ValueError("unexpected end of input") + char, pos = parse_char(pos) + type = GrammarElementType.CHAR_ALT if last_sym_start < len(out_elements) else start_type + out_elements.append(GrammarElement(type, char)) if pos[0] == "-" and pos[1] != "]": - endchar_pair = parse_char(pos + 1) # type: Tuple[int, const_char_p] - pos = endchar_pair[1] - out_elements.push_back( - LlamaGrammarElement( - llama_gretype.LLAMA_GRETYPE_CHAR_RNG_UPPER, - endchar_pair[0], - ) - ) - # pos = parse_space(pos + 1, is_nested); - pos = parse_space(pos + 1, is_nested) - # } else if (is_word_char(*pos)) { // rule reference - # const char * name_end = parse_name(pos); - # uint32_t ref_rule_id = get_symbol_id(state, pos, name_end - pos); - # pos = parse_space(name_end, is_nested); - # last_sym_start = out_elements.size(); - # out_elements.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id}); - elif is_word_char(pos[0]): # rule reference - name_end = parse_name(pos) # type: const_char_p - ref_rule_id = get_symbol_id(state, pos, name_end - pos) # type: int - pos = parse_space(name_end, is_nested) - last_sym_start = out_elements.size() - out_elements.push_back( - LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_RULE_REF, ref_rule_id) - ) - # } else if (*pos == '(') { // grouping - # // parse nested alternates into synthesized rule - # pos = parse_space(pos + 1, true); - # uint32_t sub_rule_id = generate_symbol_id(state, rule_name); - # pos = parse_alternates(state, pos, rule_name, sub_rule_id, true); - # last_sym_start = out_elements.size(); - # // output reference to synthesized rule - # out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); - # if (*pos != ')') { - # throw std::runtime_error(std::string("expecting ')' at ") + pos); - # } - # pos = parse_space(pos + 1, is_nested); - elif pos[0] == "(": # grouping - # parse nested alternates into synthesized rule - pos = parse_space(pos + 1, True) - sub_rule_id = generate_symbol_id(state, rule_name) # type: int - pos = parse_alternates(state, pos, rule_name, sub_rule_id, True) - last_sym_start = out_elements.size() - # output reference to synthesized rule - out_elements.push_back( - LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_RULE_REF, sub_rule_id) - ) + if not pos[1]: + raise ValueError("unexpected end of input") + endchar, pos = parse_char(pos[1:]) + out_elements.append(GrammarElement(GrammarElementType.CHAR_RNG_UPPER, endchar)) + pos = parse_space(pos[1:], is_nested) + elif pos and is_word_char(pos[0]): + name, rest = parse_name(pos) + ref_rule_id = get_symbol_id(state, name) + pos = parse_space(rest, is_nested) + last_sym_start = len(out_elements) + out_elements.append(GrammarElement(GrammarElementType.RULE_REF, ref_rule_id)) + elif pos.startswith("("): + pos = parse_space(pos[1:], newline_ok=True) + sub_rule_id = generate_symbol_id(state, rule_name) + pos = parse_alternates(state, pos, rule_name, sub_rule_id, is_nested=True) + last_sym_start = len(out_elements) + out_elements.append(GrammarElement(GrammarElementType.RULE_REF, sub_rule_id)) if pos[0] != ")": - raise RuntimeError("expecting ')' at " + str(pos)) - pos = parse_space(pos + 1, is_nested) - # } else if (*pos == '*' || *pos == '+' || *pos == '?') { // repetition operator - # if (last_sym_start == out_elements.size()) { - # throw std::runtime_error(std::string("expecting preceeding item to */+/? at ") + pos); - # } - elif pos[0] in ("*", "+", "?"): # repetition operator - if last_sym_start == out_elements.size(): - raise RuntimeError("expecting preceding item to */+/? at " + str(pos)) - # // apply transformation to previous symbol (last_sym_start to end) according to - # // rewrite rules: - # // S* --> S' ::= S S' | - # // S+ --> S' ::= S S' | S - # // S? --> S' ::= S | - # uint32_t sub_rule_id = generate_symbol_id(state, rule_name); - # std::vector sub_rule; - # // add preceding symbol to generated rule - # sub_rule.insert( - # sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end()); - sub_rule_id = generate_symbol_id(state, rule_name) # type: int - sub_rule = std.vector[ - LlamaGrammarElement - ]() # type: std.vector[LlamaGrammarElement] - sub_rule.insert( - sub_rule.end(), - out_elements.begin() + last_sym_start, - out_elements.end(), - ) - # if (*pos == '*' || *pos == '+') { - # // cause generated rule to recurse - # sub_rule.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); - # } - # // mark start of alternate def - # sub_rule.push_back({LLAMA_GRETYPE_ALT, 0}); - if pos[0] in ("*", "+"): - sub_rule.push_back( - LlamaGrammarElement( - llama_gretype.LLAMA_GRETYPE_RULE_REF, sub_rule_id - ) - ) - sub_rule.push_back(LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_ALT, 0)) - # if (*pos == '+') { - # // add preceding symbol as alternate only for '+' (otherwise empty) - # sub_rule.insert( - # sub_rule.end(), out_elements.begin() + last_sym_start, out_elements.end()); - # } - # sub_rule.push_back({LLAMA_GRETYPE_END, 0}); - # add_rule(state, sub_rule_id, sub_rule); - # // in original rule, replace previous symbol with reference to generated rule - # out_elements.resize(last_sym_start); - # out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); - # pos = parse_space(pos + 1, is_nested); - if pos[0] == "+": - # add preceding symbol as alternate only for '+' (otherwise empty) - sub_rule.insert( - sub_rule.end(), - out_elements.begin() + last_sym_start, - out_elements.end(), - ) - sub_rule.push_back(LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_END, 0)) - add_rule(state, sub_rule_id, sub_rule) - # in original rule, replace previous symbol with reference to generated rule - out_elements.resize(last_sym_start) - out_elements.push_back( - LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_RULE_REF, sub_rule_id) - ) - pos = parse_space(pos + 1, is_nested) - # } else { - # break; - # } + raise ValueError(f"expecting ')' at {pos}") + pos = parse_space(pos[1:], is_nested) + elif pos.startswith("."): + last_sym_start = len(out_elements) + out_elements.append(GrammarElement(GrammarElementType.CHAR_ANY, 0)) + pos = parse_space(pos[1:], is_nested) + elif pos.startswith("*"): + pos = parse_space(pos[1:], is_nested) + handle_repetitions(0, -1) + elif pos.startswith("+"): + pos = parse_space(pos[1:], is_nested) + handle_repetitions(1, -1) + elif pos.startswith("?"): + pos = parse_space(pos[1:], is_nested) + handle_repetitions(0, 1) + elif pos.startswith("{"): + pos = parse_space(pos[1:], is_nested) + + if not is_digit_char(pos): + raise ValueError(f"expecting an int at {pos}") + min_times, pos = parse_int(pos) + pos = parse_space(pos, is_nested) + + max_times = -1 + + if pos[0] == "}": + max_times = min_times + pos = parse_space(pos[1:], is_nested) + elif pos[0] == ",": + pos = parse_space(pos[1:], is_nested) + + if is_digit_char(pos): + max_times, pos = parse_int(pos) + pos = parse_space(pos, is_nested) + + if pos[0] != "}": + raise ValueError("expecting '}' at {}".format(pos)) + + pos = parse_space(pos[1:], is_nested) + else: + raise ValueError(f"expecting ',' at {pos}") + handle_repetitions(min_times, max_times) else: break - # } - # return pos; - # } return pos @@ -919,39 +584,32 @@ def parse_sequence( # add_rule(state, rule_id, rule); # return pos; # } -def parse_alternates( - state: parse_state, - src: const_char_p, - rule_name: str, - rule_id: int, - is_nested: bool, -) -> const_char_p: - rule = std.vector() # type: std.vector[LlamaGrammarElement] - pos = parse_sequence(state, src, rule_name, rule, is_nested) # type: const_char_p - while pos[0] == "|": - rule.push_back(LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_ALT, 0)) - pos = parse_space(pos + 1, True) +def parse_alternates(state: ParseState, src: str, rule_name: str, rule_id: int, is_nested: bool) -> str: + rule = [] + pos = parse_sequence(state, src, rule_name, rule, is_nested) + while pos.startswith("|"): + rule.append(GrammarElement(GrammarElementType.ALT, 0)) + pos = parse_space(pos[1:], newline_ok=True) pos = parse_sequence(state, pos, rule_name, rule, is_nested) - rule.push_back(LlamaGrammarElement(llama_gretype.LLAMA_GRETYPE_END, 0)) + rule.append(GrammarElement(GrammarElementType.END, 0)) add_rule(state, rule_id, rule) return pos -# const char * parse_rule(parse_state & state, const char * src) { +# static const char * parse_rule(parse_state & state, const char * src) { # const char * name_end = parse_name(src); # const char * pos = parse_space(name_end, false); # size_t name_len = name_end - src; # uint32_t rule_id = get_symbol_id(state, src, name_len); # const std::string name(src, name_len); - +# # if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) { # throw std::runtime_error(std::string("expecting ::= at ") + pos); # } # pos = parse_space(pos + 3, true); - +# # pos = parse_alternates(state, pos, name, rule_id, false); - - +# # if (*pos == '\r') { # pos += pos[1] == '\n' ? 2 : 1; # } else if (*pos == '\n') { @@ -961,26 +619,26 @@ def parse_alternates( # } # return parse_space(pos, true); # } -def parse_rule(state: parse_state, src: const_char_p) -> const_char_p: - name_end = parse_name(src) # type: const_char_p - pos = parse_space(name_end, False) # type: const_char_p - name_len = name_end - src # type: int - rule_id = get_symbol_id(state, src, name_len) # type: int - name = std.string(src, name_len) # type: str - - if not (pos[0] == ":" and pos[1] == ":" and pos[2] == "="): - raise RuntimeError("expecting ::= at " + str(pos)) - - pos = parse_space(pos + 3, True) # type: const_char_p - pos = parse_alternates(state, pos, name, rule_id, False) # type: const_char_p - - if pos[0] == "\r": - pos += 2 if pos[1] == "\n" else 1 - elif pos[0] == "\n": - pos += 1 - elif pos[0]: - raise RuntimeError("expecting newline or end at " + str(pos)) - return parse_space(pos, True) +def parse_rule(state: ParseState, src: str) -> str: + pos = src + name, pos = parse_name(pos) + pos = parse_space(pos, newline_ok=False) + rule_id = get_symbol_id(state, name) + + if not pos.startswith("::="): + raise ValueError(f"expecting ::= at {pos}") + + pos = parse_space(pos[3:], newline_ok=True) + + pos = parse_alternates(state, pos, name, rule_id, is_nested=False) + + if pos.startswith("\r"): + pos = pos[2:] if pos[1] == "\n" else pos[1:] + elif pos.startswith("\n"): + pos = pos[1:] + elif pos: + raise ValueError(f"expecting newline or end at {pos}") + return parse_space(pos, newline_ok=True) # parse_state parse(const char * src) { @@ -990,204 +648,255 @@ def parse_rule(state: parse_state, src: const_char_p) -> const_char_p: # while (*pos) { # pos = parse_rule(state, pos); # } +# // Validate the state to ensure that all rules are defined +# for (const auto & rule : state.rules) { +# for (const auto & elem : rule) { +# if (elem.type == LLAMA_GRETYPE_RULE_REF) { +# // Ensure that the rule at that location exists +# if (elem.value >= state.rules.size() || state.rules[elem.value].empty()) { +# // Get the name of the rule that is missing +# for (const auto & kv : state.symbol_ids) { +# if (kv.second == elem.value) { +# throw std::runtime_error("Undefined rule identifier '" + kv.first + "'"); +# } +# } +# } +# } +# } +# } # return state; # } catch (const std::exception & err) { # fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what()); # return parse_state(); # } # } -def parse(src: const_char_p) -> parse_state: - try: - state = parse_state() # type: parse_state - pos = parse_space(src, True) # type: const_char_p - while pos[0]: - pos = parse_rule(state, pos) - return state - except Exception as err: - print(f"{parse.__name__}: error parsing grammar: {err}") - return parse_state() - - -# void print_grammar_char(FILE * file, uint32_t c) { -# if (0x20 <= c && c <= 0x7f) { -# fprintf(file, "%c", static_cast(c)); -# } else { -# // cop out of encoding UTF-8 -# fprintf(file, "", c); -# } -# } -def print_grammar_char(file: TextIO, c: int) -> None: - if 0x20 <= c and c <= 0x7F: - file.write(chr(c)) - else: - # cop out of encoding UTF-8 - file.write(f"") - - -# bool is_char_element(llama_grammar_element elem) { +def parse(src: str) -> ParseState: + state = ParseState() + pos = src + pos = parse_space(pos, newline_ok=True) + while pos: + pos = parse_rule(state, pos) + # validate + for rule in state.rules: + for elem in rule: + if elem.type == GrammarElementType.RULE_REF: + if elem.value >= len(state.rules) or not state.rules[elem.value]: + for k, v in state.symbol_ids.items(): + if v == elem.value: + raise ValueError(f"Undefined rule identifier '{k}'") + return state + + +# static bool is_char_element(llama_grammar_element elem) { # switch (elem.type) { # case LLAMA_GRETYPE_CHAR: return true; # case LLAMA_GRETYPE_CHAR_NOT: return true; # case LLAMA_GRETYPE_CHAR_ALT: return true; # case LLAMA_GRETYPE_CHAR_RNG_UPPER: return true; +# case LLAMA_GRETYPE_CHAR_ANY: return true; # default: return false; # } # } -def is_char_element(elem: LlamaGrammarElement) -> bool: +def is_char_element(elem: GrammarElement) -> bool: return elem.type in ( - llama_gretype.LLAMA_GRETYPE_CHAR, - llama_gretype.LLAMA_GRETYPE_CHAR_NOT, - llama_gretype.LLAMA_GRETYPE_CHAR_ALT, - llama_gretype.LLAMA_GRETYPE_CHAR_RNG_UPPER, + GrammarElementType.CHAR, + GrammarElementType.CHAR_NOT, + GrammarElementType.CHAR_ALT, + GrammarElementType.CHAR_RNG_UPPER, + GrammarElementType.CHAR_ANY ) -# void print_rule( +def print_grammar_char(file: typing.TextIO, c: int) -> None: + if 0x20 <= c <= 0x7f: + print(chr(c), end="", file=file) + else: + print(f"", end="", file=file) + + +# static void print_rule( # FILE * file, # uint32_t rule_id, # const std::vector & rule, # const std::map & symbol_id_names) { +# if (rule.empty() || rule.back().type != LLAMA_GRETYPE_END) { +# throw std::runtime_error( +# "malformed rule, does not end with LLAMA_GRETYPE_END: " + std::to_string(rule_id)); +# } +# fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str()); +# for (size_t i = 0, end = rule.size() - 1; i < end; i++) { +# llama_grammar_element elem = rule[i]; +# switch (elem.type) { +# case LLAMA_GRETYPE_END: +# throw std::runtime_error( +# "unexpected end of rule: " + std::to_string(rule_id) + "," + +# std::to_string(i)); +# case LLAMA_GRETYPE_ALT: +# fprintf(file, "| "); +# break; +# case LLAMA_GRETYPE_RULE_REF: +# fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str()); +# break; +# case LLAMA_GRETYPE_CHAR: +# fprintf(file, "["); +# print_grammar_char(file, elem.value); +# break; +# case LLAMA_GRETYPE_CHAR_NOT: +# fprintf(file, "[^"); +# print_grammar_char(file, elem.value); +# break; +# case LLAMA_GRETYPE_CHAR_RNG_UPPER: +# if (i == 0 || !is_char_element(rule[i - 1])) { +# throw std::runtime_error( +# "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " + +# std::to_string(rule_id) + "," + std::to_string(i)); +# } +# fprintf(file, "-"); +# print_grammar_char(file, elem.value); +# break; +# case LLAMA_GRETYPE_CHAR_ALT: +# if (i == 0 || !is_char_element(rule[i - 1])) { +# throw std::runtime_error( +# "LLAMA_GRETYPE_CHAR_ALT without preceding char: " + +# std::to_string(rule_id) + "," + std::to_string(i)); +# } +# print_grammar_char(file, elem.value); +# break; +# case LLAMA_GRETYPE_CHAR_ANY: +# fprintf(file, "."); +# break; +# } +# if (is_char_element(elem)) { +# switch (rule[i + 1].type) { +# case LLAMA_GRETYPE_CHAR_ALT: +# case LLAMA_GRETYPE_CHAR_RNG_UPPER: +# case LLAMA_GRETYPE_CHAR_ANY: +# break; +# default: +# fprintf(file, "] "); +# } +# } +# } +# fprintf(file, "\n"); +# } def print_rule( - file: TextIO, + file: typing.TextIO, rule_id: int, - rule: std.vector[LlamaGrammarElement], - symbol_id_names: std.map[int, str], + rule: typing.List[GrammarElement], + symbol_id_names: typing.Dict[int, str], ) -> None: - # if (rule.empty() || rule.back().type != LLAMA_GRETYPE_END) { - # throw std::runtime_error( - # "malformed rule, does not end with LLAMA_GRETYPE_END: " + std::to_string(rule_id)); - # } - # fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str()); - if rule.empty() or rule.back().type != llama_gretype.LLAMA_GRETYPE_END: - raise RuntimeError( - "malformed rule, does not end with LLAMA_GRETYPE_END: " + str(rule_id) - ) - print(f"{symbol_id_names.at(rule_id)} ::=", file=file, end=" ") - # for (size_t i = 0, end = rule.size() - 1; i < end; i++) { - # llama_grammar_element elem = rule[i]; - # switch (elem.type) { - # case LLAMA_GRETYPE_END: - # throw std::runtime_error( - # "unexpected end of rule: " + std::to_string(rule_id) + "," + - # std::to_string(i)); - # case LLAMA_GRETYPE_ALT: - # fprintf(file, "| "); - # break; - # case LLAMA_GRETYPE_RULE_REF: - # fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str()); - # break; - # case LLAMA_GRETYPE_CHAR: - # fprintf(file, "["); - # print_grammar_char(file, elem.value); - # break; - # case LLAMA_GRETYPE_CHAR_NOT: - # fprintf(file, "[^"); - # print_grammar_char(file, elem.value); - # break; - # case LLAMA_GRETYPE_CHAR_RNG_UPPER: - # if (i == 0 || !is_char_element(rule[i - 1])) { - # throw std::runtime_error( - # "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " + - # std::to_string(rule_id) + "," + std::to_string(i)); - # } - # fprintf(file, "-"); - # print_grammar_char(file, elem.value); - # break; - # case LLAMA_GRETYPE_CHAR_ALT: - # if (i == 0 || !is_char_element(rule[i - 1])) { - # throw std::runtime_error( - # "LLAMA_GRETYPE_CHAR_ALT without preceding char: " + - # std::to_string(rule_id) + "," + std::to_string(i)); - # } - # print_grammar_char(file, elem.value); - # break; - # } + if not rule or rule[-1].type != GrammarElementType.END: + raise ValueError(f"malformed rule, does not end with LLAMA_GRETYPE_END: {rule_id}") + + print(f"{symbol_id_names[rule_id]} ::=", end=" ", file=file) + for i, elem in enumerate(rule[:-1]): - case = elem.type # type: llama_gretype - if case is llama_gretype.LLAMA_GRETYPE_END: - raise RuntimeError("unexpected end of rule: " + str(rule_id) + "," + str(i)) - elif case is llama_gretype.LLAMA_GRETYPE_ALT: - print("| ", file=file, end="") - elif case is llama_gretype.LLAMA_GRETYPE_RULE_REF: - print(f"{symbol_id_names.at(elem.value)} ", file=file, end="") - elif case is llama_gretype.LLAMA_GRETYPE_CHAR: - print("[", file=file, end="") + if elem.type == GrammarElementType.END: + raise ValueError(f"unexpected end of rule: {rule_id}, {i}") + if elem.type == GrammarElementType.ALT: + print("| ", end="", file=file) + elif elem.type == GrammarElementType.RULE_REF: + print(f"{symbol_id_names[elem.value]} ", end="", file=file) + elif elem.type == GrammarElementType.CHAR: + print("[", end="", file=file) print_grammar_char(file, elem.value) - elif case is llama_gretype.LLAMA_GRETYPE_CHAR_NOT: - print("[^", file=file, end="") + elif elem.type == GrammarElementType.CHAR_NOT: + print("[^", end="", file=file) print_grammar_char(file, elem.value) - elif case is llama_gretype.LLAMA_GRETYPE_CHAR_RNG_UPPER: + elif elem.type == GrammarElementType.CHAR_RNG_UPPER: if i == 0 or not is_char_element(rule[i - 1]): - raise RuntimeError( - "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " - + str(rule_id) - + "," - + str(i) - ) - print("-", file=file, end="") + raise ValueError(f"LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: {rule_id}, {i}") + print(f"-", end="", file=file) print_grammar_char(file, elem.value) - elif case is llama_gretype.LLAMA_GRETYPE_CHAR_ALT: + elif elem.type == GrammarElementType.CHAR_ALT: if i == 0 or not is_char_element(rule[i - 1]): - raise RuntimeError( - "LLAMA_GRETYPE_CHAR_ALT without preceding char: " - + str(rule_id) - + "," - + str(i) - ) + raise ValueError(f"LLAMA_GRETYPE_CHAR_ALT without preceding char: {rule_id}, {i}") print_grammar_char(file, elem.value) - # if (is_char_element(elem)) { - # switch (rule[i + 1].type) { - # case LLAMA_GRETYPE_CHAR_ALT: - # case LLAMA_GRETYPE_CHAR_RNG_UPPER: - # break; - # default: - # fprintf(file, "] "); + elif elem.type == GrammarElementType.CHAR_ANY: + print(".", end="", file=file) if is_char_element(elem): - if rule[i + 1].type in ( - llama_gretype.LLAMA_GRETYPE_CHAR_ALT, - llama_gretype.LLAMA_GRETYPE_CHAR_RNG_UPPER, - ): - pass - else: - print("] ", file=file, end="") - # } - # } - # } - # fprintf(file, "\n"); - # } + if rule[i + 1].type in (GrammarElementType.CHAR_ALT, GrammarElementType.CHAR_RNG_UPPER, GrammarElementType.CHAR_ANY): + continue + print("] ", end="", file=file) print(file=file) -# void print_grammar(FILE * file, const parse_state & state) { -# try { -# std::map symbol_id_names; -# for (auto kv : state.symbol_ids) { -# symbol_id_names[kv.second] = kv.first; -# } -# for (size_t i = 0, end = state.rules.size(); i < end; i++) { -# // fprintf(file, "%zu: ", i); -# // print_rule_binary(file, state.rules[i]); -# print_rule(file, i, state.rules[i], symbol_id_names); -# // fprintf(file, "\n"); -# } -# } catch (const std::exception & err) { -# fprintf(stderr, "\n%s: error printing grammar: %s\n", __func__, err.what()); -# } -# } -def print_grammar(file: TextIO, state: parse_state) -> None: +def print_grammar(file: typing.TextIO, state: ParseState) -> None: try: - symbol_id_names = std.map() # type: std.map[int, str] - for kv in state.symbol_ids.items(): - symbol_id_names[kv[1]] = kv[0] - + symbol_id_names = {v: k for k, v in state.symbol_ids.items()} for i, rule in enumerate(state.rules): print_rule(file, i, rule, symbol_id_names) except Exception as err: - print( - f"{print_grammar.__name__}: error printing grammar: {err}", - file=sys.stderr, + print(f"\nerror printing grammar: {err}", file=file) + raise err + + +class LlamaGrammar: + def __init__(self, parse_state: ParseState): + self.parse_state = parse_state + + self._grammar_rules = parse_state.rules + self._n_rules = len(self._grammar_rules) + self._start_rule_index = parse_state.symbol_ids["root"] + + self._element_lists = [ + [ + llama_cpp.llama_grammar_element(ctypes.c_int(elem.type), ctypes.c_uint32(elem.value)) + for elem in subvector + ] + for subvector in self._grammar_rules + ] + + # Step 2: Convert each list to llama_grammar_element array and get pointer + self._element_arrays = [ + (llama_cpp.llama_grammar_element * len(sublist))(*sublist) + for sublist in self._element_lists + ] + + # Step 3: Get pointer of each array + self._element_array_pointers = [ + ctypes.cast(subarray, llama_cpp.llama_grammar_element_p) for subarray in self._element_arrays + ] + + # Step 4: Make array of these pointers and get its pointer + self._rules = (llama_cpp.llama_grammar_element_p * len(self._element_array_pointers))( + *self._element_array_pointers ) + self.grammar = None + self._init_grammar() + + + def _init_grammar(self): + grammar = llama_cpp.llama_grammar_init( + self._rules, ctypes.c_size_t(self._n_rules), ctypes.c_size_t(self._start_rule_index) + ) + + if grammar is None: + raise ValueError("Failed to create grammar") + + self.grammar = grammar + + def __del__(self): + if self.grammar is not None: + llama_cpp.llama_grammar_free(self.grammar) + self.grammar = None + + def reset(self): + if self.grammar is not None: + llama_cpp.llama_grammar_free(self.grammar) + self._init_grammar() + + @classmethod + def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": + parsed_grammar = parse(grammar) + print_grammar(file=sys.stdout, state=parsed_grammar) + return cls(parsed_grammar) + + @classmethod + def from_json_schema(cls, json_schema: str, verbose: bool = True) -> "LlamaGrammar": + return cls.from_string(json_schema_to_gbnf(json_schema), verbose=verbose) + """llama.cpp gbnf rules from vendor/llama.cpp/grammars""" @@ -1358,12 +1067,13 @@ def print_grammar(file: TextIO, state: parse_state) -> None: string ::= "\"" ( [^"\\\x7F\x00-\x1F] | - "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes + "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) # escapes )* "\"" ws -number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws +number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [0-9] [1-9]{0,15})? ws -ws ::= ([ \t\n] ws)? +# Optional space: by convention, applied in this grammar after literal chars when allowed +ws ::= | " " | "\n" [ \t]{0,20} """ LIST_GBNF = r""" From 18f58fe39fceac821b9cf88ae40355292ade2a8e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 6 Aug 2024 20:21:18 -0400 Subject: [PATCH 272/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 4 ++-- vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d598cf9f16..643a321f5f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -944,7 +944,7 @@ class llama_context_params(ctypes.Structure): # int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() # enum llama_ftype ftype; // quantize to this llama_ftype # enum ggml_type output_tensor_type; // output tensor type -# enum ggml_type token_embedding_type; // itoken embeddings tensor type +# enum ggml_type token_embedding_type; // token embeddings tensor type # bool allow_requantize; // allow quantizing non-f32/f16 tensors # bool quantize_output_tensor; // quantize output.weight # bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored @@ -960,7 +960,7 @@ class llama_model_quantize_params(ctypes.Structure): nthread (int): number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() ftype (int): quantize to this llama_ftype output_tensor_type (int): output tensor type - token_embedding_type (int): itoken embeddings tensor type + token_embedding_type (int): token embeddings tensor type allow_requantize (bool): allow quantizing non-f32/f16 tensors quantize_output_tensor (bool): quantize output.weight only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 398ede5efe..725e3d9437 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 398ede5efeb07b9adf9fbda7ea63f630d476a792 +Subproject commit 725e3d94379d5b619c027347308bccf2e0ead89f From ce6466f0eeb5c28d90b3cd26fb5a12959f45138c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 6 Aug 2024 20:23:57 -0400 Subject: [PATCH 273/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc25442b5..6630f1958f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.86] + +- feat: Update llama.cpp to ggerganov/llama.cpp@398ede5efeb07b9adf9fbda7ea63f630d476a792 +- feat: Ported back new grammar changes from C++ to Python implementation by @ExtReMLapin in (#1637) +- fix: llama_grammar_accept_token arg order by @tc-wolf in (#1649) + ## [0.2.85] - feat: Update llama.cpp to ggerganov/llama.cpp@398ede5efeb07b9adf9fbda7ea63f630d476a792 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 97c8d8174f..7b672c046a 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.85" +__version__ = "0.2.86" From 198f47dc1bd202fd2b71b29e041a9f33fe40bfad Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 7 Aug 2024 09:33:05 -0400 Subject: [PATCH 274/561] feat(ci): Re-build wheel index automatically when releases are created --- .github/workflows/generate-index-from-release.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 500c4613ca..662cb6f65d 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -1,9 +1,11 @@ name: Wheels Index on: - # Trigger on any new release - release: - types: [published] + # Trigger on new release + workflow_run: + workflows: ["Release", "Build Wheels (CUDA)", "Build Wheels (Metal)"] + types: + - completed # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From a07b337241264ab3b5cadf14b8fd81d1729b86e8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 7 Aug 2024 09:33:45 -0400 Subject: [PATCH 275/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 725e3d9437..be55695eff 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 725e3d94379d5b619c027347308bccf2e0ead89f +Subproject commit be55695eff44784a141a863f273661a6bce63dfc From 9cad5714ae6e7c250af8d0bbb179f631368c928b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 7 Aug 2024 09:50:37 -0400 Subject: [PATCH 276/561] fix: Include all llama.cpp source files and subdirectories --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8345cb1f09..7aebb74f55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ wheel.packages = ["llama_cpp"] cmake.verbose = true cmake.minimum-version = "3.21" minimum-version = "0.5.1" -sdist.include = [".git", "vendor/llama.cpp/.git"] +sdist.include = [".git", "vendor/llama.cpp/*"] [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From 84321162f8cad3e1caf552d7f0b18bc869e2cebf Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 7 Aug 2024 09:52:11 -0400 Subject: [PATCH 277/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6630f1958f..e5c7f01c16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.87] + +- feat: Update llama.cpp to ggerganov/llama.cpp@be55695eff44784a141a863f273661a6bce63dfc +- fix: Include all llama.cpp source files and subdirectories by @abetlen in 9cad5714ae6e7c250af8d0bbb179f631368c928b +- feat(ci): Re-build wheel index automatically when releases are created by @abetlen in 198f47dc1bd202fd2b71b29e041a9f33fe40bfad + ## [0.2.86] - feat: Update llama.cpp to ggerganov/llama.cpp@398ede5efeb07b9adf9fbda7ea63f630d476a792 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 7b672c046a..37d91cd542 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.86" +__version__ = "0.2.87" From e966f3bcc1296fae7ba0d490eb12bf8ce3ced9fb Mon Sep 17 00:00:00 2001 From: Xu Song Date: Wed, 7 Aug 2024 22:41:39 +0800 Subject: [PATCH 278/561] feat: Add more detailed log for prefix-match (#1659) Co-authored-by: Andrei --- llama_cpp/llama.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 0cb5ca2fc4..3fa3b563a8 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -777,11 +777,12 @@ def generate( else: break if longest_prefix > 0: - if self.verbose: - print("Llama.generate: prefix-match hit", file=sys.stderr) reset = False tokens = tokens[longest_prefix:] self.n_tokens = longest_prefix + if self.verbose: + print(f"Llama.generate: {longest_prefix} prefix-match hit, " + f"remaining {len(tokens)} prompt tokens to eval", file=sys.stderr) # Reset the model state if reset: From 131db407700df32efd0ced55c75c0bd44def9b1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2024 10:42:27 -0400 Subject: [PATCH 279/561] chore(deps): bump pypa/cibuildwheel from 2.19.2 to 2.20.0 (#1657) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.19.2 to 2.20.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.19.2...v2.20.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 4 ++-- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index eeeda803db..785f388479 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -29,7 +29,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v2.20.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -56,7 +56,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v2.20.0 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 47c1c3cb52..dd8b9c8fce 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -30,7 +30,7 @@ jobs: python -m pip install -e .[all] - name: Build wheels - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v2.20.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" From 5e39a854631c0a8442374d47acb90310fa751501 Mon Sep 17 00:00:00 2001 From: Benedikt Heidrich Date: Wed, 7 Aug 2024 16:52:50 +0200 Subject: [PATCH 280/561] feat: Enable recursive search of HFFS.ls when using `from_pretrained` (#1656) Enable rercursive ls for hffs. Otherwise models in subfolder wouldn't be found. Co-authored-by: Andrei --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 3fa3b563a8..49608908ba 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2160,7 +2160,7 @@ def from_pretrained( files = [ file["name"] if isinstance(file, dict) else file - for file in hffs.ls(repo_id) + for file in hffs.ls(repo_id, recursive=True)) ] # split each file into repo_id, subfolder, filename From c5de5d3e882c6ec5abf346d34cf329b4a8a35916 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 Aug 2024 11:32:33 -0400 Subject: [PATCH 281/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index be55695eff..afd27f01fe 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit be55695eff44784a141a863f273661a6bce63dfc +Subproject commit afd27f01fe832ece3d07ef03b7d34a9e80c4a895 From 0998ea0deea076a547d54bd598d6b413b588ee2b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 Aug 2024 11:34:20 -0400 Subject: [PATCH 282/561] fix: grammar prints on each call. Closes #1666 --- llama_cpp/llama_grammar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index e60817e1d7..bc2a9cc45d 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -890,7 +890,8 @@ def reset(self): @classmethod def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": parsed_grammar = parse(grammar) - print_grammar(file=sys.stdout, state=parsed_grammar) + if verbose: + print_grammar(file=sys.stdout, state=parsed_grammar) return cls(parsed_grammar) @classmethod From 7aaf7015800b98fd246b4c747e59d766318dec71 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 Aug 2024 11:35:31 -0400 Subject: [PATCH 283/561] fix: typo --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 49608908ba..a37a62aef5 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2160,7 +2160,7 @@ def from_pretrained( files = [ file["name"] if isinstance(file, dict) else file - for file in hffs.ls(repo_id, recursive=True)) + for file in hffs.ls(repo_id, recursive=True) ] # split each file into repo_id, subfolder, filename From 45de9d5fa6ca445a6e0ef23ef5e7d15348808ab2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 10 Aug 2024 18:33:33 -0400 Subject: [PATCH 284/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 8 ++++++++ vendor/llama.cpp | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 643a321f5f..1a8424aa1b 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1505,6 +1505,14 @@ def llama_model_has_encoder(model: llama_model_p, /) -> bool: ... +# // Returns true if the model contains a decoder that requires llama_decode() call +# LLAMA_API bool llama_model_has_decoder(const struct llama_model * model); +@ctypes_function("llama_model_has_decoder", [llama_model_p_ctypes], ctypes.c_bool) +def llama_model_has_decoder(model: llama_model_p, /) -> bool: + """Returns true if the model contains a decoder that requires llama_decode() call""" + ... + + # // For encoder-decoder models, this function returns id of the token that must be provided # // to the decoder to start generating output sequence. For other models, it returns -1. # LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model); diff --git a/vendor/llama.cpp b/vendor/llama.cpp index afd27f01fe..6e02327e8b 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit afd27f01fe832ece3d07ef03b7d34a9e80c4a895 +Subproject commit 6e02327e8b7837358e0406bf90a4632e18e27846 From 4244151d7a27a29d8ac56dfbdc7ae10b68defaa5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 12 Aug 2024 02:29:30 -0400 Subject: [PATCH 285/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6e02327e8b..4134999e01 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6e02327e8b7837358e0406bf90a4632e18e27846 +Subproject commit 4134999e01f31256b15342b41c4de9e2477c4a6c From 95a1533172b7d31bf2fa9a54b4f2a9776fbd4c45 Mon Sep 17 00:00:00 2001 From: ExtReMLapin <3909752+ExtReMLapin@users.noreply.github.com> Date: Mon, 12 Aug 2024 20:12:54 +0200 Subject: [PATCH 286/561] fix: Added back from_file method to LlamaGrammar (#1673) --- llama_cpp/llama_grammar.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index bc2a9cc45d..4cd52c2d5c 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -1,6 +1,7 @@ """Python implementation of llama grammar parser directly translated from C++ source file in vendor/llama.cpp/common/grammar-parser.cpp.""" # flake8: noqa +from pathlib import Path import sys import ctypes import enum @@ -893,6 +894,23 @@ def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": if verbose: print_grammar(file=sys.stdout, state=parsed_grammar) return cls(parsed_grammar) + + @classmethod + def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGrammar": + try: + with open(file) as f: + grammar = f.read() + except Exception as err: + raise Exception( + f"{cls.from_file.__name__}: error reading grammar file: {err}" + ) + + if grammar: + return cls.from_string(grammar, verbose=verbose) + + raise ValueError( + f"{cls.from_file.__name__}: error parsing grammar file: params_grammer is empty" + ) @classmethod def from_json_schema(cls, json_schema: str, verbose: bool = True) -> "LlamaGrammar": From 9bab46f643570a29d356337eb5ab995af8862aea Mon Sep 17 00:00:00 2001 From: Laurent Sorber Date: Mon, 12 Aug 2024 20:19:26 +0200 Subject: [PATCH 287/561] fix: only print 'cache saved' in verbose mode (#1668) Co-authored-by: Andrei --- llama_cpp/llama.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index a37a62aef5..b780ef6c67 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1523,7 +1523,8 @@ def logit_bias_processor( if self.verbose: print("Llama._create_completion: cache save", file=sys.stderr) self.cache[prompt_tokens + completion_tokens] = self.save_state() - print("Llama._create_completion: cache saved", file=sys.stderr) + if self.verbose: + print("Llama._create_completion: cache saved", file=sys.stderr) return if self.cache: From 8ed663b867e6ba56dc03096984675dd35667f2b7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 12 Aug 2024 14:19:56 -0400 Subject: [PATCH 288/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4134999e01..fc4ca27b25 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4134999e01f31256b15342b41c4de9e2477c4a6c +Subproject commit fc4ca27b25464a11b3b86c9dbb5b6ed6065965c2 From fc19cc783802ed95eb4c8be358537b12c1fec78f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 13 Aug 2024 05:22:04 -0400 Subject: [PATCH 289/561] chore: Bump version --- CHANGELOG.md | 9 +++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5c7f01c16..f7738e1a95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.88] + +- feat: Update llama.cpp to ggerganov/llama.cpp@fc4ca27b25464a11b3b86c9dbb5b6ed6065965c2 +- fix: only print 'cache saved' in verbose mode by @lsorber in #1668 +- fix: Added back from_file method to LlamaGrammar by @ExtReMLapin in #1673 +- fix: grammar prints on each call by @abetlen in 0998ea0deea076a547d54bd598d6b413b588ee2b +- feat: Enable recursive search of HFFS.ls when using from_pretrained by @benHeidabetlen in #1656 +- feat: Add more detailed log for prefix-match by @xu-song in #1659 + ## [0.2.87] - feat: Update llama.cpp to ggerganov/llama.cpp@be55695eff44784a141a863f273661a6bce63dfc diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 37d91cd542..0030e8f7ef 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.87" +__version__ = "0.2.88" From 63d65acabc7d7402cdb85926a70b2ef81b472ac4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 15 Aug 2024 14:46:46 -0400 Subject: [PATCH 290/561] feat: Update llama.cpp --- llama_cpp/_internals.py | 8 ++++---- llama_cpp/llama.py | 4 ++-- llama_cpp/llama_cpp.py | 26 ++++++++++++-------------- vendor/llama.cpp | 2 +- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 357023e3c5..6dae88c8fc 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -179,11 +179,11 @@ def token_eot(self) -> int: assert self.model is not None return llama_cpp.llama_token_eot(self.model) - def add_bos_token(self) -> int: + def add_bos_token(self) -> bool: assert self.model is not None return llama_cpp.llama_add_bos_token(self.model) - def add_eos_token(self) -> int: + def add_eos_token(self) -> bool: assert self.model is not None return llama_cpp.llama_add_eos_token(self.model) @@ -691,8 +691,8 @@ def _detokenize_bpe(model: _LlamaModel, tokens: List[int]) -> str: def _should_add_bos(model: _LlamaModel) -> bool: assert model.model is not None add_bos = llama_cpp.llama_add_bos_token(model.model) - if add_bos != -1: - return add_bos != 0 + if add_bos: + return add_bos else: return llama_cpp.llama_vocab_type(model.model) == llama_cpp.LLAMA_VOCAB_TYPE_SPM diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b780ef6c67..29674c6d8b 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1058,13 +1058,13 @@ def _create_completion( if ( (isinstance(prompt, list) and suffix is None) - or self._model.add_bos_token() == 0 + or not self._model.add_bos_token() or bos_tokens[:1] == [-1] ): bos_tokens = [] if (isinstance(prompt, list) and suffix is None) or ( - self._model.add_eos_token() != 1 and sep_token_id == -1 + not self._model.add_eos_token() and sep_token_id == -1 ): eos_tokens = [] diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1a8424aa1b..00064f3240 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -314,6 +314,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, # LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, # LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, +# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, +# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -338,6 +340,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20 LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21 LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22 +LLAMA_VOCAB_PRE_TYPE_BLOOM = 23 +LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 # // note: these values should be synchronized with ggml_rope @@ -345,13 +349,11 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # enum llama_rope_type { # LLAMA_ROPE_TYPE_NONE = -1, # LLAMA_ROPE_TYPE_NORM = 0, -# LLAMA_ROPE_TYPE_NEOX = 2, -# LLAMA_ROPE_TYPE_GLM = 4, +# LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, # }; LLAMA_ROPE_TYPE_NONE = -1 LLAMA_ROPE_TYPE_NORM = 0 -LLAMA_ROPE_TYPE_NEOX = 2 -LLAMA_ROPE_TYPE_GLM = 4 +LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2 # enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file @@ -2741,19 +2743,15 @@ def llama_token_nl(model: llama_model_p, /) -> int: ... -# // Returns -1 if unknown, 1 for true or 0 for false. -# LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model); -@ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_int32) -def llama_add_bos_token(model: llama_model_p, /) -> int: - """Returns -1 if unknown, 1 for true or 0 for false.""" +# LLAMA_API bool llama_add_bos_token(const struct llama_model * model); +@ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_bool) +def llama_add_bos_token(model: llama_model_p, /) -> bool: ... -# // Returns -1 if unknown, 1 for true or 0 for false. -# LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model); -@ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_int32) -def llama_add_eos_token(model: llama_model_p, /) -> int: - """Returns -1 if unknown, 1 for true or 0 for false.""" +# LLAMA_API bool llama_add_eos_token(const struct llama_model * model); +@ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_bool) +def llama_add_eos_token(model: llama_model_p, /) -> bool: ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index fc4ca27b25..4b9afbbe90 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit fc4ca27b25464a11b3b86c9dbb5b6ed6065965c2 +Subproject commit 4b9afbbe9037f8a2d659097c0c7d9fce32c6494c From 78e35c4998f3209cb70c188a47cffd17c1f12bce Mon Sep 17 00:00:00 2001 From: Junpei Kawamoto Date: Thu, 15 Aug 2024 15:00:46 -0600 Subject: [PATCH 291/561] fix: missing dependencies for test (#1680) --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 7aebb74f55..ce50c673f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,10 @@ test = [ "pytest>=7.4.0", "httpx>=0.24.1", "scipy>=1.10", + "fastapi>=0.100.0", + "sse-starlette>=1.6.1", + "starlette-context>=0.3.6,<0.4", + "pydantic-settings>=2.0.1", ] dev = [ "black>=23.3.0", From 3c7501b35755205d1df70181cd46e769ec2e9790 Mon Sep 17 00:00:00 2001 From: Junpei Kawamoto Date: Thu, 15 Aug 2024 15:01:18 -0600 Subject: [PATCH 292/561] fix: Llama.close didn't free lora adapter (#1679) --- llama_cpp/llama.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 29674c6d8b..9b1a3d263b 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -198,6 +198,7 @@ def __init__( A Llama instance. """ self.verbose = verbose + self._stack = contextlib.ExitStack() set_verbose(verbose) @@ -365,8 +366,6 @@ def __init__( if not os.path.exists(model_path): raise ValueError(f"Model path does not exist: {model_path}") - self._stack = contextlib.ExitStack() - self._model = self._stack.enter_context( contextlib.closing( _LlamaModel( @@ -420,6 +419,15 @@ def __init__( raise RuntimeError( f"Failed to initialize LoRA adapter from lora path: {self.lora_path}" ) + + def free_lora_adapter(): + if self._lora_adapter is None: + return + llama_cpp.llama_lora_adapter_free(self._lora_adapter) + self._lora_adapter = None + + self._stack.callback(free_lora_adapter) + assert self._ctx.ctx is not None if llama_cpp.llama_lora_adapter_set( self._ctx.ctx, self._lora_adapter, self.lora_scale @@ -2085,14 +2093,9 @@ def pooling_type(self) -> str: def close(self) -> None: """Explicitly free the model from memory.""" - if hasattr(self,'_stack'): - if self._stack is not None: - self._stack.close() + self._stack.close() def __del__(self) -> None: - if hasattr(self,'_lora_adapter'): - if self._lora_adapter is not None: - llama_cpp.llama_lora_adapter_free(self._lora_adapter) self.close() @staticmethod From 7bf07ec51873ab96ac326bac126044039768ef4b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 16 Aug 2024 14:43:25 -0400 Subject: [PATCH 293/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 2 ++ vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 00064f3240..476713aee2 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -316,6 +316,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, # LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, # LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, +# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -342,6 +343,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22 LLAMA_VOCAB_PRE_TYPE_BLOOM = 23 LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 +LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 # // note: these values should be synchronized with ggml_rope diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4b9afbbe90..8b3befc0e2 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4b9afbbe9037f8a2d659097c0c7d9fce32c6494c +Subproject commit 8b3befc0e2ed8fb18b903735831496b8b0c80949 From a2ba73165223f8946f3e1180e3cb67d7d9236da2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 19 Aug 2024 13:36:31 -0400 Subject: [PATCH 294/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8b3befc0e2..cfac111e2b 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8b3befc0e2ed8fb18b903735831496b8b0c80949 +Subproject commit cfac111e2b3953cdb6b0126e67a2487687646971 From d7328efabf841a750259dc42089efece13efc0f5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 19 Aug 2024 13:36:49 -0400 Subject: [PATCH 295/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7738e1a95..063fe722bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.89] + +- feat: Update llama.cpp to ggerganov/llama.cpp@cfac111e2b3953cdb6b0126e67a2487687646971 +- fix: Llama.close didn't free lora adapter by @jkawamoto in #1679 +- fix: missing dependencies for test by @jkawamoto in #1680 + ## [0.2.88] - feat: Update llama.cpp to ggerganov/llama.cpp@fc4ca27b25464a11b3b86c9dbb5b6ed6065965c2 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 0030e8f7ef..e3cece2f62 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.88" +__version__ = "0.2.89" From a20f13fc3b08e7c2163d84d5da51da78ec9f48c3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 21 Aug 2024 10:11:21 -0400 Subject: [PATCH 296/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index cfac111e2b..fc54ef0d1c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit cfac111e2b3953cdb6b0126e67a2487687646971 +Subproject commit fc54ef0d1c138133a01933296d50a36a1ab64735 From 259ee151da9a569f58f6d4979e97cfd5d5bc3ecd Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 22 Aug 2024 09:17:45 -0400 Subject: [PATCH 297/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 8 ++++++++ vendor/llama.cpp | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 476713aee2..b35f56a798 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1530,6 +1530,14 @@ def llama_model_decoder_start_token(model: llama_model_p, /) -> int: ... +# // Returns true if the model is recurrent (like Mamba, RWKV, etc.) +# LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model); +@ctypes_function("llama_model_is_recurrent", [llama_model_p_ctypes], ctypes.c_bool) +def llama_model_is_recurrent(model: llama_model_p, /) -> bool: + """Returns true if the model is recurrent (like Mamba, RWKV, etc.)""" + ... + + # // Returns 0 on success # LLAMA_API uint32_t llama_model_quantize( # const char * fname_inp, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index fc54ef0d1c..1731d4238f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit fc54ef0d1c138133a01933296d50a36a1ab64735 +Subproject commit 1731d4238f9e4f925a750810e7f5480827c66dcf From 82ae7f93716c47b4ed0a809f280d5352e2fcdea9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 28 Aug 2024 00:47:48 -0400 Subject: [PATCH 298/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1731d4238f..20f1789dfb 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1731d4238f9e4f925a750810e7f5480827c66dcf +Subproject commit 20f1789dfb4e535d64ba2f523c64929e7891f428 From f70df824985d875226793b94dacc0c302a4256b2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 01:05:32 -0400 Subject: [PATCH 299/561] feat: Add MiniCPMv26 chat handler. --- llama_cpp/llama_chat_format.py | 92 +++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 25 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index ea8d07febe..3c73302964 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2707,6 +2707,31 @@ def last_image_embed_free(): def load_image(self, image_url: str) -> bytes: return self._load_image(image_url) + def _embed_image_bytes(self, image_bytes: bytes, n_threads_batch: int = 1): + if ( + self._last_image_embed is not None + and self._last_image_hash is not None + and hash(image_bytes) == self._last_image_hash + ): + return self._last_image_embed + with suppress_stdout_stderr(disable=self.verbose): + # Free the previous image embed + if self._last_image_embed is not None: + self._llava_cpp.llava_image_embed_free(self._last_image_embed) + self._last_image_embed = None + self._last_image_hash = None + embed = self._llava_cpp.llava_image_embed_make_with_bytes( + self.clip_ctx, + n_threads_batch, + (ctypes.c_uint8 * len(image_bytes)).from_buffer( + bytearray(image_bytes) + ), + len(image_bytes), + ) + self._last_image_embed = embed + self._last_image_hash = hash(image_bytes) + return embed + def __call__( self, *, @@ -2769,30 +2794,9 @@ def __call__( ) split_text = self.split_text_on_image_urls(text, image_urls) - def embed_image_bytes(image_bytes: bytes): - if ( - self._last_image_embed is not None - and self._last_image_hash is not None - and hash(image_bytes) == self._last_image_hash - ): - return self._last_image_embed - with suppress_stdout_stderr(disable=self.verbose): - # Free the previous image embed - if self._last_image_embed is not None: - self._llava_cpp.llava_image_embed_free(self._last_image_embed) - self._last_image_embed = None - self._last_image_hash = None - embed = self._llava_cpp.llava_image_embed_make_with_bytes( - self.clip_ctx, - llama.context_params.n_threads_batch, - (ctypes.c_uint8 * len(image_bytes)).from_buffer( - bytearray(image_bytes) - ), - len(image_bytes), - ) - self._last_image_embed = embed - self._last_image_hash = hash(image_bytes) - return embed + if self.verbose: + print(text, file=sys.stderr) + # Evaluate prompt llama.reset() @@ -2809,7 +2813,7 @@ def embed_image_bytes(image_bytes: bytes): llama.eval(tokens) else: image_bytes = self.load_image(value) - embed = embed_image_bytes(image_bytes) + embed = self._embed_image_bytes(image_bytes, llama.context_params.n_threads_batch) if llama.n_tokens + embed.contents.n_image_pos > llama.n_ctx(): raise ValueError( f"Prompt exceeds n_ctx: {llama.n_tokens + embed.contents.n_image_pos} > {llama.n_ctx()}" @@ -3308,6 +3312,44 @@ class Llama3VisionAlphaChatHandler(Llava15ChatHandler): Llama3VisionAlpha = Llama3VisionAlphaChatHandler +class MiniCPMv26(Llava15ChatHandler): + DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant." + + CHAT_FORMAT = ( + "{% for message in messages %}" + "{% if loop.first and messages[0]['role'] != 'system' %}" + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "{% endif %}" + "<|im_start|>{{ message['role'] }}\n" + "{% if message['content'] is iterable %}" + "{% for content in message['content'] %}" + "{% if content.type == 'image_url' %}" + "{% if content.image_url is string %}" + "{{ content.image_url }}" + "{% endif %}" + "{% if content.image_url is mapping %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + + "{% for content in message['content'] %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "{% if message['content'] is string %}" + "{{ message['content'] }}" + "{% endif %}" + "<|im_end|>\n" + "{% endfor %}" + "{% if add_generation_prompt %}" + "<|im_start|>assistant\n" + "{% endif %}" + ) + + @register_chat_completion_handler("chatml-function-calling") def chatml_function_calling( llama: llama.Llama, From e251a0bd7d05f993f19e79e3f8328e4d3b41110a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 01:26:34 -0400 Subject: [PATCH 300/561] fix: Update name to MiniCPMv26ChatHandler --- llama_cpp/llama_chat_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 3c73302964..21091bf509 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3312,7 +3312,7 @@ class Llama3VisionAlphaChatHandler(Llava15ChatHandler): Llama3VisionAlpha = Llama3VisionAlphaChatHandler -class MiniCPMv26(Llava15ChatHandler): +class MiniCPMv26ChatHandler(Llava15ChatHandler): DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant." CHAT_FORMAT = ( From c68e7fbedfff8c385167a3d5232ad6a4c12597f1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 01:51:25 -0400 Subject: [PATCH 301/561] fix: pull all gh releases for self-hosted python index --- .../generate-index-from-release.yaml | 3 + scripts/get-releases.sh | 47 +++++++ scripts/releases-to-pep-503.sh | 122 ++++++++++++------ 3 files changed, 133 insertions(+), 39 deletions(-) create mode 100755 scripts/get-releases.sh diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 662cb6f65d..51961088d7 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -35,7 +35,10 @@ jobs: - name: Setup Pages uses: actions/configure-pages@v5 - name: Build + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + ./scripts/get-releases.sh ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$' ./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$' ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' diff --git a/scripts/get-releases.sh b/scripts/get-releases.sh new file mode 100755 index 0000000000..4c904da78c --- /dev/null +++ b/scripts/get-releases.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Function to get all releases +get_all_releases() { + local page=1 + local per_page=100 + local releases="" + local new_releases + + # Prepare headers + local headers=(-H "Accept: application/vnd.github.v3+json") + if [ -n "$GITHUB_TOKEN" ]; then + headers+=(-H "Authorization: Bearer $GITHUB_TOKEN") + fi + + while true; do + response=$(curl -s "${headers[@]}" \ + "https://api.github.com/repos/abetlen/llama-cpp-python/releases?page=$page&per_page=$per_page") + + # Check if the response is valid JSON + if ! echo "$response" | jq empty > /dev/null 2>&1; then + echo "Error: Invalid response from GitHub API" >&2 + echo "Response: $response" >&2 + return 1 + fi + + new_releases=$(echo "$response" | jq -r '.[].tag_name') + if [ -z "$new_releases" ]; then + break + fi + releases="$releases $new_releases" + ((page++)) + done + + echo $releases +} + +# Get all releases and save to file +releases=$(get_all_releases) +if [ $? -ne 0 ]; then + echo "Failed to fetch releases. Please check your internet connection and try again later." >&2 + exit 1 +fi + +echo "$releases" | tr ' ' '\n' > all_releases.txt + +echo "All releases have been saved to all_releases.txt" diff --git a/scripts/releases-to-pep-503.sh b/scripts/releases-to-pep-503.sh index 44fbbf3cf1..71910efcbf 100755 --- a/scripts/releases-to-pep-503.sh +++ b/scripts/releases-to-pep-503.sh @@ -1,60 +1,104 @@ #!/bin/bash +# Enable exit on error +set -e + +# Function for logging +log_error() { + echo "ERROR: $1" >&2 +} + +log_info() { + echo "INFO: $1" +} + # Get output directory or default to index/whl/cpu output_dir=${1:-"index/whl/cpu"} -# Create output directory -mkdir -p $output_dir - -# Change to output directory -pushd $output_dir +# Get pattern from second arg or default to valid python package version pattern +pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"} -# Create an index html file -echo "" > index.html -echo "" >> index.html -echo " " >> index.html -echo " " >> index.html -echo " llama-cpp-python" >> index.html -echo "
" >> index.html -echo " " >> index.html -echo "" >> index.html -echo "" >> index.html +# Get the current directory (where the script is run from) +current_dir="$(pwd)" -# Create llama-cpp-python directory -mkdir -p llama-cpp-python +# Check if all_releases.txt exists +if [ ! -f "$current_dir/all_releases.txt" ]; then + log_error "all_releases.txt not found in the current directory." + exit 1 +fi -# Change to llama-cpp-python directory -pushd llama-cpp-python +# Create output directory +mkdir -p "$output_dir" # Create an index html file -echo "" > index.html -echo "" >> index.html -echo " " >> index.html -echo "

Links for llama-cpp-python

" >> index.html +cat << EOF > "$output_dir/index.html" + + + + + llama-cpp-python +
+ + -# Get all releases -releases=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases | jq -r .[].tag_name) +EOF -# Get pattern from second arg or default to valid python package version pattern -pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"} +# Create llama-cpp-python directory +mkdir -p "$output_dir/llama-cpp-python" + +# Create an index html file in llama-cpp-python directory +cat << EOF > "$output_dir/llama-cpp-python/index.html" + + + +

Links for llama-cpp-python

+EOF # Filter releases by pattern -releases=$(echo $releases | tr ' ' '\n' | grep -E $pattern) +releases=$(grep -E "$pattern" "$current_dir/all_releases.txt") + +# Prepare curl headers +headers=('--header' 'Accept: application/vnd.github.v3+json') +if [ -n "$GITHUB_TOKEN" ]; then + headers+=('--header' "authorization: Bearer $GITHUB_TOKEN") +fi +headers+=('--header' 'content-type: application/json') # For each release, get all assets for release in $releases; do - assets=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release | jq -r .assets) + log_info "Processing release: $release" + response=$(curl -s "${headers[@]}" \ + "https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release") + + if [ -z "$response" ]; then + log_error "Empty response from GitHub API for release $release" + continue + fi + + if ! echo "$response" | jq -e '.assets' > /dev/null 2>&1; then + log_error "Invalid or unexpected response from GitHub API for release $release" + log_error "Response: $response" + continue + fi + # Get release version from release ie v0.1.0-cu121 -> v0.1.0 - release_version=$(echo $release | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+") - echo "

$release_version

" >> index.html - for asset in $(echo $assets | jq -r .[].browser_download_url); do - if [[ $asset == *".whl" ]]; then - echo " $asset" >> index.html - echo "
" >> index.html - fi + release_version=$(echo "$release" | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+") + echo "

$release_version

" >> "$output_dir/llama-cpp-python/index.html" + + wheel_urls=$(echo "$response" | jq -r '.assets[] | select(.name | endswith(".whl")) | .browser_download_url') + if [ -z "$wheel_urls" ]; then + log_error "No wheel files found for release $release" + continue + fi + + echo "$wheel_urls" | while read -r asset; do + echo " $asset" >> "$output_dir/llama-cpp-python/index.html" + echo "
" >> "$output_dir/llama-cpp-python/index.html" done done -echo " " >> index.html -echo "" >> index.html -echo "" >> index.html +echo " " >> "$output_dir/llama-cpp-python/index.html" +echo "" >> "$output_dir/llama-cpp-python/index.html" +echo "" >> "$output_dir/llama-cpp-python/index.html" + +log_info "Index generation complete. Output directory: $output_dir" From 97d527eec2b879f38e9eed599f8c7d2ae3a0b23e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:19:13 -0400 Subject: [PATCH 302/561] feat: Add server chat_format minicpm-v-2.6 for MiniCPMv26ChatHandler --- llama_cpp/server/model.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index c486f88852..901998094f 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -157,6 +157,20 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler = llama_cpp.llama_chat_format.Llama3VisionAlpha( clip_model_path=settings.clip_model_path, verbose=settings.verbose ) + elif settings.chat_format == "minicpm-v-2.6": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.MiniCPMv26ChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.MiniCPMv26ChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) elif settings.chat_format == "hf-autotokenizer": assert ( settings.hf_pretrained_model_name_or_path is not None From b570fd38c1f5e2b29b801d16e8b556409f25527b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:37:08 -0400 Subject: [PATCH 303/561] =?UTF-8?q?docs:=20Add=20project=20icon=20courtesy?= =?UTF-8?q?=20of=20=F0=9F=A4=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 ++ docs/icon.svg | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 docs/icon.svg diff --git a/README.md b/README.md index b0dfdd5b5c..b8065149c0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![icon.svg](./docs/icon.svg) + # 🦙 Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) [![Documentation Status](https://readthedocs.org/projects/llama-cpp-python/badge/?version=latest)](https://llama-cpp-python.readthedocs.io/en/latest/?badge=latest) diff --git a/docs/icon.svg b/docs/icon.svg new file mode 100644 index 0000000000..9f2d087539 --- /dev/null +++ b/docs/icon.svg @@ -0,0 +1,5 @@ + + + + + From cbbfad45929ebbc90d79bfaaa1dbdf2825a2db3f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:40:06 -0400 Subject: [PATCH 304/561] docs: center icon and resize --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b8065149c0..ea544d4d15 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -![icon.svg](./docs/icon.svg) +

+ +

-# 🦙 Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) +# Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) [![Documentation Status](https://readthedocs.org/projects/llama-cpp-python/badge/?version=latest)](https://llama-cpp-python.readthedocs.io/en/latest/?badge=latest) [![Tests](https://github.com/abetlen/llama-cpp-python/actions/workflows/test.yaml/badge.svg?branch=main)](https://github.com/abetlen/llama-cpp-python/actions/workflows/test.yaml) From ad2deafa8d615e9eaf0f8c3976e465fb1a3ea15f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:51:39 -0400 Subject: [PATCH 305/561] docs: Add MiniCPM-V-2.6 to multi-modal model list --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ea544d4d15..b63efa3c87 100644 --- a/README.md +++ b/README.md @@ -503,6 +503,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [moondream2](https://huggingface.co/vikhyatk/moondream2) | `MoondreamChatHandler` | `moondream2` | | [nanollava](https://huggingface.co/abetlen/nanollava-gguf) | `NanollavaChatHandler` | `nanollava` | | [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha-gguf) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | +| [minicpm-v-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `MiniCPMv26ChatHandler` | `minicpm-v-2.6` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. From 332720d00332334f42576bebf3a8a3da1a27bfaf Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:56:37 -0400 Subject: [PATCH 306/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 20f1789dfb..1d1ccce676 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 20f1789dfb4e535d64ba2f523c64929e7891f428 +Subproject commit 1d1ccce67613674c75c9c7e3fa4c1e24e428ba48 From 077ecb6771fbd373d5507444f6e0b6e9bb7cf4e8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 02:57:36 -0400 Subject: [PATCH 307/561] chore: Bump version --- llama_cpp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index e3cece2f62..db1f1d0fa4 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.89" +__version__ = "0.2.90" From 45001ac550dc6360b4809dd7564d3304d4ad8fa6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 10:12:23 -0400 Subject: [PATCH 308/561] misc(fix): Update CHANGELOG --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 063fe722bf..f01b1b9bfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.90] + +- feat: Update llama.cpp to ggerganov/llama.cpp@1d1ccce67613674c75c9c7e3fa4c1e24e428ba48 +- feat: Add support for `MiniCPMv26ChatHandler` and `minicpm-v-26` in server by @abetlen in f70df824985d875226793b94dacc0c302a4256b2 + ## [0.2.89] - feat: Update llama.cpp to ggerganov/llama.cpp@cfac111e2b3953cdb6b0126e67a2487687646971 From 4b1e3642daa06412008ab2e7366aad09281a70bb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 10:25:21 -0400 Subject: [PATCH 309/561] docs: Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b63efa3c87..29807bbdf5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- +

# Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) From 8b853c0f256c7318938b3894721637174938af17 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 12:41:05 -0400 Subject: [PATCH 310/561] docs: Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29807bbdf5..a8f14e898e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- +

# Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) From 9cba3b8198c5f8d3c3bd12b5ddf5c32838c25040 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 29 Aug 2024 12:41:28 -0400 Subject: [PATCH 311/561] docs: Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a8f14e898e..cbf1c41586 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- +

# Python Bindings for [`llama.cpp`](https://github.com/ggerganov/llama.cpp) From d981d32dac3d35b39cc164a10b4edc11cc5b89db Mon Sep 17 00:00:00 2001 From: benniekiss <63211101+benniekiss@users.noreply.github.com> Date: Thu, 29 Aug 2024 13:04:00 -0400 Subject: [PATCH 312/561] feat: Enable detokenizing special tokens with `special=True` (#1596) * enable detokenizing special tokens * enable skipping_special_tokens in hf_tokenizer detokenize() * process prev_tokens * fix doc strings * Revert changes to LlamaTokenizer prev_tokens and set special to False by default --------- Co-authored-by: Andrei --- llama_cpp/llama.py | 9 ++++++--- llama_cpp/llama_tokenizer.py | 25 ++++++++++++++----------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 9b1a3d263b..45cbd7bca6 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -578,6 +578,8 @@ def tokenize( Args: text: The utf-8 encoded string to tokenize. + add_bos: Whether to add a beginning of sequence token. + special: Whether to tokenize special tokens. Raises: RuntimeError: If the tokenization failed. @@ -588,18 +590,19 @@ def tokenize( return self.tokenizer_.tokenize(text, add_bos, special) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None + self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False ) -> bytes: """Detokenize a list of tokens. Args: tokens: The list of tokens to detokenize. - prev_tokens: The list of previous tokens. Offset mapping will be performed if provided + prev_tokens: The list of previous tokens. Offset mapping will be performed if provided. + special: Whether to detokenize special tokens. Returns: The detokenized string. """ - return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens) + return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens, special=special) def set_cache(self, cache: Optional[BaseLlamaCache]): """Set the cache. diff --git a/llama_cpp/llama_tokenizer.py b/llama_cpp/llama_tokenizer.py index 029bf2acc1..2e7590d144 100644 --- a/llama_cpp/llama_tokenizer.py +++ b/llama_cpp/llama_tokenizer.py @@ -19,20 +19,22 @@ def tokenize( """Tokenize the text into tokens. Args: - text: The text to tokenize. + text: The utf-8 encoded string to tokenize. add_bos: Whether to add a beginning of sequence token. - special: Whether to tokenize text literally or as special tokens.""" + special: Whether to tokenize special tokens. + """ raise NotImplementedError @abc.abstractmethod def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None + self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False ) -> bytes: """Detokenize the tokens into text. Args: - tokens: The tokens to detokenize. - prev_tokens: If tokens is a continuation of a previous sequence, the previous tokens. + tokens: The list of tokens to detokenize. + prev_tokens: The list of previous tokens. Offset mapping will be performed if provided. + special: Whether to detokenize special tokens. """ raise NotImplementedError @@ -47,9 +49,9 @@ def tokenize( return self._model.tokenize(text, add_bos=add_bos, special=special) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None + self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False ) -> bytes: - return self._model.detokenize(tokens) + return self._model.detokenize(tokens, special=special) def encode( self, text: str, add_bos: bool = True, special: bool = True @@ -78,18 +80,19 @@ def tokenize( ) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None + self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False ) -> bytes: + skip_special_tokens = not special if prev_tokens is not None: - text = self.hf_tokenizer.decode(prev_tokens + tokens).encode( + text = self.hf_tokenizer.decode(prev_tokens + tokens, skip_special_tokens=skip_special_tokens).encode( "utf-8", errors="ignore" ) - prev_text = self.hf_tokenizer.decode(prev_tokens).encode( + prev_text = self.hf_tokenizer.decode(prev_tokens, skip_special_tokens=skip_special_tokens).encode( "utf-8", errors="ignore" ) return text[len(prev_text) :] else: - return self.hf_tokenizer.decode(tokens).encode("utf-8", errors="ignore") + return self.hf_tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens).encode("utf-8", errors="ignore") @classmethod def from_pretrained(cls, pretrained_model_name_or_path: str) -> "LlamaHFTokenizer": From 98eb092d3c6e7c142c4ba2faaca6c091718abbb3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 30 Aug 2024 00:14:49 -0400 Subject: [PATCH 313/561] fix: Use system message in og qwen format. Closes #1697 --- llama_cpp/llama_chat_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 21091bf509..dfb0af65e6 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1009,7 +1009,7 @@ def format_qwen( **kwargs: Any, ) -> ChatFormatterResponse: _roles = dict(user="<|im_start|>user", assistant="<|im_start|>assistant") - system_message = "You are a helpful assistant." + system_message = _get_system_message(messages) or "You are a helpful assistant." system_template = "<|im_start|>system\n{system_message}" system_message = system_template.format(system_message=system_message) _messages = _map_roles(messages, _roles) From dcb0d0cf2aae4b5f4b30da49106f69c6bd5346a8 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 30 Aug 2024 00:33:58 -0400 Subject: [PATCH 314/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 36 +++++++++++++++++++++++------------- vendor/llama.cpp | 2 +- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index b35f56a798..d2e35a2d1f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -795,8 +795,8 @@ class llama_model_params(ctypes.Structure): # uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode # uint32_t n_ubatch; // physical maximum batch size # uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) -# uint32_t n_threads; // number of threads to use for generation -# uint32_t n_threads_batch; // number of threads to use for batch processing +# int32_t n_threads; // number of threads to use for generation +# int32_t n_threads_batch; // number of threads to use for batch processing # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` # enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id @@ -901,8 +901,8 @@ class llama_context_params(ctypes.Structure): ("n_batch", ctypes.c_uint32), ("n_ubatch", ctypes.c_uint32), ("n_seq_max", ctypes.c_uint32), - ("n_threads", ctypes.c_uint32), - ("n_threads_batch", ctypes.c_uint32), + ("n_threads", ctypes.c_int32), + ("n_threads_batch", ctypes.c_int32), ("rope_scaling_type", ctypes.c_int), ("pooling_type", ctypes.c_int), ("attention_type", ctypes.c_int), @@ -1197,6 +1197,16 @@ def llama_numa_init(numa: int, /): ... +# // Optional: an auto threadpool gets created in ggml if not passed explicitly +# LLAMA_API void llama_attach_threadpool( +# struct llama_context * ctx, +# ggml_threadpool_t threadpool, +# ggml_threadpool_t threadpool_batch); + + +# LLAMA_API void llama_detach_threadpool(struct llama_context * ctx); + + # // Call once at the end of the program - currently only used for MPI # LLAMA_API void llama_backend_free(void); @ctypes_function( @@ -2478,20 +2488,20 @@ def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int: # // Set the number of threads used for decoding # // n_threads is the number of threads used for generation (single token) # // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) -# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, uint32_t n_threads, uint32_t n_threads_batch); +# LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch); @ctypes_function( "llama_set_n_threads", [ llama_context_p_ctypes, - ctypes.c_uint32, - ctypes.c_uint32, + ctypes.c_int32, + ctypes.c_int32, ], None, ) def llama_set_n_threads( ctx: llama_context_p, - n_threads: Union[ctypes.c_uint32, int], - n_threads_batch: Union[ctypes.c_uint32, int], + n_threads: Union[ctypes.c_int32, int], + n_threads_batch: Union[ctypes.c_int32, int], /, ): """Set the number of threads used for decoding @@ -2502,16 +2512,16 @@ def llama_set_n_threads( # // Get the number of threads used for generation of a single token. -# LLAMA_API uint32_t llama_n_threads(struct llama_context * ctx); -@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_uint32) +# LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); +@ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads(ctx: llama_context_p, /) -> int: """Get the number of threads used for generation of a single token""" ... # // Get the number of threads used for prompt and batch processing (multiple token). -# LLAMA_API uint32_t llama_n_threads_batch(struct llama_context * ctx); -@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_uint32) +# LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); +@ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads_batch(ctx: llama_context_p, /) -> int: """Get the number of threads used for prompt and batch processing (multiple token)""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1d1ccce676..42c76d1358 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1d1ccce67613674c75c9c7e3fa4c1e24e428ba48 +Subproject commit 42c76d1358021ccdbe8ba89109c143dd7ae166df From 9769e5719ad45d4be6bc3ebe01d0f568d3d5001e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 31 Aug 2024 08:59:41 -0400 Subject: [PATCH 315/561] feat: Update llama.cpp --- llama_cpp/llama.py | 2 +- llama_cpp/llama_cpp.py | 6 +++--- vendor/llama.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 45cbd7bca6..7fac936d15 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -153,7 +153,7 @@ def __init__( model_path: Path to the model. n_gpu_layers: Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded. split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options. - main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_LAYER: ignored + main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. rpc_servers: Comma separated list of RPC servers to use for offloading vocab_only: Only load the vocabulary no weights. diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d2e35a2d1f..c1ba14fe53 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -711,9 +711,9 @@ class llama_model_kv_override(ctypes.Structure): # enum llama_split_mode split_mode; // how to split the model across multiple GPUs # // main_gpu interpretation depends on split_mode: -# // LLAMA_SPLIT_NONE: the GPU that is used for the entire model -# // LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results -# // LLAMA_SPLIT_LAYER: ignored +# // LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model +# // LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results +# // LLAMA_SPLIT_MODE_LAYER: ignored # int32_t main_gpu; # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 42c76d1358..a47667cff4 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 42c76d1358021ccdbe8ba89109c143dd7ae166df +Subproject commit a47667cff41f5a198eb791974e0afcc1cddd3229 From c3fc80a2cf5e88360c982a3187751083e881bd16 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 1 Sep 2024 21:06:18 -0400 Subject: [PATCH 316/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 3 +++ vendor/llama.cpp | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index c1ba14fe53..5013237746 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -276,6 +276,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE # LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece # LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram +# LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization # }; LLAMA_VOCAB_TYPE_NONE = 0 """For models without vocab""" @@ -287,6 +288,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa """BERT tokenizer based on WordPiece""" LLAMA_VOCAB_TYPE_UGM = 4 """T5 tokenizer based on Unigram""" +LLAMA_VOCAB_TYPE_RWKV = 5 +"""RWKV tokenizer based on greedy tokenization""" # // pre-tokenization types diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a47667cff4..8f1d81a0b6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a47667cff41f5a198eb791974e0afcc1cddd3229 +Subproject commit 8f1d81a0b6f50b9bad72db0b6fcd299ad9ecd48c From 9497bcd6da7c1675f1309892fd3206e44c38940f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 4 Sep 2024 23:01:55 -0400 Subject: [PATCH 317/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8f1d81a0b6..581c305186 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8f1d81a0b6f50b9bad72db0b6fcd299ad9ecd48c +Subproject commit 581c305186a0ff93f360346c57e21fe16e967bb7 From c032fc65b0873337ed39e5d63e15468a5d797646 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Sep 2024 04:12:29 -0400 Subject: [PATCH 318/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 4 ++++ vendor/llama.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 5013237746..ecaf2369c4 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -443,6 +443,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -479,6 +481,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 +LLAMA_FTYPE_MOSTLY_TQ1_0 = 36 +LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 581c305186..8ebe8ddebd 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 581c305186a0ff93f360346c57e21fe16e967bb7 +Subproject commit 8ebe8ddebd68526757c631cd019de009697c63c2 From e529940f45d42ed8aa31334123b8d66bc67b0e78 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Thu, 19 Sep 2024 01:22:05 +0200 Subject: [PATCH 319/561] feat(ci): Speed up CI workflows using `uv`, add support for CUDA 12.5 wheels * Update build-wheels-cuda.yaml * Update build-wheels-cuda.yaml * revert * Bump pyhton from 3.8 to 3.9 * Remove python 3.8 * Remove Python 3.7 and 3.8 deprecated * Bump python from 3.8 to 3.9 * Add python 3.9 * Add python 3.9, remove macos-11 deprecated, add macos-14 * Bump python 3.8 to 3.9 * Add python 3.13 * Add python 3.13 * python 3.13 remove * remove python 3.13 * remove python 3.8 * Bump macos-13 to macos-14 * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update generate-index-from-release.yaml Add avx, avx2 and avx512 * Update test.yaml * Update test-pypi.yaml * Update publish.yaml * Update publish-to-test.yaml * Update build-wheels-cuda.yaml Cuda with AVX2 by default * Update build-wheels-cuda.yaml * remove DEPRECATED 32 bits * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml Upgrade matrix os to latest version * Update build-wheels-metal.yaml * Update build-wheels-cuda.yaml * Update test.yaml * Update test-pypi.yaml * Update test.yaml Add cache: 'pip' * Update publish-to-test.yaml * Update build-wheels-metal.yaml Add cache: 'pip' * Update build-wheels-cuda.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml remove x86_64 * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * revert * Remove cpu variants * Update build-wheels-metal.yaml * Update build-and-release.yaml * Update publish-to-test.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update publish.yaml * Update test-pypi.yaml * Update test.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update publish.yaml * Update test-pypi.yaml * Update publish-to-test.yaml * Update test.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update publish-to-test.yaml * Update publish.yaml * Update test-pypi.yaml * Update test.yaml * Update test.yaml * Update build-and-release.yaml * Update publish-to-test.yaml * Update build-wheels-metal.yaml * Update test-pypi.yaml * Update test.yaml * Update build-and-release.yaml * Update build-wheels-metal.yaml * Update build-wheels-metal.yaml * Update publish.yaml * Update publish-to-test.yaml * Update test-pypi.yaml * Update test.yaml * Update build-wheels-cuda.yaml * Update generate-index-from-release.yaml * Update README.md * Update README.md * Update test.yaml --------- Co-authored-by: Andrei Betlen --- .github/workflows/build-and-release.yaml | 44 +++++++++-- .github/workflows/build-wheels-cuda.yaml | 2 +- .github/workflows/build-wheels-metal.yaml | 17 ++++- .../generate-index-from-release.yaml | 1 + .github/workflows/publish-to-test.yaml | 24 +++++- .github/workflows/publish.yaml | 23 +++++- .github/workflows/test-pypi.yaml | 59 +++++++++++++-- .github/workflows/test.yaml | 75 +++++++++++++++++-- README.md | 3 +- 9 files changed, 217 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 785f388479..ab8ff4e1c1 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -21,12 +21,25 @@ jobs: # Used to host cibuildwheel - uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.9" - - name: Install dependencies + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | python -m pip install --upgrade pip - python -m pip install -e .[all] + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd - name: Build wheels uses: pypa/cibuildwheel@v2.20.0 @@ -79,16 +92,33 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - uses: actions/setup-python@v5 with: - python-version: "3.8" - - name: Install dependencies + python-version: "3.9" + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | - python -m pip install --upgrade pip build - python -m pip install -e .[all] + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Build source distribution run: | python -m build --sdist + - uses: actions/upload-artifact@v4 with: name: sdist diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 0733a68c5b..b1e5fc1a9c 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -22,7 +22,7 @@ jobs: $matrix = @{ 'os' = @('ubuntu-latest', 'windows-2019') 'pyver' = @("3.9", "3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.0") 'releasetag' = @("basic") } diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index dd8b9c8fce..551820822e 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -23,11 +23,24 @@ jobs: with: python-version: "3.12" cache: 'pip' + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash - - name: Install dependencies + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install -e .[all] + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd - name: Build wheels uses: pypa/cibuildwheel@v2.20.0 diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 51961088d7..8d80e7e47f 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -44,6 +44,7 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu125$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 diff --git a/.github/workflows/publish-to-test.yaml b/.github/workflows/publish-to-test.yaml index 19613233be..de3ae42aae 100644 --- a/.github/workflows/publish-to-test.yaml +++ b/.github/workflows/publish-to-test.yaml @@ -19,24 +19,42 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.11" cache: 'pip' + - name: Append Dev Version to __version__ run: | DEV_VERSION=${{ github.event.inputs.dev_version }} CURRENT_VERSION=$(awk -F= '/__version__ =/ {print $2}' llama_cpp/__init__.py | tr -d ' "') NEW_VERSION="${CURRENT_VERSION}.dev${DEV_VERSION}" sed -i 's/__version__ = \".*\"/__version__ = \"'"${NEW_VERSION}"'\"/' llama_cpp/__init__.py - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | - python -m pip install --upgrade pip build - python -m pip install -e .[all] + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Build source distribution run: | python -m build --sdist + - name: Publish to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index c6abb43b34..67816ea03b 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -13,17 +13,34 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.9" - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | - python -m pip install --upgrade pip build - python -m pip install -e .[all] + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Build source distribution run: | python -m build --sdist + - name: Publish distribution to PyPI # TODO: move to tag based releases # if: startsWith(github.ref, 'refs/tags') diff --git a/.github/workflows/test-pypi.yaml b/.github/workflows/test-pypi.yaml index d7131956d1..335033bba6 100644 --- a/.github/workflows/test-pypi.yaml +++ b/.github/workflows/test-pypi.yaml @@ -16,10 +16,25 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install llama-cpp-python[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install --verbose llama-cpp-python[all] + python -m pip install uv + python -m uv pip install llama-cpp-python[all] --verbose + shell: cmd + - name: Test with pytest run: | python -c "import llama_cpp" @@ -37,10 +52,25 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install llama-cpp-python[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install --verbose llama-cpp-python[all] + python -m pip install uv + python -m uv pip install llama-cpp-python[all] --verbose + shell: cmd + - name: Test with pytest run: | python -c "import llama_cpp" @@ -57,11 +87,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: 'pip' - - name: Install dependencies + cache: 'pip' + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install llama-cpp-python[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install --verbose llama-cpp-python[all] + python -m pip install uv + python -m uv pip install llama-cpp-python[all] --verbose + shell: cmd + - name: Test with pytest run: | python -c "import llama_cpp" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 78f0b49837..9de9c92296 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -20,15 +20,31 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install .[all] -v + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Test with pytest run: | python -m pytest @@ -44,15 +60,31 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip - python -m pip install .[all] -v + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Test with pytest run: | python -m pytest @@ -68,15 +100,31 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' run: | python -m pip install --upgrade pip - python -m pip install .[all] --verbose + python -m pip install uv + RUST_LOG=trace python -m uv pip install -e .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace + run: | + python -m pip install --upgrade pip + python -m pip install uv + python -m uv pip install -e .[all] --verbose + shell: cmd + - name: Test with pytest run: | python -m pytest @@ -90,11 +138,24 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Set up Python 3.9 uses: actions/setup-python@v5 with: python-version: "3.9" - - name: Install dependencies + + - name: Install dependencies (Linux/MacOS) + if: runner.os != 'Windows' + run: | + python -m pip install --upgrade pip + python -m pip install uv + RUST_LOG=trace CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose + shell: bash + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + env: + RUST_LOG: trace run: | python -m pip install --upgrade pip CMAKE_ARGS="-DGGML_METAL=on" python -m pip install .[all] --verbose diff --git a/README.md b/README.md index cbf1c41586..12b4b127d1 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements: -- CUDA Version is 12.1, 12.2, 12.3, or 12.4 +- CUDA Version is 12.1, 12.2, 12.3, 12.4 or 12.5 - Python Version is 3.10, 3.11 or 3.12 ```bash @@ -138,6 +138,7 @@ Where `` is one of the following: - `cu122`: CUDA 12.2 - `cu123`: CUDA 12.3 - `cu124`: CUDA 12.4 +- `cu124`: CUDA 12.5 For example, to install the CUDA 12.1 wheel: From a4e1451abe02c02fc9c831abb591d0559a54b849 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 19:26:51 -0400 Subject: [PATCH 320/561] chore(deps): bump pypa/cibuildwheel from 2.20.0 to 2.21.1 (#1743) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.20.0 to 2.21.1. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.20.0...v2.21.1) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei --- .github/workflows/build-and-release.yaml | 4 ++-- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index ab8ff4e1c1..3f85a30fe8 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -42,7 +42,7 @@ jobs: shell: cmd - name: Build wheels - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.21.1 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -69,7 +69,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.21.1 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 551820822e..8e6fb5cb78 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -43,7 +43,7 @@ jobs: shell: cmd - name: Build wheels - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.21.1 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" From f8fcb3ea3424bcfba3a5437626a994771a02324b Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 18 Sep 2024 20:00:19 -0400 Subject: [PATCH 321/561] feat: Update sampling API for llama.cpp (#1742) * Initial samplng api update * Fix logger * Update tests * Update * Remove seed * Add sampling chain * Remove unnused test * Use Qwen2 0.5B for ci tests * Fix typo * Fix typo * Update cache version * Use real model for tests * Add huggingface-hub as a test dependency * Remove RUST_LOG=trace * Add actual logit processor test --- .github/workflows/test.yaml | 81 ++- llama_cpp/_internals.py | 361 +++++------ llama_cpp/_logger.py | 19 +- llama_cpp/llama.py | 227 ++++--- llama_cpp/llama_cpp.py | 1178 ++++++++++++++++------------------- llama_cpp/llama_grammar.py | 881 +------------------------- pyproject.toml | 1 + tests/test_llama.py | 358 ++++------- tests/test_llama_grammar.py | 10 +- vendor/llama.cpp | 2 +- 10 files changed, 1035 insertions(+), 2083 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9de9c92296..f24f5b7e18 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,5 +1,4 @@ name: Tests - on: pull_request: branches: @@ -8,14 +7,34 @@ on: branches: - main +env: + REPO_ID: Qwen/Qwen2-0.5B-Instruct-GGUF + MODEL_FILE: qwen2-0_5b-instruct-q8_0.gguf + jobs: - build-linux: + download-model: + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.9" + - name: Install huggingface-hub + run: pip install huggingface-hub + - name: Download model + run: huggingface-cli download ${{ env.REPO_ID }} ${{ env.MODEL_FILE }} + - name: Cache model + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} + build-linux: + needs: download-model runs-on: ubuntu-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] - steps: - uses: actions/checkout@v4 with: @@ -26,36 +45,35 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - + - name: Restore model cache + uses: actions/cache@v3 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv - RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install -e .[all] --verbose shell: bash - - name: Install dependencies (Windows) if: runner.os == 'Windows' - env: - RUST_LOG: trace run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose - shell: cmd - + shell: cmd - name: Test with pytest run: | python -m pytest build-windows: - + needs: download-model runs-on: windows-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] - steps: - uses: actions/checkout@v4 with: @@ -66,19 +84,23 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' + + - name: Restore model cache + uses: actions/cache@v3 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv - RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install -e .[all] --verbose shell: bash - name: Install dependencies (Windows) if: runner.os == 'Windows' - env: - RUST_LOG: trace run: | python -m pip install --upgrade pip python -m pip install uv @@ -90,12 +112,11 @@ jobs: python -m pytest build-macos: - + needs: download-model runs-on: macos-latest strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] - steps: - uses: actions/checkout@v4 with: @@ -106,34 +127,36 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' + + - name: Restore model cache + uses: actions/cache@v3 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv - RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install -e .[all] --verbose shell: bash - name: Install dependencies (Windows) if: runner.os == 'Windows' - env: - RUST_LOG: trace run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose - shell: cmd + shell: cmd - name: Test with pytest run: | python -m pytest - build-macos-metal: - + needs: download-model runs-on: macos-latest - steps: - uses: actions/checkout@v4 with: @@ -144,18 +167,22 @@ jobs: with: python-version: "3.9" + - name: Restore model cache + uses: actions/cache@v3 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} + - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv - RUST_LOG=trace CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose + CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose shell: bash - name: Install dependencies (Windows) if: runner.os == 'Windows' - env: - RUST_LOG: trace run: | python -m pip install --upgrade pip CMAKE_ARGS="-DGGML_METAL=on" python -m pip install .[all] --verbose diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 6dae88c8fc..6475695c63 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -6,6 +6,7 @@ from typing import ( Dict, List, + Tuple, Optional, Sequence, ) @@ -25,7 +26,7 @@ # Python wrappers over llama.h structs -class _LlamaModel: +class LlamaModel: """Intermediate Python wrapper for a llama.cpp llama_model. NOTE: For stability it's recommended you use the Llama class instead.""" @@ -41,19 +42,21 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - self.model = None + model = None if not os.path.exists(path_model): raise ValueError(f"Model path does not exist: {path_model}") with suppress_stdout_stderr(disable=verbose): - self.model = llama_cpp.llama_load_model_from_file( + model = llama_cpp.llama_load_model_from_file( self.path_model.encode("utf-8"), self.params ) - if self.model is None: + if model is None: raise ValueError(f"Failed to load model from file: {path_model}") + self.model = model + def free_model(): if self.model is None: return @@ -69,128 +72,84 @@ def __del__(self): self.close() def vocab_type(self) -> int: - assert self.model is not None return llama_cpp.llama_vocab_type(self.model) def n_vocab(self) -> int: - assert self.model is not None return llama_cpp.llama_n_vocab(self.model) def n_ctx_train(self) -> int: - assert self.model is not None return llama_cpp.llama_n_ctx_train(self.model) def n_embd(self) -> int: - assert self.model is not None return llama_cpp.llama_n_embd(self.model) def rope_freq_scale_train(self) -> float: - assert self.model is not None return llama_cpp.llama_rope_freq_scale_train(self.model) def desc(self) -> str: - assert self.model is not None buf = ctypes.create_string_buffer(1024) llama_cpp.llama_model_desc(self.model, buf, 1024) return buf.value.decode("utf-8") def size(self) -> int: - assert self.model is not None return llama_cpp.llama_model_size(self.model) def n_params(self) -> int: - assert self.model is not None return llama_cpp.llama_model_n_params(self.model) def get_tensor(self, name: str) -> ctypes.c_void_p: - assert self.model is not None return llama_cpp.llama_get_model_tensor(self.model, name.encode("utf-8")) - def apply_lora_from_file( - self, - lora_path: str, - scale: float, - path_base_model: Optional[str], - n_threads: int, - ): - assert self.model is not None - return llama_cpp.llama_model_apply_lora_from_file( - self.model, - lora_path.encode("utf-8"), - scale, - ( - path_base_model.encode("utf-8") - if path_base_model is not None - else ctypes.c_char_p(0) - ), - n_threads, - ) # Vocab def token_get_text(self, token: int) -> str: - # TODO: Fix - assert self.model is not None return llama_cpp.llama_token_get_text(self.model, token).decode("utf-8") def token_get_score(self, token: int) -> float: - assert self.model is not None return llama_cpp.llama_token_get_score(self.model, token) def token_get_attr(self, token: int) -> int: - assert self.model is not None return llama_cpp.llama_token_get_attr(self.model, token) # Special tokens def token_bos(self) -> int: - assert self.model is not None return llama_cpp.llama_token_bos(self.model) def token_eos(self) -> int: - assert self.model is not None return llama_cpp.llama_token_eos(self.model) def token_cls(self) -> int: - assert self.model is not None return llama_cpp.llama_token_cls(self.model) def token_sep(self) -> int: - assert self.model is not None return llama_cpp.llama_token_sep(self.model) def token_nl(self) -> int: - assert self.model is not None return llama_cpp.llama_token_nl(self.model) def token_prefix(self) -> int: - assert self.model is not None return llama_cpp.llama_token_prefix(self.model) def token_middle(self) -> int: - assert self.model is not None return llama_cpp.llama_token_middle(self.model) def token_suffix(self) -> int: - assert self.model is not None return llama_cpp.llama_token_suffix(self.model) def token_eot(self) -> int: - assert self.model is not None return llama_cpp.llama_token_eot(self.model) def add_bos_token(self) -> bool: - assert self.model is not None return llama_cpp.llama_add_bos_token(self.model) def add_eos_token(self) -> bool: - assert self.model is not None return llama_cpp.llama_add_eos_token(self.model) # Tokenization def tokenize(self, text: bytes, add_bos: bool, special: bool): - assert self.model is not None n_ctx = self.n_ctx_train() tokens = (llama_cpp.llama_token * n_ctx)() n_tokens = llama_cpp.llama_tokenize( @@ -209,13 +168,11 @@ def tokenize(self, text: bytes, add_bos: bool, special: bool): return list(tokens[:n_tokens]) def token_to_piece(self, token: int, special: bool = False) -> bytes: - assert self.model is not None buf = ctypes.create_string_buffer(32) llama_cpp.llama_token_to_piece(self.model, token, buf, 32, 0, special) return bytes(buf) def detokenize(self, tokens: List[int], special: bool = False) -> bytes: - assert self.model is not None output = b"" size = 32 buffer = (ctypes.c_char * size)() @@ -235,7 +192,6 @@ def detokenize(self, tokens: List[int], special: bool = False) -> bytes: # Extra def metadata(self) -> Dict[str, str]: - assert self.model is not None metadata: Dict[str, str] = {} buffer_size = 1024 buffer = ctypes.create_string_buffer(buffer_size) @@ -272,14 +228,14 @@ def default_params(): return llama_cpp.llama_model_default_params() -class _LlamaContext: +class LlamaContext: """Intermediate Python wrapper for a llama.cpp llama_context. NOTE: For stability it's recommended you use the Llama class instead.""" def __init__( self, *, - model: _LlamaModel, + model: LlamaModel, params: llama_cpp.llama_context_params, verbose: bool = True, ): @@ -288,15 +244,13 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - self.ctx = None + ctx = llama_cpp.llama_new_context_with_model(self.model.model, self.params) - assert self.model.model is not None - - self.ctx = llama_cpp.llama_new_context_with_model(self.model.model, self.params) - - if self.ctx is None: + if ctx is None: raise ValueError("Failed to create llama_context") + self.ctx = ctx + def free_ctx(): if self.ctx is None: return @@ -312,35 +266,27 @@ def __del__(self): self.close() def n_ctx(self) -> int: - assert self.ctx is not None return llama_cpp.llama_n_ctx(self.ctx) def pooling_type(self) -> int: - assert self.ctx is not None return llama_cpp.llama_pooling_type(self.ctx) def kv_cache_clear(self): - assert self.ctx is not None llama_cpp.llama_kv_cache_clear(self.ctx) def kv_cache_seq_rm(self, seq_id: int, p0: int, p1: int): - assert self.ctx is not None llama_cpp.llama_kv_cache_seq_rm(self.ctx, seq_id, p0, p1) def kv_cache_seq_cp(self, seq_id_src: int, seq_id_dst: int, p0: int, p1: int): - assert self.ctx is not None llama_cpp.llama_kv_cache_seq_cp(self.ctx, seq_id_src, seq_id_dst, p0, p1) def kv_cache_seq_keep(self, seq_id: int): - assert self.ctx is not None llama_cpp.llama_kv_cache_seq_keep(self.ctx, seq_id) def kv_cache_seq_shift(self, seq_id: int, p0: int, p1: int, shift: int): - assert self.ctx is not None llama_cpp.llama_kv_cache_seq_add(self.ctx, seq_id, p0, p1, shift) def get_state_size(self) -> int: - assert self.ctx is not None return llama_cpp.llama_get_state_size(self.ctx) # TODO: copy_state_data @@ -351,9 +297,7 @@ def get_state_size(self) -> int: # TODO: llama_save_session_file - def decode(self, batch: "_LlamaBatch"): - assert self.ctx is not None - assert batch.batch is not None + def decode(self, batch: LlamaBatch): return_code = llama_cpp.llama_decode( self.ctx, batch.batch, @@ -362,25 +306,21 @@ def decode(self, batch: "_LlamaBatch"): raise RuntimeError(f"llama_decode returned {return_code}") def set_n_threads(self, n_threads: int, n_threads_batch: int): - assert self.ctx is not None llama_cpp.llama_set_n_threads(self.ctx, n_threads, n_threads_batch) def get_logits(self): - assert self.ctx is not None return llama_cpp.llama_get_logits(self.ctx) def get_logits_ith(self, i: int): - assert self.ctx is not None return llama_cpp.llama_get_logits_ith(self.ctx, i) def get_embeddings(self): - assert self.ctx is not None return llama_cpp.llama_get_embeddings(self.ctx) # Sampling functions def set_rng_seed(self, seed: int): - assert self.ctx is not None + # TODO: Fix llama_cpp.llama_set_rng_seed(self.ctx, seed) def sample_repetition_penalties( @@ -392,7 +332,6 @@ def sample_repetition_penalties( penalty_freq: float, penalty_present: float, ): - assert self.ctx is not None llama_cpp.llama_sample_repetition_penalties( self.ctx, llama_cpp.byref(candidates.candidates), @@ -404,26 +343,22 @@ def sample_repetition_penalties( ) def sample_softmax(self, candidates: "_LlamaTokenDataArray"): - assert self.ctx is not None llama_cpp.llama_sample_softmax( self.ctx, llama_cpp.byref(candidates.candidates), ) def sample_top_k(self, candidates: "_LlamaTokenDataArray", k: int, min_keep: int): - assert self.ctx is not None llama_cpp.llama_sample_top_k( self.ctx, llama_cpp.byref(candidates.candidates), k, min_keep ) def sample_top_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - assert self.ctx is not None llama_cpp.llama_sample_top_p( self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep ) def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - assert self.ctx is not None llama_cpp.llama_sample_min_p( self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep ) @@ -431,7 +366,6 @@ def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: i def sample_tail_free( self, candidates: "_LlamaTokenDataArray", z: float, min_keep: int ): - assert self.ctx is not None llama_cpp.llama_sample_tail_free( self.ctx, llama_cpp.byref(candidates.candidates), z, min_keep ) @@ -439,20 +373,16 @@ def sample_tail_free( def sample_typical( self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int ): - assert self.ctx is not None llama_cpp.llama_sample_typical( self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep ) def sample_temp(self, candidates: "_LlamaTokenDataArray", temp: float): - assert self.ctx is not None llama_cpp.llama_sample_temp( self.ctx, llama_cpp.byref(candidates.candidates), temp ) def sample_grammar(self, candidates: "_LlamaTokenDataArray", grammar: LlamaGrammar): - assert self.ctx is not None - assert grammar.grammar is not None llama_cpp.llama_sample_grammar( self.ctx, llama_cpp.byref(candidates.candidates), @@ -467,7 +397,6 @@ def sample_token_mirostat( m: int, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - assert self.ctx is not None return llama_cpp.llama_sample_token_mirostat( self.ctx, llama_cpp.byref(candidates.candidates), @@ -484,7 +413,6 @@ def sample_token_mirostat_v2( eta: float, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - assert self.ctx is not None return llama_cpp.llama_sample_token_mirostat_v2( self.ctx, llama_cpp.byref(candidates.candidates), @@ -494,14 +422,12 @@ def sample_token_mirostat_v2( ) def sample_token_greedy(self, candidates: "_LlamaTokenDataArray") -> int: - assert self.ctx is not None return llama_cpp.llama_sample_token_greedy( self.ctx, llama_cpp.byref(candidates.candidates), ) def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: - assert self.ctx is not None return llama_cpp.llama_sample_token( self.ctx, llama_cpp.byref(candidates.candidates), @@ -509,17 +435,13 @@ def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: # Grammar def grammar_accept_token(self, grammar: LlamaGrammar, token: int): - assert self.ctx is not None - assert grammar.grammar is not None llama_cpp.llama_grammar_accept_token(grammar.grammar, self.ctx, token) def reset_timings(self): - assert self.ctx is not None - llama_cpp.llama_reset_timings(self.ctx) + llama_cpp.llama_perf_context_reset(self.ctx) def print_timings(self): - assert self.ctx is not None - llama_cpp.llama_print_timings(self.ctx) + llama_cpp.llama_perf_context_print(self.ctx) # Utility functions @staticmethod @@ -528,7 +450,7 @@ def default_params(): return llama_cpp.llama_context_default_params() -class _LlamaBatch: +class LlamaBatch: def __init__( self, *, n_tokens: int, embd: int, n_seq_max: int, verbose: bool = True ): @@ -538,11 +460,15 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - self.batch = None - self.batch = llama_cpp.llama_batch_init( + batch = llama_cpp.llama_batch_init( self._n_tokens, self.embd, self.n_seq_max ) + if batch is None: + raise ValueError("Failed to create llama_batch") + + self.batch = batch + def free_batch(): if self.batch is None: return @@ -558,15 +484,12 @@ def __del__(self): self.close() def n_tokens(self) -> int: - assert self.batch is not None return self.batch.n_tokens def reset(self): - assert self.batch is not None self.batch.n_tokens = 0 def set_batch(self, batch: Sequence[int], n_past: int, logits_all: bool): - assert self.batch is not None n_tokens = len(batch) self.batch.n_tokens = n_tokens for i in range(n_tokens): @@ -578,7 +501,6 @@ def set_batch(self, batch: Sequence[int], n_past: int, logits_all: bool): self.batch.logits[n_tokens - 1] = True def add_sequence(self, batch: Sequence[int], seq_id: int, logits_all: bool): - assert self.batch is not None n_tokens = len(batch) n_tokens0 = self.batch.n_tokens self.batch.n_tokens += n_tokens @@ -592,7 +514,7 @@ def add_sequence(self, batch: Sequence[int], seq_id: int, logits_all: bool): self.batch.logits[n_tokens - 1] = True -class _LlamaTokenDataArray: +class LlamaTokenDataArray: def __init__(self, *, n_vocab: int): self.n_vocab = n_vocab self.candidates_data = np.recarray( @@ -617,90 +539,9 @@ def copy_logits(self, logits: npt.NDArray[np.single]): self.candidates.size = self.n_vocab -# Python wrappers over common/common -def _tokenize(model: _LlamaModel, text: str, add_bos: bool, special: bool) -> list[int]: - assert model.model is not None - n_tokens = len(text) + 1 if add_bos else len(text) - result = (llama_cpp.llama_token * n_tokens)() - n_tokens = llama_cpp.llama_tokenize( - model.model, - text.encode("utf-8"), - len(text), - result, - n_tokens, - add_bos, - special, - ) - if n_tokens < 0: - result = (llama_cpp.llama_token * -n_tokens)() - check = llama_cpp.llama_tokenize( - model.model, - text.encode("utf-8"), - len(text), - result, - len(result), - add_bos, - special, - ) - if check != -n_tokens: - raise RuntimeError(f'Failed to tokenize: text="{text}" n_tokens={n_tokens}') - else: - result = result[:n_tokens] - return list(result) - - -def _token_to_piece(model: _LlamaModel, token: int, special: bool = False) -> str: - assert model.model is not None - result = (ctypes.c_char * 8)(0) - n_tokens = llama_cpp.llama_token_to_piece( - model.model, token, result, 0, len(result), special - ) - if n_tokens < 0: - result = (ctypes.c_char * -n_tokens)(0) - check = llama_cpp.llama_token_to_piece( - model.model, token, result, 0, len(result), special - ) - if check != -n_tokens: - raise RuntimeError(f"Failed to get piece: token={token}") - else: - result = result[:n_tokens] - return bytes(result).decode("utf-8") - - -def _detokenize_spm(model: _LlamaModel, tokens: List[int]) -> str: - bos_id = model.token_bos() - result = "" - for i, token in enumerate(tokens): - piece = _token_to_piece(model, token) - if ( - (tokens[0] == bos_id and i == 1) or (tokens[0] != bos_id and i == 0) - ) and piece[0] == " ": - piece = piece[1:] - result += piece - return result - - -def _detokenize_bpe(model: _LlamaModel, tokens: List[int]) -> str: - result = "" - for token in tokens: - piece = _token_to_piece(model, token) - result += piece - return result - - -def _should_add_bos(model: _LlamaModel) -> bool: - assert model.model is not None - add_bos = llama_cpp.llama_add_bos_token(model.model) - if add_bos: - return add_bos - else: - return llama_cpp.llama_vocab_type(model.model) == llama_cpp.LLAMA_VOCAB_TYPE_SPM - - # Embedding functions - -def _normalize_embedding(embedding): +def normalize_embedding(embedding): norm = float(np.linalg.norm(embedding)) if norm == 0.0: return embedding @@ -711,7 +552,7 @@ def _normalize_embedding(embedding): @dataclass -class _LlamaSamplingParams: +class LlamaSamplingParams: n_prev: int = 64 n_probs: int = 0 top_k: int = 40 @@ -738,8 +579,8 @@ class _LlamaSamplingParams: @dataclass -class _LlamaSamplingContext: - params: _LlamaSamplingParams = field(default_factory=_LlamaSamplingParams) +class LlamaSamplingContext: + params: LlamaSamplingParams = field(default_factory=LlamaSamplingParams) mirostat_mu: ctypes.c_float = field(default_factory=ctypes.c_float) grammar: Optional[LlamaGrammar] = None # NOTE: Missing parsed_grammar @@ -753,7 +594,7 @@ def reset(self): self.grammar.reset() def cp(self): - return _LlamaSamplingContext( + return LlamaSamplingContext( params=self.params, mirostat_mu=self.mirostat_mu, grammar=self.grammar, @@ -767,12 +608,12 @@ def last(self) -> Optional[int]: else: return None - def prev_str(self, ctx_main: _LlamaContext, n: int) -> str: + def prev_str(self, ctx_main: LlamaContext, n: int) -> str: return ctx_main.model.detokenize(self.prev[-n:]).decode("utf-8") def sample( self, - ctx_main: _LlamaContext, + ctx_main: LlamaContext, idx: int = 0, logits_array: Optional[npt.NDArray[np.single]] = None, ): @@ -790,7 +631,7 @@ def sample( for token, logit_bias in self.params.logit_bias.items(): logits_array[token] += logit_bias - token_data_array = _LlamaTokenDataArray( + token_data_array = LlamaTokenDataArray( n_vocab=n_vocab ) # TODO: Only create this once token_data_array.copy_logits(logits_array) @@ -862,7 +703,141 @@ def sample( id = ctx_main.sample_token(token_data_array) return id - def accept(self, ctx_main: _LlamaContext, id: int, apply_grammar: bool): + def accept(self, ctx_main: LlamaContext, id: int, apply_grammar: bool): if apply_grammar and self.grammar is not None: ctx_main.grammar_accept_token(self.grammar, id) self.prev.append(id) + + +from typing import List, Callable, Optional, Union +import ctypes +import llama_cpp + +class CustomSampler: + def __init__(self, apply_func: typing.Callable[[llama_cpp.llama_token_data_array], None]): + self.apply_func = apply_func + + def apply_wrapper(sampler: llama_cpp.llama_sampler_p, cur_p: llama_cpp.llama_token_data_array_p): + self.apply_func(cur_p) + + def free_wrapper(sampler: llama_cpp.llama_sampler_p): + pass + + sampler_i = llama_cpp.llama_sampler_i() + sampler_i.apply = llama_cpp.llama_sampler_i_apply(apply_wrapper) + self._apply_wrapper_ref = apply_wrapper + + sampler_i.name = llama_cpp.llama_sampler_i_name(0) + sampler_i.accept = llama_cpp.llama_sampler_i_accept(0) + sampler_i.reset = llama_cpp.llama_sampler_i_reset(0) + sampler_i.clone = llama_cpp.llama_sampler_i_clone(0) + sampler_i.free = llama_cpp.llama_sampler_i_free(0) + + self.sampler = llama_cpp.llama_sampler() + self.sampler.iface = ctypes.pointer(sampler_i) + self.sampler.ctx = None + + def get_sampler(self) -> llama_cpp.llama_sampler_p: + return ctypes.pointer(self.sampler) + +class LlamaSampler: + def __init__(self): + params = llama_cpp.llama_sampler_chain_params() + self.sampler = llama_cpp.llama_sampler_chain_init(params) + self.samplers: List[llama_cpp.llama_sampler_p] = [] + self.custom_samplers: List[Tuple[int, CustomSampler]] = [] + + def add_greedy(self): + sampler = llama_cpp.llama_sampler_init_greedy() + self._add_sampler(sampler) + + def add_dist(self, seed: int): + sampler = llama_cpp.llama_sampler_init_dist(seed) + self._add_sampler(sampler) + + def add_softmax(self): + sampler = llama_cpp.llama_sampler_init_softmax() + self._add_sampler(sampler) + + def add_top_k(self, k: int): + sampler = llama_cpp.llama_sampler_init_top_k(k) + self._add_sampler(sampler) + + def add_top_p(self, p: float, min_keep: int): + sampler = llama_cpp.llama_sampler_init_top_p(p, min_keep) + self._add_sampler(sampler) + + def add_min_p(self, p: float, min_keep: int): + sampler = llama_cpp.llama_sampler_init_min_p(p, min_keep) + self._add_sampler(sampler) + + def add_tail_free(self, z: float, min_keep: int): + sampler = llama_cpp.llama_sampler_init_tail_free(z, min_keep) + self._add_sampler(sampler) + + def add_typical(self, p: float, min_keep: int): + sampler = llama_cpp.llama_sampler_init_typical(p, min_keep) + self._add_sampler(sampler) + + def add_temp(self, temp: float): + sampler = llama_cpp.llama_sampler_init_temp(temp) + self._add_sampler(sampler) + + def add_temp_ext(self, t: float, delta: float, exponent: float): + sampler = llama_cpp.llama_sampler_init_temp_ext(t, delta, exponent) + self._add_sampler(sampler) + + def add_mirostat(self, n_vocab: int, seed: int, tau: float, eta: float, m: int): + sampler = llama_cpp.llama_sampler_init_mirostat( + n_vocab, seed, tau, eta, m + ) + self._add_sampler(sampler) + + def add_mirostat_v2(self, seed: int, tau: float, eta: float): + sampler = llama_cpp.llama_sampler_init_mirostat_v2(seed, tau, eta) + self._add_sampler(sampler) + + def add_grammar(self, model: LlamaModel, grammar: LlamaGrammar): + sampler = llama_cpp.llama_sampler_init_grammar(model.model, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8")) + self._add_sampler(sampler) + + def add_penalties(self, n_vocab: int, special_eos_id: int, linefeed_id: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, penalize_nl: bool, ignore_eos: bool): + sampler = llama_cpp.llama_sampler_init_penalties(n_vocab, special_eos_id, linefeed_id, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, penalize_nl, ignore_eos) + self._add_sampler(sampler) + + def init_logit_bias(self, n_vocab: int, n_logit_bias, logit_bias: llama_cpp.llama_logit_bias_p): + sampler = llama_cpp.llama_sampler_init_logit_bias(n_vocab, n_logit_bias, logit_bias) + self._add_sampler(sampler) + + def add_custom(self, apply_func: Callable[[llama_cpp.llama_token_data_array], None]): + custom_sampler = CustomSampler(apply_func) + sampler = custom_sampler.get_sampler() + self._add_sampler(sampler) + # NOTE: Must remove custom samplers before free or llama.cpp will try to free them + self.custom_samplers.append((llama_cpp.llama_sampler_chain_n(self.sampler) - 1, custom_sampler)) + + def _add_sampler(self, sampler: llama_cpp.llama_sampler_p): + assert self.sampler is not None + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) + self.samplers.append(sampler) + + def get_seed(self) -> int: + assert self.sampler is not None + return llama_cpp.llama_sampler_get_seed(self.sampler) + + def sample(self, ctx: LlamaContext, idx: int) -> int: + assert self.sampler is not None + return llama_cpp.llama_sampler_sample(self.sampler, ctx.ctx, idx) + + def close(self): + if self.sampler: + # NOTE: Must remove custom samplers before free or llama.cpp will try to free them + for i, _ in reversed(self.custom_samplers): + llama_cpp.llama_sampler_chain_remove(self.sampler, i) + llama_cpp.llama_sampler_free(self.sampler) + self.sampler = None + self.samplers.clear() + self.custom_samplers.clear() + + def __del__(self): + self.close() diff --git a/llama_cpp/_logger.py b/llama_cpp/_logger.py index 7638170a97..157af692fe 100644 --- a/llama_cpp/_logger.py +++ b/llama_cpp/_logger.py @@ -5,21 +5,24 @@ import llama_cpp # enum ggml_log_level { -# GGML_LOG_LEVEL_ERROR = 2, -# GGML_LOG_LEVEL_WARN = 3, -# GGML_LOG_LEVEL_INFO = 4, -# GGML_LOG_LEVEL_DEBUG = 5 +# GGML_LOG_LEVEL_NONE = 0, +# GGML_LOG_LEVEL_INFO = 1, +# GGML_LOG_LEVEL_WARN = 2, +# GGML_LOG_LEVEL_ERROR = 3, +# GGML_LOG_LEVEL_DEBUG = 4, # }; GGML_LOG_LEVEL_TO_LOGGING_LEVEL = { - 2: logging.ERROR, - 3: logging.WARNING, - 4: logging.INFO, - 5: logging.DEBUG, + 0: logging.CRITICAL, + 1: logging.INFO, + 2: logging.WARNING, + 3: logging.ERROR, + 4: logging.DEBUG, } logger = logging.getLogger("llama-cpp-python") +# typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); @llama_cpp.llama_log_callback def llama_log_callback( level: int, diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 7fac936d15..557134a6ea 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -46,15 +46,7 @@ import numpy as np import numpy.typing as npt -from ._internals import ( - _LlamaModel, # type: ignore - _LlamaContext, # type: ignore - _LlamaBatch, # type: ignore - _LlamaTokenDataArray, # type: ignore - _LlamaSamplingParams, # type: ignore - _LlamaSamplingContext, # type: ignore - _normalize_embedding, # type: ignore -) +import llama_cpp._internals as internals from ._logger import set_verbose from ._utils import suppress_stdout_stderr @@ -368,7 +360,7 @@ def __init__( self._model = self._stack.enter_context( contextlib.closing( - _LlamaModel( + internals.LlamaModel( path_model=self.model_path, params=self.model_params, verbose=self.verbose, @@ -388,7 +380,7 @@ def __init__( self._ctx = self._stack.enter_context( contextlib.closing( - _LlamaContext( + internals.LlamaContext( model=self._model, params=self.context_params, verbose=self.verbose, @@ -398,7 +390,7 @@ def __init__( self._batch = self._stack.enter_context( contextlib.closing( - _LlamaBatch( + internals.LlamaBatch( n_tokens=self.n_batch, embd=0, n_seq_max=self.context_params.n_ctx, @@ -410,7 +402,6 @@ def __init__( self._lora_adapter: Optional[llama_cpp.llama_lora_adapter_p] = None if self.lora_path: - assert self._model.model is not None self._lora_adapter = llama_cpp.llama_lora_adapter_init( self._model.model, self.lora_path.encode("utf-8"), @@ -428,7 +419,6 @@ def free_lora_adapter(): self._stack.callback(free_lora_adapter) - assert self._ctx.ctx is not None if llama_cpp.llama_lora_adapter_set( self._ctx.ctx, self._lora_adapter, self.lora_scale ): @@ -453,7 +443,7 @@ def free_lora_adapter(): self._token_nl = self.token_nl() self._token_eos = self.token_eos() - self._candidates = _LlamaTokenDataArray(n_vocab=self._n_vocab) + self._candidates = internals.LlamaTokenDataArray(n_vocab=self._n_vocab) self.n_tokens = 0 self.input_ids: npt.NDArray[np.intc] = np.ndarray((n_ctx,), dtype=np.intc) @@ -542,14 +532,14 @@ def free_lora_adapter(): f"Using fallback chat format: {self.chat_format}", file=sys.stderr ) + self._sampler = None + @property def ctx(self) -> llama_cpp.llama_context_p: - assert self._ctx.ctx is not None return self._ctx.ctx @property def model(self) -> llama_cpp.llama_model_p: - assert self._model.model is not None return self._model.model @property @@ -618,8 +608,8 @@ def set_seed(self, seed: int): Args: seed: The random seed. """ - assert self._ctx.ctx is not None - llama_cpp.llama_set_rng_seed(self._ctx.ctx, seed) + # TODO: Fix this + # llama_cpp.llama_set_rng_seed(self._ctx.ctx, seed) def reset(self): """Reset the model state.""" @@ -631,8 +621,6 @@ def eval(self, tokens: Sequence[int]): Args: tokens: The list of tokens to evaluate. """ - assert self._ctx.ctx is not None - assert self._batch.batch is not None self._ctx.kv_cache_seq_rm(-1, self.n_tokens, -1) for i in range(0, len(tokens), self.n_batch): batch = tokens[i : min(len(tokens), i + self.n_batch)] @@ -662,6 +650,93 @@ def eval(self, tokens: Sequence[int]): # Update n_tokens self.n_tokens += n_tokens + def _init_sampler( + self, + top_k: int = 40, + top_p: float = 0.95, + min_p: float = 0.05, + typical_p: float = 1.0, + temp: float = 0.80, + repeat_penalty: float = 1.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + tfs_z: float = 1.0, + mirostat_mode: int = 0, + mirostat_eta: float = 0.1, + mirostat_tau: float = 5.0, + penalize_nl: bool = True, + logits_processor: Optional[LogitsProcessorList] = None, + grammar: Optional[LlamaGrammar] = None, + seed: Optional[int] = None, + ): + sampler = internals.LlamaSampler() + + if logits_processor is not None: + # Create and add a custom sampler + def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): + size = token_data_array.contents.size + data_soa = token_data_array.contents.data + data_soa_address = ctypes.addressof(data_soa.contents) + # NOTE: This is probably broken + recarray = np.recarray( + shape=(size,), + dtype=np.dtype( + [("id", np.intc), ("logit", np.single), ("p", np.single)], align=True + ), + buf=(llama_cpp.llama_token_data * size).from_address(data_soa_address), + ) + for logit_processor in logits_processor: + recarray.logit[:] = logit_processor(self._input_ids, recarray.logit) + sampler.add_custom(apply_func) + + sampler.add_penalties( + n_vocab=self._n_vocab, + special_eos_id=self._token_eos, + linefeed_id=self._token_nl, + penalty_last_n=self.last_n_tokens_size, + penalty_repeat=repeat_penalty, + penalty_freq=frequency_penalty, + penalty_present=presence_penalty, + penalize_nl=penalize_nl, + ignore_eos=False + ) + + if grammar is not None: + sampler.add_grammar(self._model, grammar) + + if temp < 0.0: + sampler.add_softmax() + sampler.add_dist(seed or llama_cpp.LLAMA_DEFAULT_SEED) + elif temp == 0.0: + sampler.add_greedy() + else: + if mirostat_mode == 1: + mirostat_m = 100 + sampler.add_mirostat( + self._n_vocab, + seed or llama_cpp.LLAMA_DEFAULT_SEED, + mirostat_tau, + mirostat_eta, + mirostat_m, + ) + elif mirostat_mode == 2: + sampler.add_mirostat_v2( + seed or llama_cpp.LLAMA_DEFAULT_SEED, + mirostat_tau, + mirostat_eta, + ) + else: + n_probs = 0 + min_keep = max(1, n_probs) + sampler.add_top_k(top_k) + sampler.add_tail_free(tfs_z, min_keep) + sampler.add_typical(typical_p, min_keep) + sampler.add_top_p(top_p, min_keep) + sampler.add_min_p(min_p, min_keep) + sampler.add_temp(temp) + sampler.add_dist(seed or llama_cpp.LLAMA_DEFAULT_SEED) + return sampler + def sample( self, top_k: int = 40, @@ -692,49 +767,35 @@ def sample( Returns: The sampled token. """ - assert self._ctx is not None assert self.n_tokens > 0 - if idx is None: - logits: npt.NDArray[np.single] = self._scores[-1, :] - else: - logits = self._scores[idx, :] - - if logits_processor is not None: - logits[:] = ( - logits_processor(self._input_ids, logits) - if idx is None - else logits_processor(self._input_ids[: idx + 1], logits) + tmp_sampler = False + + if self._sampler is None: + tmp_sampler = True + self._sampler = self._init_sampler( + top_k=top_k, + top_p=top_p, + min_p=min_p, + typical_p=typical_p, + temp=temp, + repeat_penalty=repeat_penalty, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + tfs_z=tfs_z, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + penalize_nl=penalize_nl, + logits_processor=logits_processor, + grammar=grammar, ) - sampling_params = _LlamaSamplingParams( - top_k=top_k, - top_p=top_p, - min_p=min_p, - tfs_z=tfs_z, - typical_p=typical_p, - temp=temp, - penalty_last_n=self.last_n_tokens_size, - penalty_repeat=repeat_penalty, - penalty_freq=frequency_penalty, - penalty_present=presence_penalty, - mirostat=mirostat_mode, - mirostat_tau=mirostat_tau, - mirostat_eta=mirostat_eta, - penalize_nl=penalize_nl, - ) - sampling_context = _LlamaSamplingContext( - params=sampling_params, - grammar=grammar, - ) - sampling_context.prev = list(self.eval_tokens) - id = sampling_context.sample(ctx_main=self._ctx, logits_array=logits) - sampling_context.accept( - ctx_main=self._ctx, - id=id, - apply_grammar=grammar is not None, - ) - return id + assert self.ctx is not None + token = self._sampler.sample(self._ctx, -1) + if tmp_sampler: + self._sampler = None + return token def generate( self, @@ -756,6 +817,7 @@ def generate( logits_processor: Optional[LogitsProcessorList] = None, stopping_criteria: Optional[StoppingCriteriaList] = None, grammar: Optional[LlamaGrammar] = None, + seed: Optional[int] = None, ) -> Generator[int, Optional[Sequence[int]], None]: """Create a generator of tokens from a prompt. @@ -778,6 +840,24 @@ def generate( """ # Reset mirostat sampling self._mirostat_mu = ctypes.c_float(2.0 * mirostat_tau) + self._sampler = self._init_sampler( + top_k=top_k, + top_p=top_p, + min_p=min_p, + typical_p=typical_p, + temp=temp, + repeat_penalty=repeat_penalty, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + tfs_z=tfs_z, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + penalize_nl=penalize_nl, + logits_processor=logits_processor, + grammar=grammar, + seed=seed, + ) # Check for kv cache prefix match if reset and self.n_tokens > 0: @@ -799,9 +879,9 @@ def generate( if reset: self.reset() - # Reset the grammar - if grammar is not None: - grammar.reset() + # # Reset the grammar + # if grammar is not None: + # grammar.reset() sample_idx = self.n_tokens + len(tokens) - 1 tokens = list(tokens) @@ -867,7 +947,6 @@ def create_embedding( Returns: An embedding object. """ - assert self._model.model is not None model_name: str = model if model is not None else self.model_path input = input if isinstance(input, list) else [input] @@ -912,7 +991,6 @@ def embed( Returns: A list of embeddings """ - assert self._ctx.ctx is not None n_embd = self.n_embd() n_batch = self.n_batch @@ -926,7 +1004,7 @@ def embed( ) if self.verbose: - llama_cpp.llama_reset_timings(self._ctx.ctx) + llama_cpp.llama_perf_context_reset(self._ctx.ctx) if isinstance(input, str): inputs = [input] @@ -940,7 +1018,6 @@ def embed( data: Union[List[List[float]], List[List[List[float]]]] = [] def decode_batch(seq_sizes: List[int]): - assert self._ctx.ctx is not None llama_cpp.llama_kv_cache_clear(self._ctx.ctx) self._ctx.decode(self._batch) self._batch.reset() @@ -955,7 +1032,7 @@ def decode_batch(seq_sizes: List[int]): for j in range(size) ] if normalize: - embedding = [_normalize_embedding(e) for e in embedding] + embedding = [internals.normalize_embedding(e) for e in embedding] data.append(embedding) pos += size else: @@ -963,7 +1040,7 @@ def decode_batch(seq_sizes: List[int]): ptr = llama_cpp.llama_get_embeddings_seq(self._ctx.ctx, i) embedding: List[float] = ptr[:n_embd] if normalize: - embedding = _normalize_embedding(embedding) + embedding = internals.normalize_embedding(embedding) data.append(embedding) # init state @@ -1006,7 +1083,7 @@ def decode_batch(seq_sizes: List[int]): decode_batch(s_batch) if self.verbose: - llama_cpp.llama_print_timings(self._ctx.ctx) + llama_cpp.llama_perf_context_print(self._ctx.ctx) output = data[0] if isinstance(input, str) else data @@ -1048,7 +1125,6 @@ def _create_completion( ) -> Union[ Iterator[CreateCompletionResponse], Iterator[CreateCompletionStreamResponse] ]: - assert self._ctx is not None assert suffix is None or suffix.__class__ is str completion_id: str = f"cmpl-{str(uuid.uuid4())}" @@ -1211,8 +1287,9 @@ def logit_bias_processor( if self.verbose: print("Llama._create_completion: cache miss", file=sys.stderr) - if seed is not None: - self._ctx.set_rng_seed(seed) + # TODO: Fix this + # if seed is not None: + # self._ctx.set_rng_seed(seed) finish_reason = "length" multibyte_fix = 0 @@ -1233,8 +1310,8 @@ def logit_bias_processor( stopping_criteria=stopping_criteria, logits_processor=logits_processor, grammar=grammar, + seed=seed, ): - assert self._model.model is not None if llama_cpp.llama_token_is_eog(self._model.model, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) finish_reason = "stop" @@ -2019,7 +2096,6 @@ def __setstate__(self, state): self.__init__(**state) def save_state(self) -> LlamaState: - assert self._ctx.ctx is not None if self.verbose: print("Llama.save_state: saving llama state", file=sys.stderr) state_size = llama_cpp.llama_get_state_size(self._ctx.ctx) @@ -2049,7 +2125,6 @@ def save_state(self) -> LlamaState: ) def load_state(self, state: LlamaState) -> None: - assert self._ctx.ctx is not None # Only filling in up to `n_tokens` and then zero-ing out the rest self.scores[: state.n_tokens, :] = state.scores.copy() self.scores[state.n_tokens :, :] = 0.0 diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index ecaf2369c4..261d7c6b41 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -233,6 +233,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_DEFAULT_SEED 0xFFFFFFFF LLAMA_DEFAULT_SEED = 0xFFFFFFFF +# define LLAMA_TOKEN_NULL -1 +LLAMA_TOKEN_NULL = -1 + # define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' LLAMA_FILE_MAGIC_GGLA = 0x67676C61 @@ -244,8 +247,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa # define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN -# define LLAMA_SESSION_VERSION 8 -LLAMA_SESSION_VERSION = 8 +# define LLAMA_SESSION_VERSION 9 +LLAMA_SESSION_VERSION = 9 # define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ @@ -260,6 +263,9 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa llama_context_p = NewType("llama_context_p", int) llama_context_p_ctypes = ctypes.c_void_p +# # struct llama_sampler; +# llama_sampler_p = NewType("llama_sampler_p", int) +# llama_sampler_p_ctypes = ctypes.c_void_p # typedef int32_t llama_pos; llama_pos = ctypes.c_int32 @@ -561,6 +567,7 @@ class llama_token_data(ctypes.Structure): # typedef struct llama_token_data_array { # llama_token_data * data; # size_t size; +# int64_t selected; // this is the index in the data array (i.e. not the token id) # bool sorted; # } llama_token_data_array; class llama_token_data_array(ctypes.Structure): @@ -569,16 +576,19 @@ class llama_token_data_array(ctypes.Structure): Attributes: data (ctypes.Array[llama_token_data]): token data size (int): size of the array + selected (int): index in the data array (i.e. not the token id) sorted (bool): whether the array is sorted""" if TYPE_CHECKING: data: CtypesArray[llama_token_data] size: int + selected: int sorted: bool _fields_ = [ ("data", llama_token_data_p), ("size", ctypes.c_size_t), + ("selected", ctypes.c_int64), ("sorted", ctypes.c_bool), ] @@ -797,7 +807,6 @@ class llama_model_params(ctypes.Structure): # // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations # // https://github.com/ggerganov/llama.cpp/pull/7544 # struct llama_context_params { -# uint32_t seed; // RNG seed, -1 for random # uint32_t n_ctx; // text context, 0 = from model # uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode # uint32_t n_ubatch; // physical maximum batch size @@ -830,6 +839,7 @@ class llama_model_params(ctypes.Structure): # bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU # bool flash_attn; // whether to use flash attention [EXPERIMENTAL] +# bool no_perf; // whether to measure performance timings # // Abort callback @@ -842,7 +852,6 @@ class llama_context_params(ctypes.Structure): """Parameters for llama_context Attributes: - seed (int): RNG seed, -1 for random n_ctx (int): text context, 0 = from model n_batch (int): logical maximum batch size that can be submitted to llama_decode n_ubatch (int): physical maximum batch size @@ -873,7 +882,6 @@ class llama_context_params(ctypes.Structure): """ if TYPE_CHECKING: - seed: int n_ctx: int n_batch: int n_ubatch: int @@ -903,7 +911,6 @@ class llama_context_params(ctypes.Structure): abort_callback_data: ctypes.c_void_p _fields_ = [ - ("seed", ctypes.c_uint32), ("n_ctx", ctypes.c_uint32), ("n_batch", ctypes.c_uint32), ("n_ubatch", ctypes.c_uint32), @@ -1009,101 +1016,44 @@ class llama_model_quantize_params(ctypes.Structure): ] -# // grammar types -# struct llama_grammar; -llama_grammar_p = ctypes.c_void_p - -# // grammar element type -# enum llama_gretype { -# // end of rule definition -# LLAMA_GRETYPE_END = 0, - -# // start of alternate definition for rule -# LLAMA_GRETYPE_ALT = 1, - -# // non-terminal element: reference to rule -# LLAMA_GRETYPE_RULE_REF = 2, - -# // terminal element: character (code point) -# LLAMA_GRETYPE_CHAR = 3, - -# // inverse char(s) ([^a], [^a-b] [^abc]) -# LLAMA_GRETYPE_CHAR_NOT = 4, - -# // modifies a preceding LLAMA_GRETYPE_CHAR or LLAMA_GRETYPE_CHAR_ALT to -# // be an inclusive range ([a-z]) -# LLAMA_GRETYPE_CHAR_RNG_UPPER = 5, +# typedef struct llama_logit_bias { +# llama_token token; +# float bias; +# } llama_logit_bias; +class llama_logit_bias(ctypes.Structure): + """Used to store logit bias -# // modifies a preceding LLAMA_GRETYPE_CHAR or -# // LLAMA_GRETYPE_CHAR_RNG_UPPER to add an alternate char to match ([ab], [a-zA]) -# LLAMA_GRETYPE_CHAR_ALT = 6, + Attributes: + token (llama_token): token id + bias (float): bias""" -# // any character (.) -# LLAMA_GRETYPE_CHAR_ANY = 7, -# }; -LLAMA_GRETYPE_END = 0 -LLAMA_GRETYPE_ALT = 1 -LLAMA_GRETYPE_RULE_REF = 2 -LLAMA_GRETYPE_CHAR = 3 -LLAMA_GRETYPE_CHAR_NOT = 4 -LLAMA_GRETYPE_CHAR_RNG_UPPER = 5 -LLAMA_GRETYPE_CHAR_ALT = 6 -LLAMA_GRETYPE_CHAR_ANY = 7 - - -# typedef struct llama_grammar_element { -# enum llama_gretype type; -# uint32_t value; // Unicode code point or rule ID -# } llama_grammar_element; -class llama_grammar_element(ctypes.Structure): if TYPE_CHECKING: - type: int - value: int + token: llama_token + bias: float _fields_ = [ - ("type", ctypes.c_int), - ("value", ctypes.c_uint32), + ("token", llama_token), + ("bias", ctypes.c_float), ] -llama_grammar_element_p = ctypes.POINTER(llama_grammar_element) +llama_logit_bias_p = ctypes.POINTER(llama_logit_bias) -# // performance timing information -# struct llama_timings { -# double t_start_ms; -# double t_end_ms; -# double t_load_ms; -# double t_sample_ms; -# double t_p_eval_ms; -# double t_eval_ms; +# typedef struct llama_sampler_chain_params { +# bool no_perf; // whether to measure performance timings +# } llama_sampler_chain_params; +class llama_sampler_chain_params(ctypes.Structure): + """Parameters for llama_sampler_chain + + Attributes: + no_perf (bool): whether to measure performance timings""" -# int32_t n_sample; -# int32_t n_p_eval; -# int32_t n_eval; -# }; -class llama_timings(ctypes.Structure): if TYPE_CHECKING: - t_start_ms: float - t_end_ms: float - t_load_ms: float - t_sample_ms: float - t_p_eval_ms: float - t_eval_ms: float - n_sample: int - n_p_eval: int - n_eval: int + no_perf: bool _fields_ = [ - ("t_start_ms", ctypes.c_double), - ("t_end_ms", ctypes.c_double), - ("t_load_ms", ctypes.c_double), - ("t_sample_ms", ctypes.c_double), - ("t_p_eval_ms", ctypes.c_double), - ("t_eval_ms", ctypes.c_double), - ("n_sample", ctypes.c_int32), - ("n_p_eval", ctypes.c_int32), - ("n_eval", ctypes.c_int32), + ("no_perf", ctypes.c_bool), ] @@ -1126,7 +1076,7 @@ class llama_chat_message(ctypes.Structure): # // Helpers for getting default parameters -# LLAMA_API struct llama_model_params llama_model_default_params(void); +# LLAMA_API struct llama_model_params llama_model_default_params(void); @ctypes_function( "llama_model_default_params", [], @@ -1137,7 +1087,7 @@ def llama_model_default_params() -> llama_model_params: ... -# LLAMA_API struct llama_context_params llama_context_default_params(void); +# LLAMA_API struct llama_context_params llama_context_default_params(void); @ctypes_function( "llama_context_default_params", [], @@ -1148,6 +1098,17 @@ def llama_context_default_params() -> llama_context_params: ... +# LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void); +@ctypes_function( + "llama_sampler_chain_default_params", + [], + llama_sampler_chain_params, +) +def llama_sampler_chain_default_params() -> llama_sampler_chain_params: + """Get default parameters for llama_sampler_chain""" + ... + + # LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void); @ctypes_function( "llama_model_quantize_default_params", @@ -1200,8 +1161,7 @@ def llama_backend_init(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): - ... +def llama_numa_init(numa: int, /): ... # // Optional: an auto threadpool gets created in ggml if not passed explicitly @@ -1228,7 +1188,7 @@ def llama_backend_free(): # LLAMA_API struct llama_model * llama_load_model_from_file( # const char * path_model, -# struct llama_model_params params); +# struct llama_model_params params); @ctypes_function( "llama_load_model_from_file", [ctypes.c_char_p, llama_model_params], @@ -1236,8 +1196,7 @@ def llama_backend_free(): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: - ... +) -> Optional[llama_model_p]: ... # LLAMA_API void llama_free_model(struct llama_model * model); @@ -1246,8 +1205,7 @@ def llama_load_model_from_file( [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): - ... +def llama_free_model(model: llama_model_p, /): ... # LLAMA_API struct llama_context * llama_new_context_with_model( @@ -1260,8 +1218,7 @@ def llama_free_model(model: llama_model_p, /): ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: - ... +) -> Optional[llama_context_p]: ... # // Frees all allocated memory @@ -1282,104 +1239,87 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: - ... +def llama_time_us() -> int: ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: - ... +def llama_max_devices() -> int: ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: - ... +def llama_supports_mmap() -> bool: ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: - ... +def llama_supports_mlock() -> bool: ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: - ... - - -# LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); -@ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: - ... +def llama_supports_gpu_offload() -> bool: ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: - ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: - ... +def llama_n_batch(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); @ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ubatch(ctx: llama_context_p, /) -> int: - ... +def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); @ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_seq_max(ctx: llama_context_p, /) -> int: - ... - - -# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); -@ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) -def llama_pooling_type(ctx: llama_context_p, /) -> int: - ... - - -# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); -@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: - ... - - -# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); -@ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: - ... +def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: - ... +def llama_n_vocab(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: - ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_embd (const struct llama_model * model); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: - ... +def llama_n_embd(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_n_layer (const struct llama_model * model); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: - ... +def llama_n_layer(model: llama_model_p, /) -> int: ... + + +# LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); +@ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... + + +# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); +@ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) +def llama_pooling_type(ctx: llama_context_p, /) -> int: ... + + +# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); +@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) +def llama_vocab_type(model: llama_model_p, /) -> int: ... + + +# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); +@ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) +def llama_rope_type(model: llama_model_p, /) -> int: ... # // Get the model's RoPE frequency scaling factor @@ -2040,7 +1980,7 @@ def llama_kv_cache_update(ctx: llama_context_p, /): # // Returns the *actual* size in bytes of the state -# // (rng, logits, embedding and kv_cache) +# // (logits, embedding and kv_cache) # // Only use when saving the state, not when restoring it, otherwise the size may be too small. # LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); @ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) @@ -2170,8 +2110,7 @@ def llama_state_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: - ... +) -> bool: ... # LLAMA_API DEPRECATED(bool llama_load_session_file( @@ -2199,8 +2138,7 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: - ... +) -> int: ... # LLAMA_API bool llama_state_save_file( @@ -2224,8 +2162,7 @@ def llama_state_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: - ... +) -> bool: ... # LLAMA_API DEPRECATED(bool llama_save_session_file( @@ -2250,8 +2187,7 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: - ... +) -> int: ... # // Get the exact size needed to copy the KV cache of a single sequence @@ -2349,8 +2285,7 @@ def llama_state_seq_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: - ... +) -> int: ... # LLAMA_API size_t llama_state_seq_load_file( @@ -2380,8 +2315,7 @@ def llama_state_seq_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: - ... +) -> int: ... # // @@ -2686,8 +2620,7 @@ def llama_get_embeddings_seq( ) def llama_token_get_text( model: llama_model_p, token: Union[llama_token, int], / -) -> bytes: - ... +) -> bytes: ... # LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); @@ -2696,8 +2629,7 @@ def llama_token_get_text( ) def llama_token_get_score( model: llama_model_p, token: Union[llama_token, int], / -) -> float: - ... +) -> float: ... # LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token); @@ -2706,8 +2638,7 @@ def llama_token_get_score( ) def llama_token_get_attr( model: llama_model_p, token: Union[llama_token, int], / -) -> int: - ... +) -> int: ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) @@ -2772,14 +2703,12 @@ def llama_token_nl(model: llama_model_p, /) -> int: # LLAMA_API bool llama_add_bos_token(const struct llama_model * model); @ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_bos_token(model: llama_model_p, /) -> bool: - ... +def llama_add_bos_token(model: llama_model_p, /) -> bool: ... # LLAMA_API bool llama_add_eos_token(const struct llama_model * model); @ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_eos_token(model: llama_model_p, /) -> bool: - ... +def llama_add_eos_token(model: llama_model_p, /) -> bool: ... # // Codellama infill tokens @@ -2792,20 +2721,17 @@ def llama_token_prefix(model: llama_model_p) -> int: # LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: - ... +def llama_token_middle(model: llama_model_p, /) -> int: ... # LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: - ... +def llama_token_suffix(model: llama_model_p, /) -> int: ... # LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle @ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: - ... +def llama_token_eot(model: llama_model_p, /) -> int: ... # // @@ -3006,419 +2932,296 @@ def llama_chat_apply_template( chat: CtypesArray[llama_chat_message], n_msg: int, /, -) -> int: - ... +) -> int: ... # // -# // Grammar +# // Sampling API +# // +# // Sample usage: +# // +# // // prepare the sampling chain at the start +# // auto sparams = llama_sampler_chain_default_params(); +# // +# // llama_sampler * smpl = llama_sampler_chain_init(sparams); +# // +# // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50)); +# // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1)); +# // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8)); +# // +# // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat" +# // // this sampler will be responsible to select the actual token +# // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); +# // +# // ... +# // +# // // decoding loop: +# // while (...) { +# // ... +# // +# // llama_decode(ctx, batch); +# // +# // // sample from the logits of the last token in the batch +# // const llama_token id = llama_sampler_sample(smpl, ctx, -1); +# // +# // // accepting the token updates the internal state of certain samplers (e.g. grammar, repetition, etc.) +# // llama_sampler_accept(smpl, id); +# // ... +# // } +# // +# // llama_sampler_free(smpl); +# // +# // TODO: In the future, llama_sampler will be utilized to offload the sampling to the backends (e.g. GPU). +# // TODO: in the future, the entire sampling API that uses llama_model should start using llama_vocab # // +# typedef void * llama_sampler_context_t; +llama_sampler_context_t = ctypes.c_void_p -# LLAMA_API struct llama_grammar * llama_grammar_init( -# const llama_grammar_element ** rules, -# size_t n_rules, -# size_t start_rule_index); -@ctypes_function( - "llama_grammar_init", - [ - ctypes.POINTER(llama_grammar_element_p), - ctypes.c_size_t, - ctypes.c_size_t, - ], - llama_grammar_p, -) -def llama_grammar_init( - rules: CtypesArray[ - CtypesPointer[llama_grammar_element] - ], # NOTE: This might be wrong type sig - n_rules: Union[ctypes.c_size_t, int], - start_rule_index: Union[ctypes.c_size_t, int], - /, -) -> Optional[llama_grammar_p]: - """Initialize a grammar from a set of rules.""" - ... + +# // user code can implement the interface below in order to create custom llama_sampler +# struct llama_sampler_i { +# const char * (*name) (const struct llama_sampler * smpl); // can be NULL +# void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL +# void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required +# void (*reset) ( struct llama_sampler * smpl); // can be NULL +# struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL +# void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL +# +# // TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph +# //void (*apply_ggml) (struct llama_sampler * smpl, ...); +# }; +class llama_sampler_i(ctypes.Structure): ... -# LLAMA_API void llama_grammar_free(struct llama_grammar * grammar); -@ctypes_function( - "llama_grammar_free", - [llama_grammar_p], - None, +# struct llama_sampler { +# struct llama_sampler_i * iface; +# llama_sampler_context_t ctx; +# }; +class llama_sampler(ctypes.Structure): + _fields_ = [ + ("iface", ctypes.POINTER(llama_sampler_i)), + ("ctx", llama_sampler_context_t), + ] + + +if TYPE_CHECKING: + llama_sampler_p = CtypesPointer[llama_sampler] + +llama_sampler_p_ctypes = ctypes.POINTER(llama_sampler) + +llama_sampler_i_name = ctypes.CFUNCTYPE(ctypes.c_char_p, llama_sampler_p_ctypes) +llama_sampler_i_accept = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes, llama_token) +llama_sampler_i_apply = ctypes.CFUNCTYPE( + None, llama_sampler_p_ctypes, llama_token_data_array_p ) -def llama_grammar_free(grammar: llama_grammar_p, /): - """Free a grammar.""" - ... +llama_sampler_i_reset = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i_clone = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, llama_sampler_p_ctypes) +llama_sampler_i_free = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i._fields_ = [ + ("name", llama_sampler_i_name), + ("accept", llama_sampler_i_accept), + ("apply", llama_sampler_i_apply), + ("reset", llama_sampler_i_reset), + ("clone", llama_sampler_i_clone), + ("free", llama_sampler_i_free), +] -# LLAMA_API struct llama_grammar * llama_grammar_copy(const struct llama_grammar * grammar); + +# // mirror of llama_sampler_i: +# LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl); @ctypes_function( - "llama_grammar_copy", - [llama_grammar_p], - llama_grammar_p, + "llama_sampler_name", + [llama_sampler_p_ctypes], + ctypes.c_char_p, ) -def llama_grammar_copy(grammar: llama_grammar_p, /) -> llama_grammar_p: - """Copy a grammar.""" - ... +def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: ... -# /// @details Apply constraints from grammar -# LLAMA_API void llama_grammar_sample( -# const struct llama_grammar * grammar, -# const struct llama_context * ctx, -# llama_token_data_array * candidates); +# LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token); @ctypes_function( - "llama_grammar_sample", - [ - llama_grammar_p, - llama_context_p_ctypes, - llama_token_data_array_p, - ], + "llama_sampler_accept", + [llama_sampler_p_ctypes, llama_token], None, ) -def llama_grammar_sample( - grammar: llama_grammar_p, - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - /, -): - """Apply constraints from grammar""" - ... +def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /): ... -# LLAMA_API DEPRECATED(void llama_sample_grammar( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# const struct llama_grammar * grammar), -# "use llama_grammar_sample instead"); +# LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p); @ctypes_function( - "llama_sample_grammar", - [llama_context_p_ctypes, llama_token_data_array_p, llama_grammar_p], + "llama_sampler_apply", + [llama_sampler_p_ctypes, llama_token_data_array_p], None, ) -def llama_sample_grammar( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - grammar, # type: llama_grammar_p - /, -): - """Apply constraints from grammar - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - grammar: A grammar object containing the rules and constraints to apply to the generated text. - """ - ... +def llama_sampler_apply( + smpl: llama_sampler_p, cur_p: CtypesArray[llama_token_data_array], / +): ... -# /// @details Accepts the sampled token into the grammar -# LLAMA_API void llama_grammar_accept_token( -# struct llama_grammar * grammar, -# struct llama_context * ctx, -# llama_token token); +# LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl); @ctypes_function( - "llama_grammar_accept_token", - [llama_grammar_p, llama_context_p_ctypes, llama_token], + "llama_sampler_reset", + [llama_sampler_p_ctypes], None, ) -def llama_grammar_accept_token( - grammar: llama_grammar_p, - ctx: llama_context_p, - token: Union[llama_token, int], - /, -): - """Accepts the sampled token into the grammar""" - ... +def llama_sampler_reset(smpl: llama_sampler_p, /): ... -# // -# // Sampling functions -# // +# LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl); +@ctypes_function( + "llama_sampler_clone", + [llama_sampler_p_ctypes], + llama_sampler_p_ctypes, +) +def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... -# // Sets the current rng seed. -# LLAMA_API void llama_set_rng_seed(struct llama_context * ctx, uint32_t seed); +# // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) +# LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); @ctypes_function( - "llama_set_rng_seed", - [llama_context_p_ctypes, ctypes.c_uint32], + "llama_sampler_free", + [llama_sampler_p_ctypes], None, ) -def llama_set_rng_seed(ctx: llama_context_p, seed: Union[ctypes.c_uint32, int], /): - """Sets the current rng seed.""" - ... +def llama_sampler_free(smpl: llama_sampler_p, /): ... -# /// @details Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. -# /// @details Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. -# LLAMA_API void llama_sample_repetition_penalties( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# const llama_token * last_tokens, -# size_t penalty_last_n, -# float penalty_repeat, -# float penalty_freq, -# float penalty_present); -@ctypes_function( - "llama_sample_repetition_penalties", - [ - llama_context_p_ctypes, - llama_token_data_array_p, - llama_token_p, - ctypes.c_size_t, - ctypes.c_float, - ctypes.c_float, - ctypes.c_float, - ], - None, +# // llama_sampler_chain +# // a type of llama_sampler that can chain multiple samplers one after another +# +# LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params); +@ctypes_function( + "llama_sampler_chain_init", + [llama_sampler_chain_params], + llama_sampler_p_ctypes, ) -def llama_sample_repetition_penalties( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - last_tokens_data: CtypesArray[llama_token], - penalty_last_n: Union[ctypes.c_size_t, int], - penalty_repeat: Union[ctypes.c_float, float], - penalty_freq: Union[ctypes.c_float, float], - penalty_present: Union[ctypes.c_float, float], - /, -): - """Repetition penalty described in CTRL academic paper https://arxiv.org/abs/1909.05858, with negative logit fix. - Frequency and presence penalties described in OpenAI API https://platform.openai.com/docs/api-reference/parameter-details. - """ - ... +def llama_sampler_chain_init( + params: llama_sampler_chain_params, / +) -> llama_sampler_p: ... -# /// @details Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806 -# /// @param logits Logits extracted from the original generation context. -# /// @param logits_guidance Logits extracted from a separate context from the same model. Other than a negative prompt at the beginning, it should have all generated and user input tokens copied from the main context. -# /// @param scale Guidance strength. 1.0f means no guidance. Higher values mean stronger guidance. -# LLAMA_API void llama_sample_apply_guidance( -# struct llama_context * ctx, -# float * logits, -# float * logits_guidance, -# float scale); +# // important: takes ownership of the sampler object and will free it when llama_sampler_free is called +# LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl); @ctypes_function( - "llama_sample_apply_guidance", - [ - llama_context_p_ctypes, - ctypes.POINTER(ctypes.c_float), - ctypes.POINTER(ctypes.c_float), - ctypes.c_float, - ], + "llama_sampler_chain_add", + [llama_sampler_p_ctypes, llama_sampler_p_ctypes], None, ) -def llama_sample_apply_guidance( - ctx: llama_context_p, - logits: CtypesArray[ctypes.c_float], - logits_guidance: CtypesArray[ctypes.c_float], - scale: Union[ctypes.c_float, float], - /, -): - """Apply classifier-free guidance to the logits as described in academic paper "Stay on topic with Classifier-Free Guidance" https://arxiv.org/abs/2306.17806""" - ... +def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): ... -# /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. -# LLAMA_API void llama_sample_softmax( -# struct llama_context * ctx, -# llama_token_data_array * candidates); +# LLAMA_API struct llama_sampler * llama_sampler_chain_get(const struct llama_sampler * chain, int32_t i); @ctypes_function( - "llama_sample_softmax", - [llama_context_p_ctypes, llama_token_data_array_p], - None, + "llama_sampler_chain_get", + [llama_sampler_p_ctypes, ctypes.c_int32], + llama_sampler_p_ctypes, ) -def llama_sample_softmax( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - /, -): - """Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits.""" - ... +def llama_sampler_chain_get( + chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / +) -> llama_sampler_p: ... -# /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 -# LLAMA_API void llama_sample_top_k( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# int32_t k, -# size_t min_keep); +# LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); @ctypes_function( - "llama_sample_top_k", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_int32, ctypes.c_size_t], - None, + "llama_sampler_chain_n", + [llama_sampler_p_ctypes], + ctypes.c_int, ) -def llama_sample_top_k( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - k: Union[ctypes.c_int, int], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751""" - ... +def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ... + + +# // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed +# LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i); +@ctypes_function( + "llama_sampler_chain_remove", + [llama_sampler_p_ctypes, ctypes.c_int32], + llama_sampler_p_ctypes, +) +def llama_sampler_chain_remove( + chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / +) -> llama_sampler_p: ... + + +# // available samplers: +# +# LLAMA_API struct llama_sampler * llama_sampler_init_greedy (void); +@ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes) +def llama_sampler_init_greedy() -> llama_sampler_p: ... + + +# LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed); +@ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes) +def llama_sampler_init_dist(seed: int) -> llama_sampler_p: ... + + +# /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. +# LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void); +@ctypes_function("llama_sampler_init_softmax", [], llama_sampler_p_ctypes) +def llama_sampler_init_softmax() -> llama_sampler_p: ... + + +# /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 +# LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); +@ctypes_function("llama_sampler_init_top_k", [ctypes.c_int32], llama_sampler_p_ctypes) +def llama_sampler_init_top_k(k: int) -> llama_sampler_p: ... # /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 -# LLAMA_API void llama_sample_top_p( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float p, -# size_t min_keep); +# LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep); @ctypes_function( - "llama_sample_top_p", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t], - None, + "llama_sampler_init_top_p", + [ctypes.c_float, ctypes.c_size_t], + llama_sampler_p_ctypes, ) -def llama_sample_top_p( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - p: Union[ctypes.c_float, float], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751""" - ... +def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: ... # /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841 -# LLAMA_API void llama_sample_min_p( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float p, -# size_t min_keep); +# LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep); @ctypes_function( - "llama_sample_min_p", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t], - None, + "llama_sampler_init_min_p", + [ctypes.c_float, ctypes.c_size_t], + llama_sampler_p_ctypes, ) -def llama_sample_min_p( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - p: Union[ctypes.c_float, float], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841""" - ... +def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ... # /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. -# LLAMA_API void llama_sample_tail_free( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float z, -# size_t min_keep); +# LLAMA_API struct llama_sampler * llama_sampler_init_tail_free (float z, size_t min_keep); @ctypes_function( - "llama_sample_tail_free", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t], - None, + "llama_sampler_init_tail_free", + [ctypes.c_float, ctypes.c_size_t], + llama_sampler_p_ctypes, ) -def llama_sample_tail_free( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - z: Union[ctypes.c_float, float], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.""" - ... +def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: ... # /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. -# LLAMA_API void llama_sample_typical( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float p, -# size_t min_keep); +# LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep); @ctypes_function( - "llama_sample_typical", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float, ctypes.c_size_t], - None, + "llama_sampler_init_typical", + [ctypes.c_float, ctypes.c_size_t], + llama_sampler_p_ctypes, ) -def llama_sample_typical( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - p: Union[ctypes.c_float, float], - min_keep: Union[ctypes.c_size_t, int], - /, -): - """Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.""" - ... +def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: ... -# /// @details Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772. -# LLAMA_API void llama_sample_entropy( -# struct llama_context * ctx, -# llama_token_data_array * candidates_p, -# float min_temp, -# float max_temp, -# float exponent_val); -@ctypes_function( - "llama_sample_entropy", - [ - llama_context_p_ctypes, - llama_token_data_array_p, - ctypes.c_float, - ctypes.c_float, - ctypes.c_float, - ], - None, -) -def llama_sample_entropy( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - min_temp: Union[ctypes.c_float, float], - max_temp: Union[ctypes.c_float, float], - exponent_val: Union[ctypes.c_float, float], - /, -): - """Dynamic temperature implementation described in the paper https://arxiv.org/abs/2309.02772.""" - ... +# LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); +@ctypes_function("llama_sampler_init_temp", [ctypes.c_float], llama_sampler_p_ctypes) +def llama_sampler_init_temp(t: float) -> llama_sampler_p: ... -# LLAMA_API void llama_sample_temp( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float temp); +# /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. +# LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent); @ctypes_function( - "llama_sample_temp", - [llama_context_p_ctypes, llama_token_data_array_p, ctypes.c_float], - None, + "llama_sampler_init_temp_ext", + [ctypes.c_float, ctypes.c_float, ctypes.c_float], + llama_sampler_p_ctypes, ) -def llama_sample_temp( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - temp: Union[ctypes.c_float, float], - /, -): - """Temperature sampling described in academic paper "Generating Long Sequences with Sparse Transformers" https://arxiv.org/abs/1904.10509 - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - temp: The temperature value to use for the sampling. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - """ - ... +def llama_sampler_init_temp_ext( + t: float, delta: float, exponent: float +) -> llama_sampler_p: ... # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3427,46 +3230,20 @@ def llama_sample_temp( # /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. # /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. # /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. -# LLAMA_API llama_token llama_sample_token_mirostat( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float tau, -# float eta, -# int32_t m, -# float * mu); +# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat( +# int32_t n_vocab, +# uint32_t seed, +# float tau, +# float eta, +# int32_t m); @ctypes_function( - "llama_sample_token_mirostat", - [ - llama_context_p_ctypes, - llama_token_data_array_p, - ctypes.c_float, - ctypes.c_float, - ctypes.c_int32, - ctypes.POINTER(ctypes.c_float), - ], - llama_token, + "llama_sampler_init_mirostat", + [ctypes.c_int32, ctypes.c_uint32, ctypes.c_float, ctypes.c_float, ctypes.c_int32], + llama_sampler_p_ctypes, ) -def llama_sample_token_mirostat( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - tau: Union[ctypes.c_float, float], - eta: Union[ctypes.c_float, float], - m: Union[ctypes.c_int, int], - mu: CtypesPointerOrRef[ctypes.c_float], - /, -) -> int: - """Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. - m: The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. - mu: Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. - """ - ... +def llama_sampler_init_mirostat( + n_vocab: int, seed: int, tau: float, eta: float, m: int, / +) -> llama_sampler_p: ... # /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3474,83 +3251,116 @@ def llama_sample_token_mirostat( # /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. # /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. # /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. -# LLAMA_API llama_token llama_sample_token_mirostat_v2( -# struct llama_context * ctx, -# llama_token_data_array * candidates, -# float tau, -# float eta, -# float * mu); -@ctypes_function( - "llama_sample_token_mirostat_v2", +# LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2( +# uint32_t seed, +# float tau, +# float eta); +@ctypes_function( + "llama_sampler_init_mirostat_v2", + [ctypes.c_uint32, ctypes.c_float, ctypes.c_float], + llama_sampler_p_ctypes, +) +def llama_sampler_init_mirostat_v2( + seed: int, tau: float, eta: float, / +) -> llama_sampler_p: ... + + +# LLAMA_API struct llama_sampler * llama_sampler_init_grammar( +# const struct llama_model * model, +# const char * grammar_str, +# const char * grammar_root); +@ctypes_function( + "llama_sampler_init_grammar", + [llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_char_p], + llama_sampler_p_ctypes, +) +def llama_sampler_init_grammar( + model: llama_model_p, grammar_str: bytes, grammar_root: bytes, / +) -> llama_sampler_p: ... + + +# LLAMA_API struct llama_sampler * llama_sampler_init_penalties( +# int32_t n_vocab, // llama_n_vocab() +# llama_token special_eos_id, // llama_token_eos() +# llama_token linefeed_id, // llama_token_nl() +# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) +# float penalty_repeat, // 1.0 = disabled +# float penalty_freq, // 0.0 = disabled +# float penalty_present, // 0.0 = disabled +# bool penalize_nl, // consider newlines as a repeatable token +# bool ignore_eos); // ignore the end-of-sequence token +@ctypes_function( + "llama_sampler_init_penalties", [ - llama_context_p_ctypes, - llama_token_data_array_p, + ctypes.c_int32, + llama_token, + llama_token, + ctypes.c_int32, ctypes.c_float, ctypes.c_float, - ctypes.POINTER(ctypes.c_float), - ], - llama_token, -) -def llama_sample_token_mirostat_v2( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] + ctypes.c_float, + ctypes.c_bool, + ctypes.c_bool, ], - tau: Union[ctypes.c_float, float], - eta: Union[ctypes.c_float, float], - mu: CtypesPointerOrRef[ctypes.c_float], + llama_sampler_p_ctypes, +) +def llama_sampler_init_penalties( + n_vocab: int, + special_eos_id: int, + linefeed_id: int, + penalty_last_n: int, + penalty_repeat: float, + penalty_freq: float, + penalty_present: float, + penalize_nl: bool, + ignore_eos: bool, /, -) -> int: - """Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. - - Parameters: - candidates: A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. - tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. - eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. - mu: Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. - """ - ... +) -> llama_sampler_p: ... -# /// @details Selects the token with the highest probability. -# /// Does not compute the token probabilities. Use llama_sample_softmax() instead. -# LLAMA_API llama_token llama_sample_token_greedy( -# struct llama_context * ctx, -# llama_token_data_array * candidates); +# LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( +# int32_t n_vocab, +# int32_t n_logit_bias, +# const llama_logit_bias * logit_bias); @ctypes_function( - "llama_sample_token_greedy", - [llama_context_p_ctypes, llama_token_data_array_p], - llama_token, + "llama_sampler_init_logit_bias", + [ctypes.c_int32, ctypes.c_int32, llama_logit_bias_p], + llama_sampler_p_ctypes, ) -def llama_sample_token_greedy( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - /, -) -> int: - """Selects the token with the highest probability.""" - ... +def llama_sampler_init_logit_bias( + n_vocab: int, n_logit_bias: int, logit_bias: CtypesArray[llama_logit_bias], / +) -> llama_sampler_p: ... -# /// @details Randomly selects a token from the candidates based on their probabilities using the RNG of ctx. -# LLAMA_API llama_token llama_sample_token( -# struct llama_context * ctx, -# llama_token_data_array * candidates); +# // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise +# LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl); @ctypes_function( - "llama_sample_token", - [llama_context_p_ctypes, llama_token_data_array_p], + "llama_sampler_get_seed", + [llama_sampler_p_ctypes], + ctypes.c_uint32, +) +def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ... + + +# /// @details Sample and accept a token from the idx-th output of the last evaluation +# // +# // Shorthand for: +# // const auto * logits = llama_get_logits_ith(ctx, idx); +# // llama_token_data_array cur_p = { ... init from logits ... }; +# // llama_sampler_apply(smpl, &cur_p); +# // auto token = cur_p.data[cur_p.selected].id; +# // llama_sampler_accept(smpl, token); +# // return token; +# // Returns the sampled token +# LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx); +@ctypes_function( + "llama_sampler_sample", + [llama_sampler_p_ctypes, llama_context_p_ctypes, ctypes.c_int32], llama_token, ) -def llama_sample_token( - ctx: llama_context_p, - candidates: Union[ - CtypesArray[llama_token_data_array], CtypesPointerOrRef[llama_token_data_array] - ], - /, -) -> int: - """Randomly selects a token from the candidates based on their probabilities.""" - ... +def llama_sampler_sample( + smpl: llama_sampler_p, ctx: llama_context_p, idx: int, / +) -> int: ... # // @@ -3600,79 +3410,131 @@ def llama_split_prefix( ... -# Performance information +# // Print system information +# LLAMA_API const char * llama_print_system_info(void); +@ctypes_function("llama_print_system_info", [], ctypes.c_char_p) +def llama_print_system_info() -> bytes: ... -# LLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx); +# // Set callback for all future logging events. +# // If this is not called, or NULL is supplied, everything is output on stderr. +# LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function( - "llama_get_timings", - [llama_context_p_ctypes], - llama_timings, + "llama_log_set", + [ctypes.c_void_p, ctypes.c_void_p], + None, ) -def llama_get_timings(ctx: llama_context_p, /) -> llama_timings: - """Get performance information""" +def llama_log_set( + log_callback: Optional[CtypesFuncPointer], + user_data: ctypes.c_void_p, + /, +): + """Set callback for all future logging events. + + If this is not called, or NULL is supplied, everything is output on stderr.""" ... -# LLAMA_API void llama_print_timings(struct llama_context * ctx); +# // +# // Performance utils +# // +# // NOTE: Used by llama.cpp examples, avoid using in third-party apps. Instead, do your own performance measurements. +# // + + +# struct llama_perf_context_data { +# double t_start_ms; +# double t_load_ms; +# double t_p_eval_ms; +# double t_eval_ms; +# +# int32_t n_p_eval; +# int32_t n_eval; +# }; +class llama_perf_context_data(ctypes.Structure): + _fields_ = [ + ("t_start_ms", ctypes.c_double), + ("t_load_ms", ctypes.c_double), + ("t_p_eval_ms", ctypes.c_double), + ("t_eval_ms", ctypes.c_double), + ("n_p_eval", ctypes.c_int32), + ("n_eval", ctypes.c_int32), + ] + + +# struct llama_perf_sampler_data { +# double t_sample_ms; +# +# int32_t n_sample; +# }; +class llama_perf_sampler_data(ctypes.Structure): + _fields_ = [ + ("t_sample_ms", ctypes.c_double), + ("n_sample", ctypes.c_int32), + ] + + +# LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx); @ctypes_function( - "llama_print_timings", + "llama_perf_context", + [llama_context_p_ctypes], + llama_perf_context_data, +) +def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: ... + + +# LLAMA_API void llama_perf_context_print(const struct llama_context * ctx); +@ctypes_function( + "llama_perf_context_print", [llama_context_p_ctypes], None, ) -def llama_print_timings(ctx: llama_context_p, /): - """Print performance information""" - ... +def llama_perf_context_print(ctx: llama_context_p, /): ... -# LLAMA_API void llama_reset_timings(struct llama_context * ctx); +# LLAMA_API void llama_perf_context_reset( struct llama_context * ctx); @ctypes_function( - "llama_reset_timings", + "llama_perf_context_reset", [llama_context_p_ctypes], None, ) -def llama_reset_timings(ctx: llama_context_p, /): - """Reset performance information""" - ... +def llama_perf_context_reset(ctx: llama_context_p, /): ... -# Print system information -# LLAMA_API const char * llama_print_system_info(void); +# // NOTE: the following work only with samplers constructed via llama_sampler_chain_init +# LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain); @ctypes_function( - "llama_print_system_info", - [], - ctypes.c_char_p, + "llama_perf_sampler", + [llama_sampler_p_ctypes], + llama_perf_sampler_data, ) -def llama_print_system_info() -> bytes: - """Print system information""" - ... +def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: ... -# NOTE: THIS IS CURRENTLY BROKEN AS ggml_log_callback IS NOT EXPOSED IN LLAMA.H -# // Set callback for all future logging events. -# // If this is not called, or NULL is supplied, everything is output on stderr. -# LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data); +# LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); @ctypes_function( - "llama_log_set", - [ctypes.c_void_p, ctypes.c_void_p], + "llama_perf_sampler_print", + [llama_sampler_p_ctypes], None, ) -def llama_log_set( - log_callback: Optional[CtypesFuncPointer], - user_data: ctypes.c_void_p, - /, -): - """Set callback for all future logging events. +def llama_perf_sampler_print(chain: llama_sampler_p, /): ... - If this is not called, or NULL is supplied, everything is output on stderr.""" - ... + +# LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); +@ctypes_function( + "llama_perf_sampler_reset", + [llama_sampler_p_ctypes], + None, +) +def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... -# LLAMA_API void llama_dump_timing_info_yaml(FILE * stream, const struct llama_context * ctx); +# LLAMA_API void llama_perf_dump_yaml(FILE * stream, const struct llama_context * ctx); @ctypes_function( - "llama_dump_timing_info_yaml", - [ctypes.c_void_p, llama_context_p_ctypes], + "llama_perf_dump_yaml", + [ctypes.POINTER(ctypes.c_void_p), llama_context_p_ctypes], None, ) -def llama_dump_timing_info_yaml(stream: ctypes.c_void_p, ctx: llama_context_p, /): - ... +def llama_perf_dump_yaml( + stream: ctypes.POINTER(ctypes.c_void_p), ctx: llama_context_p, / +): ... diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 4cd52c2d5c..785aa88d73 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -2,11 +2,6 @@ # flake8: noqa from pathlib import Path -import sys -import ctypes -import enum -import typing -import dataclasses from itertools import groupby from typing import ( @@ -18,882 +13,16 @@ Union, ) -import llama_cpp.llama_cpp as llama_cpp - -class GrammarElementType(enum.IntEnum): - END = llama_cpp.LLAMA_GRETYPE_END - ALT = llama_cpp.LLAMA_GRETYPE_ALT - RULE_REF = llama_cpp.LLAMA_GRETYPE_RULE_REF - CHAR = llama_cpp.LLAMA_GRETYPE_CHAR - CHAR_NOT = llama_cpp.LLAMA_GRETYPE_CHAR_NOT - CHAR_RNG_UPPER = llama_cpp.LLAMA_GRETYPE_CHAR_RNG_UPPER - CHAR_ALT = llama_cpp.LLAMA_GRETYPE_CHAR_ALT - CHAR_ANY = llama_cpp.LLAMA_GRETYPE_CHAR_ANY - - -@dataclasses.dataclass -class GrammarElement: - type: GrammarElementType - value: int - - -@dataclasses.dataclass -class ParseState: - symbol_ids: typing.Dict[str, int] = dataclasses.field(default_factory=dict) - rules: typing.List[typing.List[GrammarElement]] = dataclasses.field(default_factory=list) - - -# static std::pair decode_utf8(const char * src) { -# static const int lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 }; -# uint8_t first_byte = static_cast(*src); -# uint8_t highbits = first_byte >> 4; -# int len = lookup[highbits]; -# uint8_t mask = (1 << (8 - len)) - 1; -# uint32_t value = first_byte & mask; -# const char * end = src + len; // may overrun! -# const char * pos = src + 1; -# for ( ; pos < end && *pos; pos++) { -# value = (value << 6) + (static_cast(*pos) & 0x3F); -# } -# return std::make_pair(value, pos); -# } -def decode_utf8(src: str) -> typing.Tuple[int, str]: - lookup: list[int] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4] - first_byte: int = ord(src[0]) - highbits: int = first_byte >> 4 - length: int = lookup[highbits] - mask: int = (1 << (8 - length)) - 1 - value: int = first_byte & mask - end: int = min(len(src), length) # Prevent overrun - - pos: int = 1 - for pos in range(1, end): - if not src[pos]: - break - value = (value << 6) + (ord(src[pos]) & 0x3F) - - return value, src[pos:] if pos < len(src) else "" - - -# static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) { -# uint32_t next_id = static_cast(state.symbol_ids.size()); -# auto result = state.symbol_ids.emplace(std::string(src, len), next_id); -# return result.first->second; -# } -def get_symbol_id(state: ParseState, name: str) -> int: - next_id = len(state.symbol_ids) - return state.symbol_ids.setdefault(name, next_id) - - -# static uint32_t generate_symbol_id(parse_state & state, const std::string & base_name) { -# uint32_t next_id = static_cast(state.symbol_ids.size()); -# state.symbol_ids[base_name + '_' + std::to_string(next_id)] = next_id; -# return next_id; -# } -def generate_symbol_id(state: ParseState, base_name: str) -> int: - next_id = len(state.symbol_ids) - state.symbol_ids[f"{base_name}_{next_id}"] = next_id - return next_id - - -# static void add_rule( -# parse_state & state, -# uint32_t rule_id, -# const std::vector & rule) { -# if (state.rules.size() <= rule_id) { -# state.rules.resize(rule_id + 1); -# } -# state.rules[rule_id] = rule; -# } -def add_rule(state: ParseState, rule_id: int, rule: typing.List[GrammarElement]) -> None: - if len(state.rules) <= rule_id: - state.rules.extend([[]] * (rule_id + 1 - len(state.rules))) - state.rules[rule_id] = rule - - -# static bool is_digit_char(char c) { -# return '0' <= c && c <= '9'; -# } -def is_digit_char(c: str) -> bool: - return "0" <= c <= "9" - - -# static bool is_word_char(char c) { -# return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || c == '-' || is_digit_char(c); -# } -def is_word_char(c: str) -> bool: - return ("a" <= c <= "z") or ("A" <= c <= "Z") or c == "-" or is_digit_char(c) - - -# static std::pair parse_hex(const char * src, int size) { -# const char * pos = src; -# const char * end = src + size; -# uint32_t value = 0; -# for ( ; pos < end && *pos; pos++) { -# value <<= 4; -# char c = *pos; -# if ('a' <= c && c <= 'f') { -# value += c - 'a' + 10; -# } else if ('A' <= c && c <= 'F') { -# value += c - 'A' + 10; -# } else if ('0' <= c && c <= '9') { -# value += c - '0'; -# } else { -# break; -# } -# } -# if (pos != end) { -# throw std::runtime_error("expecting " + std::to_string(size) + " hex chars at " + src); -# } -# return std::make_pair(value, pos); -# } -def parse_hex(src: str, size: int) -> typing.Tuple[int, str]: - pos = 0 - value = 0 - for _ in range(size): - value <<= 4 - c = src[pos] - if "a" <= c <= "f": - value += ord(c) - ord("a") + 10 - elif "A" <= c <= "F": - value += ord(c) - ord("A") + 10 - elif "0" <= c <= "9": - value += ord(c) - ord("0") - else: - break - pos += 1 - if pos != size: - raise ValueError(f"expecting {size} hex chars at {src}") - return value, src[pos:] - - -# static const char * parse_space(const char * src, bool newline_ok) { -# const char * pos = src; -# while (*pos == ' ' || *pos == '\t' || *pos == '#' || -# (newline_ok && (*pos == '\r' || *pos == '\n'))) { -# if (*pos == '#') { -# while (*pos && *pos != '\r' && *pos != '\n') { -# pos++; -# } -# } else { -# pos++; -# } -# } -# return pos; -# } -def parse_space(src: str, newline_ok: bool) -> str: - pos = src - while pos and (pos[0] in (' ', '\t', '#') or (newline_ok and pos[0] in ('\r', '\n'))): - if pos[0] == "#": - while pos and pos[0] not in ("\r", "\n"): - pos = pos[1:] - else: - pos = pos[1:] - return pos - - -# static const char * parse_name(const char * src) { -# const char * pos = src; -# while (is_word_char(*pos)) { -# pos++; -# } -# if (pos == src) { -# throw std::runtime_error(std::string("expecting name at ") + src); -# } -# return pos; -# } -def parse_name(src: str) -> typing.Tuple[str, str]: - pos = src - while pos and is_word_char(pos[0]): - pos = pos[1:] - if pos == src: - raise ValueError(f"expecting name at {src}") - return src[:len(src) - len(pos)], pos - -# static const char * parse_int(const char * src) { -# const char * pos = src; -# while (is_digit_char(*pos)) { -# pos++; -# } -# if (pos == src) { -# throw std::runtime_error(std::string("expecting integer at ") + src); -# } -# return pos; -# } -def parse_int(src: str) -> typing.Tuple[int, str]: - pos = src - while pos and is_digit_char(pos[0]): - pos = pos[1:] - if pos == src: - raise ValueError(f"expecting integer at {src}") - return int(src[:len(src) - len(pos)]), pos - - -# static std::pair parse_char(const char * src) { -# if (*src == '\\') { -# switch (src[1]) { -# case 'x': return parse_hex(src + 2, 2); -# case 'u': return parse_hex(src + 2, 4); -# case 'U': return parse_hex(src + 2, 8); -# case 't': return std::make_pair('\t', src + 2); -# case 'r': return std::make_pair('\r', src + 2); -# case 'n': return std::make_pair('\n', src + 2); -# case '\\': -# case '"': -# case '[': -# case ']': -# return std::make_pair(src[1], src + 2); -# default: -# throw std::runtime_error(std::string("unknown escape at ") + src); -# } -# } else if (*src) { -# return decode_utf8(src); -# } -# throw std::runtime_error("unexpected end of input"); -# } -def parse_char(src: str) -> typing.Tuple[int, str]: - if not src: - raise ValueError("unexpected end of input") - if src[0] == "\\": - if src[1] == "x": - return parse_hex(src[2:], 2) - elif src[1] == "u": - return parse_hex(src[2:], 4) - elif src[1] == "U": - return parse_hex(src[2:], 8) - elif src[1] == "t": - return ord("\t"), src[2:] - elif src[1] == "r": - return ord("\r"), src[2:] - elif src[1] == "n": - return ord("\n"), src[2:] - elif src[1] in ('\\', '"', '[', ']'): - return ord(src[1]), src[2:] - else: - raise ValueError(f"unknown escape at {src}") - return decode_utf8(src) - -# static const char * parse_sequence( -# parse_state & state, -# const char * src, -# const std::string & rule_name, -# std::vector & out_elements, -# bool is_nested) { -# size_t last_sym_start = out_elements.size(); -# const char * pos = src; -# -# auto handle_repetitions = [&](int min_times, int max_times) { -# -# if (last_sym_start == out_elements.size()) { -# throw std::runtime_error(std::string("expecting preceding item to */+/?/{ at ") + pos); -# } -# -# // apply transformation to previous symbol (last_sym_start to end) according to -# // the following rewrite rules: -# // S{m,n} --> S S S (m times) S'(n-m) -# // S'(x) ::= S S'(x-1) | -# // (... n-m definitions of these S' rules ...) -# // S'(1) ::= S | -# // S{m,} --> S S S (m times) S' -# // S' ::= S S' | -# // S* --> S{0,} -# // --> S' ::= S S' | -# // S+ --> S{1,} -# // --> S S' -# // S' ::= S S' | -# // S? --> S{0,1} -# // --> S' -# // S' ::= S | -# -# std::vector previous_elements(out_elements.begin() + last_sym_start, out_elements.end()); -# if (min_times == 0) { -# out_elements.resize(last_sym_start); -# } else { -# // Repeat the previous elements (min_times - 1) times -# for (int i = 1; i < min_times; i++) { -# out_elements.insert(out_elements.end(), previous_elements.begin(), previous_elements.end()); -# } -# } -# -# uint32_t last_rec_rule_id = 0; -# auto n_opt = max_times < 0 ? 1 : max_times - min_times; -# -# std::vector rec_rule(previous_elements); -# for (int i = 0; i < n_opt; i++) { -# rec_rule.resize(previous_elements.size()); -# uint32_t rec_rule_id = generate_symbol_id(state, rule_name); -# if (i > 0 || max_times < 0) { -# rec_rule.push_back({LLAMA_GRETYPE_RULE_REF, max_times < 0 ? rec_rule_id : last_rec_rule_id}); -# } -# rec_rule.push_back({LLAMA_GRETYPE_ALT, 0}); -# rec_rule.push_back({LLAMA_GRETYPE_END, 0}); -# add_rule(state, rec_rule_id, rec_rule); -# last_rec_rule_id = rec_rule_id; -# } -# if (n_opt > 0) { -# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, last_rec_rule_id}); -# } -# }; -# -# while (*pos) { -# if (*pos == '"') { // literal string -# pos++; -# last_sym_start = out_elements.size(); -# while (*pos != '"') { -# if (!*pos) { -# throw std::runtime_error("unexpected end of input"); -# } -# auto char_pair = parse_char(pos); -# pos = char_pair.second; -# out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first}); -# } -# pos = parse_space(pos + 1, is_nested); -# } else if (*pos == '[') { // char range(s) -# pos++; -# enum llama_gretype start_type = LLAMA_GRETYPE_CHAR; -# if (*pos == '^') { -# pos++; -# start_type = LLAMA_GRETYPE_CHAR_NOT; -# } -# last_sym_start = out_elements.size(); -# while (*pos != ']') { -# if (!*pos) { -# throw std::runtime_error("unexpected end of input"); -# } -# auto char_pair = parse_char(pos); -# pos = char_pair.second; -# enum llama_gretype type = last_sym_start < out_elements.size() -# ? LLAMA_GRETYPE_CHAR_ALT -# : start_type; -# -# out_elements.push_back({type, char_pair.first}); -# if (pos[0] == '-' && pos[1] != ']') { -# if (!pos[1]) { -# throw std::runtime_error("unexpected end of input"); -# } -# auto endchar_pair = parse_char(pos + 1); -# pos = endchar_pair.second; -# out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first}); -# } -# } -# pos = parse_space(pos + 1, is_nested); -# } else if (is_word_char(*pos)) { // rule reference -# const char * name_end = parse_name(pos); -# uint32_t ref_rule_id = get_symbol_id(state, pos, name_end - pos); -# pos = parse_space(name_end, is_nested); -# last_sym_start = out_elements.size(); -# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, ref_rule_id}); -# } else if (*pos == '(') { // grouping -# // parse nested alternates into synthesized rule -# pos = parse_space(pos + 1, true); -# uint32_t sub_rule_id = generate_symbol_id(state, rule_name); -# pos = parse_alternates(state, pos, rule_name, sub_rule_id, true); -# last_sym_start = out_elements.size(); -# // output reference to synthesized rule -# out_elements.push_back({LLAMA_GRETYPE_RULE_REF, sub_rule_id}); -# if (*pos != ')') { -# throw std::runtime_error(std::string("expecting ')' at ") + pos); -# } -# pos = parse_space(pos + 1, is_nested); -# } else if (*pos == '.') { // any char -# last_sym_start = out_elements.size(); -# out_elements.push_back({LLAMA_GRETYPE_CHAR_ANY, 0}); -# pos = parse_space(pos + 1, is_nested); -# } else if (*pos == '*') { -# pos = parse_space(pos + 1, is_nested); -# handle_repetitions(0, -1); -# } else if (*pos == '+') { -# pos = parse_space(pos + 1, is_nested); -# handle_repetitions(1, -1); -# } else if (*pos == '?') { -# pos = parse_space(pos + 1, is_nested); -# handle_repetitions(0, 1); -# } else if (*pos == '{') { -# pos = parse_space(pos + 1, is_nested); -# -# if (!is_digit_char(*pos)) { -# throw std::runtime_error(std::string("expecting an int at ") + pos); -# } -# const char * int_end = parse_int(pos); -# int min_times = std::stoul(std::string(pos, int_end - pos)); -# pos = parse_space(int_end, is_nested); -# -# int max_times = -1; -# -# if (*pos == '}') { -# max_times = min_times; -# pos = parse_space(pos + 1, is_nested); -# } else if (*pos == ',') { -# pos = parse_space(pos + 1, is_nested); -# -# if (is_digit_char(*pos)) { -# const char * int_end = parse_int(pos); -# max_times = std::stoul(std::string(pos, int_end - pos)); -# pos = parse_space(int_end, is_nested); -# } -# -# if (*pos != '}') { -# throw std::runtime_error(std::string("expecting '}' at ") + pos); -# } -# pos = parse_space(pos + 1, is_nested); -# } else { -# throw std::runtime_error(std::string("expecting ',' at ") + pos); -# } -# handle_repetitions(min_times, max_times); -# } else { -# break; -# } -# } -# return pos; -# } -def parse_sequence(state: ParseState, src: str, rule_name: str, out_elements: typing.List[GrammarElement], is_nested: bool) -> str: - last_sym_start = len(out_elements) - pos = src - - def handle_repetitions(min_times: int, max_times: int) -> None: - nonlocal state, src, rule_name, out_elements, is_nested, last_sym_start, pos - - if last_sym_start == len(out_elements): - raise ValueError(f"expecting preceding item to */+/?/{{ at {pos}") - - previous_elements = out_elements[last_sym_start:] - if min_times == 0: - del out_elements[last_sym_start:] - else: - for i in range(1, min_times): - out_elements.extend(previous_elements) - - last_rec_rule_id = 0 - n_opt = 1 if max_times < 0 else max_times - min_times - - rec_rule = previous_elements[:] - for i in range(n_opt): - rec_rule = rec_rule[:len(previous_elements)] - rec_rule_id = generate_symbol_id(state, rule_name) - if i > 0 or max_times < 0: - rec_rule.append(GrammarElement(GrammarElementType.RULE_REF, rec_rule_id if max_times < 0 else last_rec_rule_id)) - rec_rule.append(GrammarElement(GrammarElementType.ALT, 0)) - rec_rule.append(GrammarElement(GrammarElementType.END, 0)) - add_rule(state, rec_rule_id, rec_rule) - last_rec_rule_id = rec_rule_id - if n_opt > 0: - out_elements.append(GrammarElement(GrammarElementType.RULE_REF, last_rec_rule_id)) - - while pos: - if pos[0] == '"': - pos = pos[1:] - last_sym_start = len(out_elements) - while not pos.startswith('"'): - if not pos: - raise ValueError("unexpected end of input") - char, pos = parse_char(pos) - out_elements.append(GrammarElement(GrammarElementType.CHAR, char)) - pos = parse_space(pos[1:], is_nested) - elif pos[0] == "[": - pos = pos[1:] - start_type = GrammarElementType.CHAR - if pos[0] == "^": - pos = pos[1:] - start_type = GrammarElementType.CHAR_NOT - last_sym_start = len(out_elements) - while pos[0] != "]": - if not pos: - raise ValueError("unexpected end of input") - char, pos = parse_char(pos) - type = GrammarElementType.CHAR_ALT if last_sym_start < len(out_elements) else start_type - out_elements.append(GrammarElement(type, char)) - if pos[0] == "-" and pos[1] != "]": - if not pos[1]: - raise ValueError("unexpected end of input") - endchar, pos = parse_char(pos[1:]) - out_elements.append(GrammarElement(GrammarElementType.CHAR_RNG_UPPER, endchar)) - pos = parse_space(pos[1:], is_nested) - elif pos and is_word_char(pos[0]): - name, rest = parse_name(pos) - ref_rule_id = get_symbol_id(state, name) - pos = parse_space(rest, is_nested) - last_sym_start = len(out_elements) - out_elements.append(GrammarElement(GrammarElementType.RULE_REF, ref_rule_id)) - elif pos.startswith("("): - pos = parse_space(pos[1:], newline_ok=True) - sub_rule_id = generate_symbol_id(state, rule_name) - pos = parse_alternates(state, pos, rule_name, sub_rule_id, is_nested=True) - last_sym_start = len(out_elements) - out_elements.append(GrammarElement(GrammarElementType.RULE_REF, sub_rule_id)) - if pos[0] != ")": - raise ValueError(f"expecting ')' at {pos}") - pos = parse_space(pos[1:], is_nested) - elif pos.startswith("."): - last_sym_start = len(out_elements) - out_elements.append(GrammarElement(GrammarElementType.CHAR_ANY, 0)) - pos = parse_space(pos[1:], is_nested) - elif pos.startswith("*"): - pos = parse_space(pos[1:], is_nested) - handle_repetitions(0, -1) - elif pos.startswith("+"): - pos = parse_space(pos[1:], is_nested) - handle_repetitions(1, -1) - elif pos.startswith("?"): - pos = parse_space(pos[1:], is_nested) - handle_repetitions(0, 1) - elif pos.startswith("{"): - pos = parse_space(pos[1:], is_nested) - - if not is_digit_char(pos): - raise ValueError(f"expecting an int at {pos}") - min_times, pos = parse_int(pos) - pos = parse_space(pos, is_nested) - - max_times = -1 - - if pos[0] == "}": - max_times = min_times - pos = parse_space(pos[1:], is_nested) - elif pos[0] == ",": - pos = parse_space(pos[1:], is_nested) - - if is_digit_char(pos): - max_times, pos = parse_int(pos) - pos = parse_space(pos, is_nested) - - if pos[0] != "}": - raise ValueError("expecting '}' at {}".format(pos)) - - pos = parse_space(pos[1:], is_nested) - else: - raise ValueError(f"expecting ',' at {pos}") - handle_repetitions(min_times, max_times) - else: - break - return pos - - -# const char * parse_alternates( -# parse_state & state, -# const char * src, -# const std::string & rule_name, -# uint32_t rule_id, -# bool is_nested) { -# std::vector rule; -# const char * pos = parse_sequence(state, src, rule_name, rule, is_nested); -# while (*pos == '|') { -# rule.push_back({LLAMA_GRETYPE_ALT, 0}); -# pos = parse_space(pos + 1, true); -# pos = parse_sequence(state, pos, rule_name, rule, is_nested); -# } -# rule.push_back({LLAMA_GRETYPE_END, 0}); -# add_rule(state, rule_id, rule); -# return pos; -# } -def parse_alternates(state: ParseState, src: str, rule_name: str, rule_id: int, is_nested: bool) -> str: - rule = [] - pos = parse_sequence(state, src, rule_name, rule, is_nested) - while pos.startswith("|"): - rule.append(GrammarElement(GrammarElementType.ALT, 0)) - pos = parse_space(pos[1:], newline_ok=True) - pos = parse_sequence(state, pos, rule_name, rule, is_nested) - rule.append(GrammarElement(GrammarElementType.END, 0)) - add_rule(state, rule_id, rule) - return pos - - -# static const char * parse_rule(parse_state & state, const char * src) { -# const char * name_end = parse_name(src); -# const char * pos = parse_space(name_end, false); -# size_t name_len = name_end - src; -# uint32_t rule_id = get_symbol_id(state, src, name_len); -# const std::string name(src, name_len); -# -# if (!(pos[0] == ':' && pos[1] == ':' && pos[2] == '=')) { -# throw std::runtime_error(std::string("expecting ::= at ") + pos); -# } -# pos = parse_space(pos + 3, true); -# -# pos = parse_alternates(state, pos, name, rule_id, false); -# -# if (*pos == '\r') { -# pos += pos[1] == '\n' ? 2 : 1; -# } else if (*pos == '\n') { -# pos++; -# } else if (*pos) { -# throw std::runtime_error(std::string("expecting newline or end at ") + pos); -# } -# return parse_space(pos, true); -# } -def parse_rule(state: ParseState, src: str) -> str: - pos = src - name, pos = parse_name(pos) - pos = parse_space(pos, newline_ok=False) - rule_id = get_symbol_id(state, name) - - if not pos.startswith("::="): - raise ValueError(f"expecting ::= at {pos}") - - pos = parse_space(pos[3:], newline_ok=True) - - pos = parse_alternates(state, pos, name, rule_id, is_nested=False) - - if pos.startswith("\r"): - pos = pos[2:] if pos[1] == "\n" else pos[1:] - elif pos.startswith("\n"): - pos = pos[1:] - elif pos: - raise ValueError(f"expecting newline or end at {pos}") - return parse_space(pos, newline_ok=True) - - -# parse_state parse(const char * src) { -# try { -# parse_state state; -# const char * pos = parse_space(src, true); -# while (*pos) { -# pos = parse_rule(state, pos); -# } -# // Validate the state to ensure that all rules are defined -# for (const auto & rule : state.rules) { -# for (const auto & elem : rule) { -# if (elem.type == LLAMA_GRETYPE_RULE_REF) { -# // Ensure that the rule at that location exists -# if (elem.value >= state.rules.size() || state.rules[elem.value].empty()) { -# // Get the name of the rule that is missing -# for (const auto & kv : state.symbol_ids) { -# if (kv.second == elem.value) { -# throw std::runtime_error("Undefined rule identifier '" + kv.first + "'"); -# } -# } -# } -# } -# } -# } -# return state; -# } catch (const std::exception & err) { -# fprintf(stderr, "%s: error parsing grammar: %s\n", __func__, err.what()); -# return parse_state(); -# } -# } -def parse(src: str) -> ParseState: - state = ParseState() - pos = src - pos = parse_space(pos, newline_ok=True) - while pos: - pos = parse_rule(state, pos) - # validate - for rule in state.rules: - for elem in rule: - if elem.type == GrammarElementType.RULE_REF: - if elem.value >= len(state.rules) or not state.rules[elem.value]: - for k, v in state.symbol_ids.items(): - if v == elem.value: - raise ValueError(f"Undefined rule identifier '{k}'") - return state - - -# static bool is_char_element(llama_grammar_element elem) { -# switch (elem.type) { -# case LLAMA_GRETYPE_CHAR: return true; -# case LLAMA_GRETYPE_CHAR_NOT: return true; -# case LLAMA_GRETYPE_CHAR_ALT: return true; -# case LLAMA_GRETYPE_CHAR_RNG_UPPER: return true; -# case LLAMA_GRETYPE_CHAR_ANY: return true; -# default: return false; -# } -# } -def is_char_element(elem: GrammarElement) -> bool: - return elem.type in ( - GrammarElementType.CHAR, - GrammarElementType.CHAR_NOT, - GrammarElementType.CHAR_ALT, - GrammarElementType.CHAR_RNG_UPPER, - GrammarElementType.CHAR_ANY - ) - - -def print_grammar_char(file: typing.TextIO, c: int) -> None: - if 0x20 <= c <= 0x7f: - print(chr(c), end="", file=file) - else: - print(f"", end="", file=file) - - -# static void print_rule( -# FILE * file, -# uint32_t rule_id, -# const std::vector & rule, -# const std::map & symbol_id_names) { -# if (rule.empty() || rule.back().type != LLAMA_GRETYPE_END) { -# throw std::runtime_error( -# "malformed rule, does not end with LLAMA_GRETYPE_END: " + std::to_string(rule_id)); -# } -# fprintf(file, "%s ::= ", symbol_id_names.at(rule_id).c_str()); -# for (size_t i = 0, end = rule.size() - 1; i < end; i++) { -# llama_grammar_element elem = rule[i]; -# switch (elem.type) { -# case LLAMA_GRETYPE_END: -# throw std::runtime_error( -# "unexpected end of rule: " + std::to_string(rule_id) + "," + -# std::to_string(i)); -# case LLAMA_GRETYPE_ALT: -# fprintf(file, "| "); -# break; -# case LLAMA_GRETYPE_RULE_REF: -# fprintf(file, "%s ", symbol_id_names.at(elem.value).c_str()); -# break; -# case LLAMA_GRETYPE_CHAR: -# fprintf(file, "["); -# print_grammar_char(file, elem.value); -# break; -# case LLAMA_GRETYPE_CHAR_NOT: -# fprintf(file, "[^"); -# print_grammar_char(file, elem.value); -# break; -# case LLAMA_GRETYPE_CHAR_RNG_UPPER: -# if (i == 0 || !is_char_element(rule[i - 1])) { -# throw std::runtime_error( -# "LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: " + -# std::to_string(rule_id) + "," + std::to_string(i)); -# } -# fprintf(file, "-"); -# print_grammar_char(file, elem.value); -# break; -# case LLAMA_GRETYPE_CHAR_ALT: -# if (i == 0 || !is_char_element(rule[i - 1])) { -# throw std::runtime_error( -# "LLAMA_GRETYPE_CHAR_ALT without preceding char: " + -# std::to_string(rule_id) + "," + std::to_string(i)); -# } -# print_grammar_char(file, elem.value); -# break; -# case LLAMA_GRETYPE_CHAR_ANY: -# fprintf(file, "."); -# break; -# } -# if (is_char_element(elem)) { -# switch (rule[i + 1].type) { -# case LLAMA_GRETYPE_CHAR_ALT: -# case LLAMA_GRETYPE_CHAR_RNG_UPPER: -# case LLAMA_GRETYPE_CHAR_ANY: -# break; -# default: -# fprintf(file, "] "); -# } -# } -# } -# fprintf(file, "\n"); -# } -def print_rule( - file: typing.TextIO, - rule_id: int, - rule: typing.List[GrammarElement], - symbol_id_names: typing.Dict[int, str], -) -> None: - if not rule or rule[-1].type != GrammarElementType.END: - raise ValueError(f"malformed rule, does not end with LLAMA_GRETYPE_END: {rule_id}") - - print(f"{symbol_id_names[rule_id]} ::=", end=" ", file=file) - - for i, elem in enumerate(rule[:-1]): - if elem.type == GrammarElementType.END: - raise ValueError(f"unexpected end of rule: {rule_id}, {i}") - if elem.type == GrammarElementType.ALT: - print("| ", end="", file=file) - elif elem.type == GrammarElementType.RULE_REF: - print(f"{symbol_id_names[elem.value]} ", end="", file=file) - elif elem.type == GrammarElementType.CHAR: - print("[", end="", file=file) - print_grammar_char(file, elem.value) - elif elem.type == GrammarElementType.CHAR_NOT: - print("[^", end="", file=file) - print_grammar_char(file, elem.value) - elif elem.type == GrammarElementType.CHAR_RNG_UPPER: - if i == 0 or not is_char_element(rule[i - 1]): - raise ValueError(f"LLAMA_GRETYPE_CHAR_RNG_UPPER without preceding char: {rule_id}, {i}") - print(f"-", end="", file=file) - print_grammar_char(file, elem.value) - elif elem.type == GrammarElementType.CHAR_ALT: - if i == 0 or not is_char_element(rule[i - 1]): - raise ValueError(f"LLAMA_GRETYPE_CHAR_ALT without preceding char: {rule_id}, {i}") - print_grammar_char(file, elem.value) - elif elem.type == GrammarElementType.CHAR_ANY: - print(".", end="", file=file) - if is_char_element(elem): - if rule[i + 1].type in (GrammarElementType.CHAR_ALT, GrammarElementType.CHAR_RNG_UPPER, GrammarElementType.CHAR_ANY): - continue - print("] ", end="", file=file) - print(file=file) - - -def print_grammar(file: typing.TextIO, state: ParseState) -> None: - try: - symbol_id_names = {v: k for k, v in state.symbol_ids.items()} - for i, rule in enumerate(state.rules): - print_rule(file, i, rule, symbol_id_names) - except Exception as err: - print(f"\nerror printing grammar: {err}", file=file) - raise err - +LLAMA_GRAMMAR_DEFAULT_ROOT = "root" class LlamaGrammar: - def __init__(self, parse_state: ParseState): - self.parse_state = parse_state - - self._grammar_rules = parse_state.rules - self._n_rules = len(self._grammar_rules) - self._start_rule_index = parse_state.symbol_ids["root"] - - self._element_lists = [ - [ - llama_cpp.llama_grammar_element(ctypes.c_int(elem.type), ctypes.c_uint32(elem.value)) - for elem in subvector - ] - for subvector in self._grammar_rules - ] - - # Step 2: Convert each list to llama_grammar_element array and get pointer - self._element_arrays = [ - (llama_cpp.llama_grammar_element * len(sublist))(*sublist) - for sublist in self._element_lists - ] - - # Step 3: Get pointer of each array - self._element_array_pointers = [ - ctypes.cast(subarray, llama_cpp.llama_grammar_element_p) for subarray in self._element_arrays - ] - - # Step 4: Make array of these pointers and get its pointer - self._rules = (llama_cpp.llama_grammar_element_p * len(self._element_array_pointers))( - *self._element_array_pointers - ) - - self.grammar = None - self._init_grammar() - - - def _init_grammar(self): - grammar = llama_cpp.llama_grammar_init( - self._rules, ctypes.c_size_t(self._n_rules), ctypes.c_size_t(self._start_rule_index) - ) - - if grammar is None: - raise ValueError("Failed to create grammar") - - self.grammar = grammar - - def __del__(self): - if self.grammar is not None: - llama_cpp.llama_grammar_free(self.grammar) - self.grammar = None - - def reset(self): - if self.grammar is not None: - llama_cpp.llama_grammar_free(self.grammar) - self._init_grammar() + def __init__(self, *args, _grammar: str, **kwargs): + self._grammar = _grammar + self._root = LLAMA_GRAMMAR_DEFAULT_ROOT @classmethod def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": - parsed_grammar = parse(grammar) - if verbose: - print_grammar(file=sys.stdout, state=parsed_grammar) - return cls(parsed_grammar) + return cls(_grammar=grammar) @classmethod def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGrammar": diff --git a/pyproject.toml b/pyproject.toml index ce50c673f4..9983ef777d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ test = [ "sse-starlette>=1.6.1", "starlette-context>=0.3.6,<0.4", "pydantic-settings>=2.0.1", + "huggingface-hub>=0.23.0" ] dev = [ "black>=23.3.0", diff --git a/tests/test_llama.py b/tests/test_llama.py index 469ef91cab..cf134c2e74 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -1,14 +1,24 @@ import ctypes +import multiprocessing import numpy as np -import pytest from scipy.special import log_softmax +from huggingface_hub import hf_hub_download + +import pytest + import llama_cpp +import llama_cpp._internals as internals + MODEL = "./vendor/llama.cpp/models/ggml-vocab-llama-spm.gguf" +def test_llama_cpp_version(): + assert llama_cpp.__version__ + + def test_llama_cpp_tokenization(): llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True, verbose=False) @@ -47,247 +57,117 @@ def test_llama_cpp_tokenization(): @pytest.fixture -def mock_llama(monkeypatch): - def setup_mock(llama: llama_cpp.Llama, output_text: str): - n_ctx = llama.n_ctx() - n_vocab = llama.n_vocab() - output_tokens = llama.tokenize( - output_text.encode("utf-8"), add_bos=True, special=True - ) - logits = (ctypes.c_float * (n_vocab * n_ctx))(-100.0) - for i in range(n_ctx): - output_idx = i + 1 # logits for first tokens predict second token - if output_idx < len(output_tokens): - logits[i * n_vocab + output_tokens[output_idx]] = 100.0 - else: - logits[i * n_vocab + llama.token_eos()] = 100.0 - n = 0 - last_n_tokens = 0 - - def mock_decode(ctx: llama_cpp.llama_context_p, batch: llama_cpp.llama_batch): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - assert batch.n_tokens > 0, "no tokens in batch" - assert all( - batch.n_seq_id[i] == 1 for i in range(batch.n_tokens) - ), "n_seq >1 not supported by mock_llama" - assert all( - batch.seq_id[i][0] == 0 for i in range(batch.n_tokens) - ), "n_seq >1 not supported by mock_llama" - assert batch.logits[ - batch.n_tokens - 1 - ], "logits not allocated for last token" - # Update the mock context state - nonlocal n - nonlocal last_n_tokens - n = max(batch.pos[i] for i in range(batch.n_tokens)) + 1 - last_n_tokens = batch.n_tokens - return 0 - - def mock_get_logits(ctx: llama_cpp.llama_context_p): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - assert n > 0, "mock_llama_decode not called" - assert last_n_tokens > 0, "mock_llama_decode not called" - # Return view of logits for last_n_tokens - return (ctypes.c_float * (last_n_tokens * n_vocab)).from_address( - ctypes.addressof(logits) - + (n - last_n_tokens) * n_vocab * ctypes.sizeof(ctypes.c_float) - ) - - monkeypatch.setattr("llama_cpp.llama_cpp.llama_decode", mock_decode) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_get_logits", mock_get_logits) - - def mock_kv_cache_clear(ctx: llama_cpp.llama_context_p): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - def mock_kv_cache_seq_rm( - ctx: llama_cpp.llama_context_p, - seq_id: llama_cpp.llama_seq_id, - pos0: llama_cpp.llama_pos, - pos1: llama_cpp.llama_pos, - ): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - def mock_kv_cache_seq_cp( - ctx: llama_cpp.llama_context_p, - seq_id_src: llama_cpp.llama_seq_id, - seq_id_dst: llama_cpp.llama_seq_id, - pos0: llama_cpp.llama_pos, - pos1: llama_cpp.llama_pos, - ): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - def mock_kv_cache_seq_keep( - ctx: llama_cpp.llama_context_p, - seq_id: llama_cpp.llama_seq_id, - ): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - def mock_kv_cache_seq_add( - ctx: llama_cpp.llama_context_p, - seq_id: llama_cpp.llama_seq_id, - pos0: llama_cpp.llama_pos, - pos1: llama_cpp.llama_pos, - ): - # Test some basic invariants of this mocking technique - assert ctx == llama._ctx.ctx, "context does not match mock_llama" - return - - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_clear", mock_kv_cache_clear) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_seq_rm", mock_kv_cache_seq_rm) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_seq_cp", mock_kv_cache_seq_cp) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_seq_keep", mock_kv_cache_seq_keep) - monkeypatch.setattr("llama_cpp.llama_cpp.llama_kv_cache_seq_add", mock_kv_cache_seq_add) - - return setup_mock - - -def test_llama_patch(mock_llama): - n_ctx = 128 - llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True, n_ctx=n_ctx) - n_vocab = llama_cpp.llama_n_vocab(llama._model.model) - assert n_vocab == 32000 - - text = "The quick brown fox" - output_text = " jumps over the lazy dog." - all_text = text + output_text - - ## Test basic completion from bos until eos - mock_llama(llama, all_text) - completion = llama.create_completion("", max_tokens=36) - assert completion["choices"][0]["text"] == all_text - assert completion["choices"][0]["finish_reason"] == "stop" - - ## Test basic completion until eos - mock_llama(llama, all_text) - completion = llama.create_completion(text, max_tokens=20) - assert completion["choices"][0]["text"] == output_text - assert completion["choices"][0]["finish_reason"] == "stop" - - ## Test streaming completion until eos - mock_llama(llama, all_text) - chunks = list(llama.create_completion(text, max_tokens=20, stream=True)) - assert "".join(chunk["choices"][0]["text"] for chunk in chunks) == output_text - assert chunks[-1]["choices"][0]["finish_reason"] == "stop" - - ## Test basic completion until stop sequence - mock_llama(llama, all_text) - completion = llama.create_completion(text, max_tokens=20, stop=["lazy"]) - assert completion["choices"][0]["text"] == " jumps over the " - assert completion["choices"][0]["finish_reason"] == "stop" - - ## Test streaming completion until stop sequence - mock_llama(llama, all_text) - chunks = list( - llama.create_completion(text, max_tokens=20, stream=True, stop=["lazy"]) - ) - assert ( - "".join(chunk["choices"][0]["text"] for chunk in chunks) == " jumps over the " +def llama_cpp_model_path(): + repo_id = "Qwen/Qwen2-0.5B-Instruct-GGUF" + filename = "qwen2-0_5b-instruct-q8_0.gguf" + model_path = hf_hub_download(repo_id, filename) + return model_path + + +def test_real_model(llama_cpp_model_path): + import os + assert os.path.exists(llama_cpp_model_path) + + params = llama_cpp.llama_model_default_params() + params.use_mmap = llama_cpp.llama_supports_mmap() + params.use_mlock = llama_cpp.llama_supports_mlock() + params.check_tensors = False + + model = internals.LlamaModel(path_model=llama_cpp_model_path, params=params) + + cparams = llama_cpp.llama_context_default_params() + cparams.n_ctx = 16 + cparams.n_batch = 16 + cparams.n_ubatch = 16 + cparams.n_threads = multiprocessing.cpu_count() + cparams.n_threads_batch = multiprocessing.cpu_count() + cparams.logits_all = False + cparams.flash_attn = True + + context = internals.LlamaContext(model=model, params=cparams) + tokens = model.tokenize(b"Hello, world!", add_bos=True, special=True) + + assert tokens == [9707, 11, 1879, 0] + + tokens = model.tokenize(b"The quick brown fox jumps", add_bos=True, special=True) + + batch = internals.LlamaBatch(n_tokens=len(tokens), embd=0, n_seq_max=1) + + seed = 1337 + sampler = internals.LlamaSampler() + sampler.add_top_k(50) + sampler.add_top_p(0.9, 1) + sampler.add_temp(0.8) + sampler.add_dist(seed) + + result = tokens + n_eval = 0 + for _ in range(4): + batch.set_batch(tokens, n_past=n_eval, logits_all=False) + context.decode(batch) + n_eval += len(tokens) + token_id = sampler.sample(context, -1) + tokens = [token_id] + result += tokens + + output = result[5:] + output_text = model.detokenize(output, special=True) + assert output_text == b" over the lazy dog" + +def test_real_llama(llama_cpp_model_path): + model = llama_cpp.Llama( + llama_cpp_model_path, + n_ctx=32, + n_batch=32, + n_ubatch=32, + n_threads=multiprocessing.cpu_count(), + n_threads_batch=multiprocessing.cpu_count(), + logits_all=False, + flash_attn=True, ) - assert chunks[-1]["choices"][0]["finish_reason"] == "stop" - - ## Test basic completion until length - mock_llama(llama, all_text) - completion = llama.create_completion(text, max_tokens=2) - assert completion["choices"][0]["text"] == " jumps" - assert completion["choices"][0]["finish_reason"] == "length" - - ## Test streaming completion until length - mock_llama(llama, all_text) - chunks = list(llama.create_completion(text, max_tokens=2, stream=True)) - assert "".join(chunk["choices"][0]["text"] for chunk in chunks) == " jumps" - assert chunks[-1]["choices"][0]["finish_reason"] == "length" - - -def test_llama_pickle(): - import pickle - import tempfile - - fp = tempfile.TemporaryFile() - llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True) - pickle.dump(llama, fp) - fp.seek(0) - llama = pickle.load(fp) - - assert llama - assert llama.ctx is not None - - text = b"Hello World" - - assert llama.detokenize(llama.tokenize(text)) == text - - -def test_utf8(mock_llama): - llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True, logits_all=True) - - output_text = "😀" - - ## Test basic completion with utf8 multibyte - mock_llama(llama, output_text) - completion = llama.create_completion("", max_tokens=4) - assert completion["choices"][0]["text"] == output_text - - ## Test basic completion with incomplete utf8 multibyte - mock_llama(llama, output_text) - completion = llama.create_completion("", max_tokens=1) - assert completion["choices"][0]["text"] == "" + output = model.create_completion( + "The quick brown fox jumps", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + seed=1337 + ) + assert output["choices"][0]["text"] == " over the lazy dog" + + + output = model.create_completion( + "The capital of france is paris, 'true' or 'false'?:\n", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + seed=1337, + grammar=llama_cpp.LlamaGrammar.from_string(""" +root ::= "true" | "false" +""") + ) + assert output["choices"][0]["text"] == "true" -def test_llama_server(): - from fastapi.testclient import TestClient - from llama_cpp.server.app import create_app, Settings + suffix = b"rot" + tokens = model.tokenize(suffix, add_bos=True, special=True) + def logit_processor_func(input_ids, logits): + for token in tokens: + logits[token] *= 1000 + return logits - settings = Settings( - model=MODEL, - vocab_only=True, + logit_processors = llama_cpp.LogitsProcessorList( + [logit_processor_func] ) - app = create_app(settings) - client = TestClient(app) - response = client.get("/v1/models") - assert response.json() == { - "object": "list", - "data": [ - { - "id": MODEL, - "object": "model", - "owned_by": "me", - "permissions": [], - } - ], - } - - -@pytest.mark.parametrize( - "size_and_axis", - [ - ((32_000,), -1), # last token's next-token logits - ((10, 32_000), -1), # many tokens' next-token logits, or batch of last tokens - ((4, 10, 32_000), -1), # batch of texts - ], -) -@pytest.mark.parametrize("convert_to_list", [True, False]) -def test_logits_to_logprobs(size_and_axis, convert_to_list: bool, atol: float = 1e-7): - size, axis = size_and_axis - logits: np.ndarray = -np.random.uniform(low=0, high=60, size=size) - logits = logits.astype(np.single) - if convert_to_list: - # Currently, logits are converted from arrays to lists. This may change soon - logits = logits.tolist() - log_probs = llama_cpp.Llama.logits_to_logprobs(logits, axis=axis) - log_probs_correct = log_softmax(logits, axis=axis) - assert log_probs.dtype == np.single - assert log_probs.shape == size - assert np.allclose(log_probs, log_probs_correct, atol=atol) - -def test_llama_cpp_version(): - assert llama_cpp.__version__ + output = model.create_completion( + "The capital of france is par", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + seed=1337, + logits_processor=logit_processors + ) + assert output["choices"][0]["text"].lower().startswith("rot") diff --git a/tests/test_llama_grammar.py b/tests/test_llama_grammar.py index cb221880a6..34ef2874df 100644 --- a/tests/test_llama_grammar.py +++ b/tests/test_llama_grammar.py @@ -10,9 +10,9 @@ def test_grammar_from_string(): grammar = llama_cpp.LlamaGrammar.from_string(tree) - assert grammar._n_rules == 3 - assert grammar._start_rule_index == 2 - assert grammar.grammar is not None + # assert grammar._n_rules == 3 + # assert grammar._start_rule_index == 2 + # assert grammar.grammar is not None def test_composed_pydantic_grammar(): @@ -49,7 +49,7 @@ class B(BaseModel): grammar = llama_cpp.LlamaGrammar.from_json_schema(json.dumps(schema)) - assert grammar.grammar is not None + # assert grammar.grammar is not None def test_grammar_anyof(): @@ -75,4 +75,4 @@ def test_grammar_anyof(): grammar = llama_cpp.LlamaGrammar.from_json_schema(json.dumps(sch)) - assert grammar.grammar is not None \ No newline at end of file + # assert grammar.grammar is not None diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8ebe8ddebd..6262d13e0b 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8ebe8ddebd68526757c631cd019de009697c63c2 +Subproject commit 6262d13e0b2da91f230129a93a996609a2f5a2f2 From 1e64664e0facb7e3595efdf4716f01527f4047ba Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 18 Sep 2024 20:07:48 -0400 Subject: [PATCH 322/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 227 +++++++++++++++++++++++++++-------------- vendor/llama.cpp | 2 +- 2 files changed, 153 insertions(+), 76 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 261d7c6b41..6b82753e1d 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1161,7 +1161,8 @@ def llama_backend_init(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): ... +def llama_numa_init(numa: int, /): + ... # // Optional: an auto threadpool gets created in ggml if not passed explicitly @@ -1196,7 +1197,8 @@ def llama_backend_free(): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: ... +) -> Optional[llama_model_p]: + ... # LLAMA_API void llama_free_model(struct llama_model * model); @@ -1205,7 +1207,8 @@ def llama_load_model_from_file( [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): ... +def llama_free_model(model: llama_model_p, /): + ... # LLAMA_API struct llama_context * llama_new_context_with_model( @@ -1218,7 +1221,8 @@ def llama_free_model(model: llama_model_p, /): ... ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: ... +) -> Optional[llama_context_p]: + ... # // Frees all allocated memory @@ -1239,87 +1243,110 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: ... +def llama_time_us() -> int: + ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: ... +def llama_max_devices() -> int: + ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: ... +def llama_supports_mmap() -> bool: + ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: ... +def llama_supports_mlock() -> bool: + ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: ... +def llama_supports_gpu_offload() -> bool: + ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: ... +def llama_n_batch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); @ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... +def llama_n_ubatch(ctx: llama_context_p, /) -> int: + ... # LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); @ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... +def llama_n_seq_max(ctx: llama_context_p, /) -> int: + ... # LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); @ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: ... +def llama_n_vocab(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_embd (const struct llama_model * model); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: ... +def llama_n_embd(model: llama_model_p, /) -> int: + ... # LLAMA_API int32_t llama_n_layer (const struct llama_model * model); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: ... +def llama_n_layer(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_n_head (const struct llama_model * model); +@ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32) +def llama_n_head(model: llama_model_p, /) -> int: + ... # LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: + ... # LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) -def llama_pooling_type(ctx: llama_context_p, /) -> int: ... +def llama_pooling_type(ctx: llama_context_p, /) -> int: + ... # LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: ... +def llama_vocab_type(model: llama_model_p, /) -> int: + ... # LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); @ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: ... +def llama_rope_type(model: llama_model_p, /) -> int: + ... # // Get the model's RoPE frequency scaling factor @@ -2110,7 +2137,8 @@ def llama_state_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_load_session_file( @@ -2138,7 +2166,8 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: ... +) -> int: + ... # LLAMA_API bool llama_state_save_file( @@ -2162,7 +2191,8 @@ def llama_state_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: ... +) -> bool: + ... # LLAMA_API DEPRECATED(bool llama_save_session_file( @@ -2187,7 +2217,8 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: ... +) -> int: + ... # // Get the exact size needed to copy the KV cache of a single sequence @@ -2285,7 +2316,8 @@ def llama_state_seq_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: ... +) -> int: + ... # LLAMA_API size_t llama_state_seq_load_file( @@ -2315,7 +2347,8 @@ def llama_state_seq_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: ... +) -> int: + ... # // @@ -2620,7 +2653,8 @@ def llama_get_embeddings_seq( ) def llama_token_get_text( model: llama_model_p, token: Union[llama_token, int], / -) -> bytes: ... +) -> bytes: + ... # LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); @@ -2629,7 +2663,8 @@ def llama_token_get_text( ) def llama_token_get_score( model: llama_model_p, token: Union[llama_token, int], / -) -> float: ... +) -> float: + ... # LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token); @@ -2638,7 +2673,8 @@ def llama_token_get_score( ) def llama_token_get_attr( model: llama_model_p, token: Union[llama_token, int], / -) -> int: ... +) -> int: + ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) @@ -2703,12 +2739,14 @@ def llama_token_nl(model: llama_model_p, /) -> int: # LLAMA_API bool llama_add_bos_token(const struct llama_model * model); @ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_bos_token(model: llama_model_p, /) -> bool: ... +def llama_add_bos_token(model: llama_model_p, /) -> bool: + ... # LLAMA_API bool llama_add_eos_token(const struct llama_model * model); @ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_eos_token(model: llama_model_p, /) -> bool: ... +def llama_add_eos_token(model: llama_model_p, /) -> bool: + ... # // Codellama infill tokens @@ -2721,17 +2759,20 @@ def llama_token_prefix(model: llama_model_p) -> int: # LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: ... +def llama_token_middle(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: ... +def llama_token_suffix(model: llama_model_p, /) -> int: + ... # LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle @ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: ... +def llama_token_eot(model: llama_model_p, /) -> int: + ... # // @@ -2932,7 +2973,8 @@ def llama_chat_apply_template( chat: CtypesArray[llama_chat_message], n_msg: int, /, -) -> int: ... +) -> int: + ... # // @@ -2991,7 +3033,8 @@ def llama_chat_apply_template( # // TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph # //void (*apply_ggml) (struct llama_sampler * smpl, ...); # }; -class llama_sampler_i(ctypes.Structure): ... +class llama_sampler_i(ctypes.Structure): + ... # struct llama_sampler { @@ -3036,7 +3079,8 @@ class llama_sampler(ctypes.Structure): [llama_sampler_p_ctypes], ctypes.c_char_p, ) -def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: ... +def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: + ... # LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token); @@ -3045,7 +3089,8 @@ def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: ... [llama_sampler_p_ctypes, llama_token], None, ) -def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /): ... +def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /): + ... # LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p); @@ -3056,7 +3101,8 @@ def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], ) def llama_sampler_apply( smpl: llama_sampler_p, cur_p: CtypesArray[llama_token_data_array], / -): ... +): + ... # LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl); @@ -3065,7 +3111,8 @@ def llama_sampler_apply( [llama_sampler_p_ctypes], None, ) -def llama_sampler_reset(smpl: llama_sampler_p, /): ... +def llama_sampler_reset(smpl: llama_sampler_p, /): + ... # LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl); @@ -3074,7 +3121,8 @@ def llama_sampler_reset(smpl: llama_sampler_p, /): ... [llama_sampler_p_ctypes], llama_sampler_p_ctypes, ) -def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... +def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: + ... # // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) @@ -3084,7 +3132,8 @@ def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... [llama_sampler_p_ctypes], None, ) -def llama_sampler_free(smpl: llama_sampler_p, /): ... +def llama_sampler_free(smpl: llama_sampler_p, /): + ... # // llama_sampler_chain @@ -3096,9 +3145,8 @@ def llama_sampler_free(smpl: llama_sampler_p, /): ... [llama_sampler_chain_params], llama_sampler_p_ctypes, ) -def llama_sampler_chain_init( - params: llama_sampler_chain_params, / -) -> llama_sampler_p: ... +def llama_sampler_chain_init(params: llama_sampler_chain_params, /) -> llama_sampler_p: + ... # // important: takes ownership of the sampler object and will free it when llama_sampler_free is called @@ -3108,7 +3156,8 @@ def llama_sampler_chain_init( [llama_sampler_p_ctypes, llama_sampler_p_ctypes], None, ) -def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): ... +def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): + ... # LLAMA_API struct llama_sampler * llama_sampler_chain_get(const struct llama_sampler * chain, int32_t i); @@ -3119,7 +3168,8 @@ def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): . ) def llama_sampler_chain_get( chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); @@ -3128,7 +3178,8 @@ def llama_sampler_chain_get( [llama_sampler_p_ctypes], ctypes.c_int, ) -def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ... +def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: + ... # // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed @@ -3140,31 +3191,36 @@ def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ... ) def llama_sampler_chain_remove( chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # // available samplers: # # LLAMA_API struct llama_sampler * llama_sampler_init_greedy (void); @ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes) -def llama_sampler_init_greedy() -> llama_sampler_p: ... +def llama_sampler_init_greedy() -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed); @ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes) -def llama_sampler_init_dist(seed: int) -> llama_sampler_p: ... +def llama_sampler_init_dist(seed: int) -> llama_sampler_p: + ... # /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. # LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void); @ctypes_function("llama_sampler_init_softmax", [], llama_sampler_p_ctypes) -def llama_sampler_init_softmax() -> llama_sampler_p: ... +def llama_sampler_init_softmax() -> llama_sampler_p: + ... # /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 # LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); @ctypes_function("llama_sampler_init_top_k", [ctypes.c_int32], llama_sampler_p_ctypes) -def llama_sampler_init_top_k(k: int) -> llama_sampler_p: ... +def llama_sampler_init_top_k(k: int) -> llama_sampler_p: + ... # /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 @@ -3174,7 +3230,8 @@ def llama_sampler_init_top_k(k: int) -> llama_sampler_p: ... [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: ... +def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: + ... # /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841 @@ -3184,7 +3241,8 @@ def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: ... [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ... +def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: + ... # /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. @@ -3194,7 +3252,8 @@ def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ... [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: ... +def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: + ... # /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. @@ -3204,12 +3263,14 @@ def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: .. [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: ... +def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); @ctypes_function("llama_sampler_init_temp", [ctypes.c_float], llama_sampler_p_ctypes) -def llama_sampler_init_temp(t: float) -> llama_sampler_p: ... +def llama_sampler_init_temp(t: float) -> llama_sampler_p: + ... # /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. @@ -3221,7 +3282,8 @@ def llama_sampler_init_temp(t: float) -> llama_sampler_p: ... ) def llama_sampler_init_temp_ext( t: float, delta: float, exponent: float -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3243,7 +3305,8 @@ def llama_sampler_init_temp_ext( ) def llama_sampler_init_mirostat( n_vocab: int, seed: int, tau: float, eta: float, m: int, / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3262,7 +3325,8 @@ def llama_sampler_init_mirostat( ) def llama_sampler_init_mirostat_v2( seed: int, tau: float, eta: float, / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_grammar( @@ -3276,7 +3340,8 @@ def llama_sampler_init_mirostat_v2( ) def llama_sampler_init_grammar( model: llama_model_p, grammar_str: bytes, grammar_root: bytes, / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( @@ -3315,7 +3380,8 @@ def llama_sampler_init_penalties( penalize_nl: bool, ignore_eos: bool, /, -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( @@ -3329,7 +3395,8 @@ def llama_sampler_init_penalties( ) def llama_sampler_init_logit_bias( n_vocab: int, n_logit_bias: int, logit_bias: CtypesArray[llama_logit_bias], / -) -> llama_sampler_p: ... +) -> llama_sampler_p: + ... # // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise @@ -3339,7 +3406,8 @@ def llama_sampler_init_logit_bias( [llama_sampler_p_ctypes], ctypes.c_uint32, ) -def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ... +def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: + ... # /// @details Sample and accept a token from the idx-th output of the last evaluation @@ -3360,7 +3428,8 @@ def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ... ) def llama_sampler_sample( smpl: llama_sampler_p, ctx: llama_context_p, idx: int, / -) -> int: ... +) -> int: + ... # // @@ -3413,7 +3482,8 @@ def llama_split_prefix( # // Print system information # LLAMA_API const char * llama_print_system_info(void); @ctypes_function("llama_print_system_info", [], ctypes.c_char_p) -def llama_print_system_info() -> bytes: ... +def llama_print_system_info() -> bytes: + ... # // Set callback for all future logging events. @@ -3480,7 +3550,8 @@ class llama_perf_sampler_data(ctypes.Structure): [llama_context_p_ctypes], llama_perf_context_data, ) -def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: ... +def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: + ... # LLAMA_API void llama_perf_context_print(const struct llama_context * ctx); @@ -3489,7 +3560,8 @@ def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: ... [llama_context_p_ctypes], None, ) -def llama_perf_context_print(ctx: llama_context_p, /): ... +def llama_perf_context_print(ctx: llama_context_p, /): + ... # LLAMA_API void llama_perf_context_reset( struct llama_context * ctx); @@ -3498,7 +3570,8 @@ def llama_perf_context_print(ctx: llama_context_p, /): ... [llama_context_p_ctypes], None, ) -def llama_perf_context_reset(ctx: llama_context_p, /): ... +def llama_perf_context_reset(ctx: llama_context_p, /): + ... # // NOTE: the following work only with samplers constructed via llama_sampler_chain_init @@ -3508,7 +3581,8 @@ def llama_perf_context_reset(ctx: llama_context_p, /): ... [llama_sampler_p_ctypes], llama_perf_sampler_data, ) -def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: ... +def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: + ... # LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); @@ -3517,7 +3591,8 @@ def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: .. [llama_sampler_p_ctypes], None, ) -def llama_perf_sampler_print(chain: llama_sampler_p, /): ... +def llama_perf_sampler_print(chain: llama_sampler_p, /): + ... # LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); @@ -3526,7 +3601,8 @@ def llama_perf_sampler_print(chain: llama_sampler_p, /): ... [llama_sampler_p_ctypes], None, ) -def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... +def llama_perf_sampler_reset(chain: llama_sampler_p, /): + ... # LLAMA_API void llama_perf_dump_yaml(FILE * stream, const struct llama_context * ctx); @@ -3537,4 +3613,5 @@ def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... ) def llama_perf_dump_yaml( stream: ctypes.POINTER(ctypes.c_void_p), ctx: llama_context_p, / -): ... +): + ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6262d13e0b..64c6af3195 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6262d13e0b2da91f230129a93a996609a2f5a2f2 +Subproject commit 64c6af3195c3cd4aa3328a1282d29cd2635c34c9 From 9b64bb5b137385cdf535598df5b2c34ed459450f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 18 Sep 2024 20:11:02 -0400 Subject: [PATCH 323/561] misc: Format --- llama_cpp/_internals.py | 66 ++++++++++++++++++------ llama_cpp/llama.py | 98 ++++++++++++++++++++---------------- llama_cpp/llama_cache.py | 6 +-- llama_cpp/llama_grammar.py | 3 +- llama_cpp/llama_tokenizer.py | 33 ++++++++---- llama_cpp/llava_cpp.py | 21 +++++--- 6 files changed, 147 insertions(+), 80 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 6475695c63..0aff348442 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -100,7 +100,6 @@ def n_params(self) -> int: def get_tensor(self, name: str) -> ctypes.c_void_p: return llama_cpp.llama_get_model_tensor(self.model, name.encode("utf-8")) - # Vocab def token_get_text(self, token: int) -> str: @@ -460,9 +459,7 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - batch = llama_cpp.llama_batch_init( - self._n_tokens, self.embd, self.n_seq_max - ) + batch = llama_cpp.llama_batch_init(self._n_tokens, self.embd, self.n_seq_max) if batch is None: raise ValueError("Failed to create llama_batch") @@ -541,6 +538,7 @@ def copy_logits(self, logits: npt.NDArray[np.single]): # Embedding functions + def normalize_embedding(embedding): norm = float(np.linalg.norm(embedding)) if norm == 0.0: @@ -713,11 +711,17 @@ def accept(self, ctx_main: LlamaContext, id: int, apply_grammar: bool): import ctypes import llama_cpp + class CustomSampler: - def __init__(self, apply_func: typing.Callable[[llama_cpp.llama_token_data_array], None]): + def __init__( + self, apply_func: typing.Callable[[llama_cpp.llama_token_data_array], None] + ): self.apply_func = apply_func - def apply_wrapper(sampler: llama_cpp.llama_sampler_p, cur_p: llama_cpp.llama_token_data_array_p): + def apply_wrapper( + sampler: llama_cpp.llama_sampler_p, + cur_p: llama_cpp.llama_token_data_array_p, + ): self.apply_func(cur_p) def free_wrapper(sampler: llama_cpp.llama_sampler_p): @@ -740,6 +744,7 @@ def free_wrapper(sampler: llama_cpp.llama_sampler_p): def get_sampler(self) -> llama_cpp.llama_sampler_p: return ctypes.pointer(self.sampler) + class LlamaSampler: def __init__(self): params = llama_cpp.llama_sampler_chain_params() @@ -788,9 +793,7 @@ def add_temp_ext(self, t: float, delta: float, exponent: float): self._add_sampler(sampler) def add_mirostat(self, n_vocab: int, seed: int, tau: float, eta: float, m: int): - sampler = llama_cpp.llama_sampler_init_mirostat( - n_vocab, seed, tau, eta, m - ) + sampler = llama_cpp.llama_sampler_init_mirostat(n_vocab, seed, tau, eta, m) self._add_sampler(sampler) def add_mirostat_v2(self, seed: int, tau: float, eta: float): @@ -798,23 +801,54 @@ def add_mirostat_v2(self, seed: int, tau: float, eta: float): self._add_sampler(sampler) def add_grammar(self, model: LlamaModel, grammar: LlamaGrammar): - sampler = llama_cpp.llama_sampler_init_grammar(model.model, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8")) + sampler = llama_cpp.llama_sampler_init_grammar( + model.model, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8") + ) self._add_sampler(sampler) - def add_penalties(self, n_vocab: int, special_eos_id: int, linefeed_id: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, penalize_nl: bool, ignore_eos: bool): - sampler = llama_cpp.llama_sampler_init_penalties(n_vocab, special_eos_id, linefeed_id, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, penalize_nl, ignore_eos) + def add_penalties( + self, + n_vocab: int, + special_eos_id: int, + linefeed_id: int, + penalty_last_n: int, + penalty_repeat: float, + penalty_freq: float, + penalty_present: float, + penalize_nl: bool, + ignore_eos: bool, + ): + sampler = llama_cpp.llama_sampler_init_penalties( + n_vocab, + special_eos_id, + linefeed_id, + penalty_last_n, + penalty_repeat, + penalty_freq, + penalty_present, + penalize_nl, + ignore_eos, + ) self._add_sampler(sampler) - def init_logit_bias(self, n_vocab: int, n_logit_bias, logit_bias: llama_cpp.llama_logit_bias_p): - sampler = llama_cpp.llama_sampler_init_logit_bias(n_vocab, n_logit_bias, logit_bias) + def init_logit_bias( + self, n_vocab: int, n_logit_bias, logit_bias: llama_cpp.llama_logit_bias_p + ): + sampler = llama_cpp.llama_sampler_init_logit_bias( + n_vocab, n_logit_bias, logit_bias + ) self._add_sampler(sampler) - def add_custom(self, apply_func: Callable[[llama_cpp.llama_token_data_array], None]): + def add_custom( + self, apply_func: Callable[[llama_cpp.llama_token_data_array], None] + ): custom_sampler = CustomSampler(apply_func) sampler = custom_sampler.get_sampler() self._add_sampler(sampler) # NOTE: Must remove custom samplers before free or llama.cpp will try to free them - self.custom_samplers.append((llama_cpp.llama_sampler_chain_n(self.sampler) - 1, custom_sampler)) + self.custom_samplers.append( + (llama_cpp.llama_sampler_chain_n(self.sampler) - 1, custom_sampler) + ) def _add_sampler(self, sampler: llama_cpp.llama_sampler_p): assert self.sampler is not None diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 557134a6ea..7dd8625156 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -255,28 +255,28 @@ def __init__( for i, (k, v) in enumerate(kv_overrides.items()): self._kv_overrides_array[i].key = k.encode("utf-8") if isinstance(v, bool): - self._kv_overrides_array[i].tag = ( - llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL - ) + self._kv_overrides_array[ + i + ].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL self._kv_overrides_array[i].value.val_bool = v elif isinstance(v, int): - self._kv_overrides_array[i].tag = ( - llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT - ) + self._kv_overrides_array[ + i + ].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT self._kv_overrides_array[i].value.val_i64 = v elif isinstance(v, float): - self._kv_overrides_array[i].tag = ( - llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT - ) + self._kv_overrides_array[ + i + ].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT self._kv_overrides_array[i].value.val_f64 = v elif isinstance(v, str): # type: ignore v_bytes = v.encode("utf-8") if len(v_bytes) > 128: # TODO: Make this a constant raise ValueError(f"Value for {k} is too long: {v}") v_bytes = v_bytes.ljust(128, b"\0") - self._kv_overrides_array[i].tag = ( - llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR - ) + self._kv_overrides_array[ + i + ].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR # copy min(v_bytes, 128) to str_value address = typing.cast( int, @@ -292,9 +292,9 @@ def __init__( else: raise ValueError(f"Unknown value type for {k}: {v}") - self._kv_overrides_array[-1].key = ( - b"\0" # ensure sentinel element is zeroed - ) + self._kv_overrides_array[ + -1 + ].key = b"\0" # ensure sentinel element is zeroed self.model_params.kv_overrides = self._kv_overrides_array self.n_batch = min(n_ctx, n_batch) # ??? @@ -431,9 +431,9 @@ def free_lora_adapter(): self.chat_format = chat_format self.chat_handler = chat_handler - self._chat_handlers: Dict[str, llama_chat_format.LlamaChatCompletionHandler] = ( - {} - ) + self._chat_handlers: Dict[ + str, llama_chat_format.LlamaChatCompletionHandler + ] = {} self.draft_model = draft_model @@ -580,7 +580,10 @@ def tokenize( return self.tokenizer_.tokenize(text, add_bos, special) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False + self, + tokens: List[int], + prev_tokens: Optional[List[int]] = None, + special: bool = False, ) -> bytes: """Detokenize a list of tokens. @@ -592,7 +595,9 @@ def detokenize( Returns: The detokenized string. """ - return self.tokenizer_.detokenize(tokens, prev_tokens=prev_tokens, special=special) + return self.tokenizer_.detokenize( + tokens, prev_tokens=prev_tokens, special=special + ) def set_cache(self, cache: Optional[BaseLlamaCache]): """Set the cache. @@ -681,12 +686,16 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): recarray = np.recarray( shape=(size,), dtype=np.dtype( - [("id", np.intc), ("logit", np.single), ("p", np.single)], align=True + [("id", np.intc), ("logit", np.single), ("p", np.single)], + align=True, + ), + buf=(llama_cpp.llama_token_data * size).from_address( + data_soa_address ), - buf=(llama_cpp.llama_token_data * size).from_address(data_soa_address), ) for logit_processor in logits_processor: recarray.logit[:] = logit_processor(self._input_ids, recarray.logit) + sampler.add_custom(apply_func) sampler.add_penalties( @@ -698,7 +707,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): penalty_freq=frequency_penalty, penalty_present=presence_penalty, penalize_nl=penalize_nl, - ignore_eos=False + ignore_eos=False, ) if grammar is not None: @@ -841,22 +850,22 @@ def generate( # Reset mirostat sampling self._mirostat_mu = ctypes.c_float(2.0 * mirostat_tau) self._sampler = self._init_sampler( - top_k=top_k, - top_p=top_p, - min_p=min_p, - typical_p=typical_p, - temp=temp, - repeat_penalty=repeat_penalty, - frequency_penalty=frequency_penalty, - presence_penalty=presence_penalty, - tfs_z=tfs_z, - mirostat_mode=mirostat_mode, - mirostat_tau=mirostat_tau, - mirostat_eta=mirostat_eta, - penalize_nl=penalize_nl, - logits_processor=logits_processor, - grammar=grammar, - seed=seed, + top_k=top_k, + top_p=top_p, + min_p=min_p, + typical_p=typical_p, + temp=temp, + repeat_penalty=repeat_penalty, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + tfs_z=tfs_z, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + penalize_nl=penalize_nl, + logits_processor=logits_processor, + grammar=grammar, + seed=seed, ) # Check for kv cache prefix match @@ -872,8 +881,11 @@ def generate( tokens = tokens[longest_prefix:] self.n_tokens = longest_prefix if self.verbose: - print(f"Llama.generate: {longest_prefix} prefix-match hit, " - f"remaining {len(tokens)} prompt tokens to eval", file=sys.stderr) + print( + f"Llama.generate: {longest_prefix} prefix-match hit, " + f"remaining {len(tokens)} prompt tokens to eval", + file=sys.stderr, + ) # Reset the model state if reset: @@ -1032,7 +1044,9 @@ def decode_batch(seq_sizes: List[int]): for j in range(size) ] if normalize: - embedding = [internals.normalize_embedding(e) for e in embedding] + embedding = [ + internals.normalize_embedding(e) for e in embedding + ] data.append(embedding) pos += size else: diff --git a/llama_cpp/llama_cache.py b/llama_cpp/llama_cache.py index 5220c79337..e059e98e1b 100644 --- a/llama_cpp/llama_cache.py +++ b/llama_cpp/llama_cache.py @@ -52,9 +52,9 @@ class LlamaRAMCache(BaseLlamaCache): def __init__(self, capacity_bytes: int = (2 << 30)): super().__init__(capacity_bytes) self.capacity_bytes = capacity_bytes - self.cache_state: OrderedDict[Tuple[int, ...], "llama_cpp.llama.LlamaState"] = ( - OrderedDict() - ) + self.cache_state: OrderedDict[ + Tuple[int, ...], "llama_cpp.llama.LlamaState" + ] = OrderedDict() @property def cache_size(self): diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 785aa88d73..b95c77ab5e 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -15,6 +15,7 @@ LLAMA_GRAMMAR_DEFAULT_ROOT = "root" + class LlamaGrammar: def __init__(self, *args, _grammar: str, **kwargs): self._grammar = _grammar @@ -23,7 +24,7 @@ def __init__(self, *args, _grammar: str, **kwargs): @classmethod def from_string(cls, grammar: str, verbose: bool = True) -> "LlamaGrammar": return cls(_grammar=grammar) - + @classmethod def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGrammar": try: diff --git a/llama_cpp/llama_tokenizer.py b/llama_cpp/llama_tokenizer.py index 2e7590d144..1375e1392d 100644 --- a/llama_cpp/llama_tokenizer.py +++ b/llama_cpp/llama_tokenizer.py @@ -27,7 +27,10 @@ def tokenize( @abc.abstractmethod def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False + self, + tokens: List[int], + prev_tokens: Optional[List[int]] = None, + special: bool = False, ) -> bytes: """Detokenize the tokens into text. @@ -49,7 +52,10 @@ def tokenize( return self._model.tokenize(text, add_bos=add_bos, special=special) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False + self, + tokens: List[int], + prev_tokens: Optional[List[int]] = None, + special: bool = False, ) -> bytes: return self._model.detokenize(tokens, special=special) @@ -80,19 +86,24 @@ def tokenize( ) def detokenize( - self, tokens: List[int], prev_tokens: Optional[List[int]] = None, special: bool = False + self, + tokens: List[int], + prev_tokens: Optional[List[int]] = None, + special: bool = False, ) -> bytes: - skip_special_tokens = not special + skip_special_tokens = not special if prev_tokens is not None: - text = self.hf_tokenizer.decode(prev_tokens + tokens, skip_special_tokens=skip_special_tokens).encode( - "utf-8", errors="ignore" - ) - prev_text = self.hf_tokenizer.decode(prev_tokens, skip_special_tokens=skip_special_tokens).encode( - "utf-8", errors="ignore" - ) + text = self.hf_tokenizer.decode( + prev_tokens + tokens, skip_special_tokens=skip_special_tokens + ).encode("utf-8", errors="ignore") + prev_text = self.hf_tokenizer.decode( + prev_tokens, skip_special_tokens=skip_special_tokens + ).encode("utf-8", errors="ignore") return text[len(prev_text) :] else: - return self.hf_tokenizer.decode(tokens, skip_special_tokens=skip_special_tokens).encode("utf-8", errors="ignore") + return self.hf_tokenizer.decode( + tokens, skip_special_tokens=skip_special_tokens + ).encode("utf-8", errors="ignore") @classmethod def from_pretrained(cls, pretrained_model_name_or_path: str) -> "LlamaHFTokenizer": diff --git a/llama_cpp/llava_cpp.py b/llama_cpp/llava_cpp.py index b80d859133..7d97dc0fd0 100644 --- a/llama_cpp/llava_cpp.py +++ b/llama_cpp/llava_cpp.py @@ -165,7 +165,8 @@ class llava_image_embed(Structure): ) def llava_validate_embed_size( ctx_llama: llama_cpp.llama_context_p, ctx_clip: clip_ctx_p, / -) -> bool: ... +) -> bool: + ... # /** build an image embed from image file bytes */ @@ -181,7 +182,8 @@ def llava_image_embed_make_with_bytes( image_bytes: CtypesArray[c_uint8], image_bytes_length: Union[c_int, int], /, -) -> "_Pointer[llava_image_embed]": ... +) -> "_Pointer[llava_image_embed]": + ... # /** build an image embed from a path to an image filename */ @@ -193,13 +195,15 @@ def llava_image_embed_make_with_bytes( ) def llava_image_embed_make_with_filename( ctx_clip: clip_ctx_p, n_threads: Union[c_int, int], image_path: bytes, / -) -> "_Pointer[llava_image_embed]": ... +) -> "_Pointer[llava_image_embed]": + ... # LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed); # /** free an embedding made with llava_image_embed_make_* */ @ctypes_function("llava_image_embed_free", [POINTER(llava_image_embed)], None) -def llava_image_embed_free(embed: "_Pointer[llava_image_embed]", /): ... +def llava_image_embed_free(embed: "_Pointer[llava_image_embed]", /): + ... # /** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */ @@ -220,7 +224,8 @@ def llava_eval_image_embed( n_batch: Union[c_int, int], n_past: "_Pointer[c_int]", /, -) -> bool: ... +) -> bool: + ... ################################################ @@ -233,10 +238,12 @@ def llava_eval_image_embed( @ctypes_function("clip_model_load", [c_char_p, c_int], clip_ctx_p_ctypes) def clip_model_load( fname: bytes, verbosity: Union[c_int, int], / -) -> Optional[clip_ctx_p]: ... +) -> Optional[clip_ctx_p]: + ... # /** free mmproj model */ # CLIP_API void clip_free(struct clip_ctx * ctx); @ctypes_function("clip_free", [clip_ctx_p_ctypes], None) -def clip_free(ctx: clip_ctx_p, /): ... +def clip_free(ctx: clip_ctx_p, /): + ... From 22cedad8a9f010bdea6186ee564da7aaa21b6684 Mon Sep 17 00:00:00 2001 From: Xu Song Date: Thu, 19 Sep 2024 10:11:01 +0800 Subject: [PATCH 324/561] fix: Fix memory allocation of ndarray (#1704) * Fix memory allocation of ndarray * Add basic LlamaState tests * Improve LlamaState test and fix rng / seed --------- Co-authored-by: Andrei --- llama_cpp/llama.py | 34 +++++++++++++++++++--------------- tests/test_llama.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 7dd8625156..80fadffe24 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -7,6 +7,7 @@ import json import ctypes import typing +import random import fnmatch import warnings import contextlib @@ -301,9 +302,11 @@ def __init__( self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1) self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count() + # Used by the sampler + self._seed = seed or llama_cpp.LLAMA_DEFAULT_SEED + # Context Params self.context_params = llama_cpp.llama_context_default_params() - self.context_params.seed = seed self.context_params.n_ctx = n_ctx self.context_params.n_batch = self.n_batch self.context_params.n_threads = self.n_threads @@ -613,8 +616,7 @@ def set_seed(self, seed: int): Args: seed: The random seed. """ - # TODO: Fix this - # llama_cpp.llama_set_rng_seed(self._ctx.ctx, seed) + self._seed = seed def reset(self): """Reset the model state.""" @@ -672,7 +674,6 @@ def _init_sampler( penalize_nl: bool = True, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - seed: Optional[int] = None, ): sampler = internals.LlamaSampler() @@ -715,7 +716,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): if temp < 0.0: sampler.add_softmax() - sampler.add_dist(seed or llama_cpp.LLAMA_DEFAULT_SEED) + sampler.add_dist(self._seed) elif temp == 0.0: sampler.add_greedy() else: @@ -723,14 +724,14 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): mirostat_m = 100 sampler.add_mirostat( self._n_vocab, - seed or llama_cpp.LLAMA_DEFAULT_SEED, + self._seed, mirostat_tau, mirostat_eta, mirostat_m, ) elif mirostat_mode == 2: sampler.add_mirostat_v2( - seed or llama_cpp.LLAMA_DEFAULT_SEED, + self._seed, mirostat_tau, mirostat_eta, ) @@ -743,7 +744,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): sampler.add_top_p(top_p, min_keep) sampler.add_min_p(min_p, min_keep) sampler.add_temp(temp) - sampler.add_dist(seed or llama_cpp.LLAMA_DEFAULT_SEED) + sampler.add_dist(self._seed) return sampler def sample( @@ -826,7 +827,6 @@ def generate( logits_processor: Optional[LogitsProcessorList] = None, stopping_criteria: Optional[StoppingCriteriaList] = None, grammar: Optional[LlamaGrammar] = None, - seed: Optional[int] = None, ) -> Generator[int, Optional[Sequence[int]], None]: """Create a generator of tokens from a prompt. @@ -865,7 +865,6 @@ def generate( penalize_nl=penalize_nl, logits_processor=logits_processor, grammar=grammar, - seed=seed, ) # Check for kv cache prefix match @@ -1301,9 +1300,10 @@ def logit_bias_processor( if self.verbose: print("Llama._create_completion: cache miss", file=sys.stderr) - # TODO: Fix this - # if seed is not None: - # self._ctx.set_rng_seed(seed) + if seed is not None: + self.set_seed(seed) + else: + self.set_seed(random.Random(self._seed).randint(0, 2 ** 32)) finish_reason = "length" multibyte_fix = 0 @@ -1324,7 +1324,6 @@ def logit_bias_processor( stopping_criteria=stopping_criteria, logits_processor=logits_processor, grammar=grammar, - seed=seed, ): if llama_cpp.llama_token_is_eog(self._model.model, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) @@ -2136,14 +2135,17 @@ def save_state(self) -> LlamaState: n_tokens=self.n_tokens, llama_state=bytes(llama_state_compact), llama_state_size=n_bytes, + seed=self._seed, ) def load_state(self, state: LlamaState) -> None: # Only filling in up to `n_tokens` and then zero-ing out the rest self.scores[: state.n_tokens, :] = state.scores.copy() - self.scores[state.n_tokens :, :] = 0.0 + rest = self.scores[state.n_tokens :, :] + rest[rest > 0] = 0.0 self.input_ids = state.input_ids.copy() self.n_tokens = state.n_tokens + self._seed = state.seed state_size = state.llama_state_size LLamaStateArrayType = ctypes.c_uint8 * state_size llama_state = LLamaStateArrayType.from_buffer_copy(state.llama_state) @@ -2321,12 +2323,14 @@ def __init__( n_tokens: int, llama_state: bytes, llama_state_size: int, + seed: int, ): self.input_ids = input_ids self.scores = scores self.n_tokens = n_tokens self.llama_state = llama_state self.llama_state_size = llama_state_size + self.seed = seed LogitsProcessor = Callable[ diff --git a/tests/test_llama.py b/tests/test_llama.py index cf134c2e74..fc182ae20f 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -171,3 +171,48 @@ def logit_processor_func(input_ids, logits): logits_processor=logit_processors ) assert output["choices"][0]["text"].lower().startswith("rot") + + model.set_seed(1337) + + state = model.save_state() + + output = model.create_completion( + "Pick a number from 1 to 10?:\n", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + grammar=llama_cpp.LlamaGrammar.from_string(""" +root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" +""") + ) + number_1 = output["choices"][0]["text"] + + output = model.create_completion( + "Pick a number from 1 to 10?:\n", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + grammar=llama_cpp.LlamaGrammar.from_string(""" +root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" +""") + ) + number_2 = output["choices"][0]["text"] + + model.load_state(state) + + output = model.create_completion( + "Pick a number from 1 to 10?:\n", + max_tokens=4, + top_k=50, + top_p=0.9, + temperature=0.8, + grammar=llama_cpp.LlamaGrammar.from_string(""" +root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" +""") + ) + number_3 = output["choices"][0]["text"] + + assert number_1 != number_2 + assert number_1 == number_3 From 29afcfdff5e75d7df4c13bad0122c98661d251ab Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 18 Sep 2024 23:04:59 -0400 Subject: [PATCH 325/561] fix: Don't store scores internally unless logits_all=True. Reduces memory requirements for large context. Closes #1542 --- llama_cpp/llama.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 80fadffe24..70623d813c 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -451,7 +451,7 @@ def free_lora_adapter(): self.n_tokens = 0 self.input_ids: npt.NDArray[np.intc] = np.ndarray((n_ctx,), dtype=np.intc) self.scores: npt.NDArray[np.single] = np.ndarray( - (n_ctx, self._n_vocab), dtype=np.single + (n_ctx if logits_all == True else n_batch, self._n_vocab), dtype=np.single ) self._mirostat_mu = ctypes.c_float( @@ -648,12 +648,14 @@ def eval(self, tokens: Sequence[int]): ) self.scores[n_past : n_past + n_tokens, :].reshape(-1)[::] = logits else: - rows = 1 - cols = self._n_vocab - logits = np.ctypeslib.as_array( - self._ctx.get_logits(), shape=(rows * cols,) - ) - self.scores[n_past + n_tokens - 1, :].reshape(-1)[::] = logits + # rows = 1 + # cols = self._n_vocab + # logits = np.ctypeslib.as_array( + # self._ctx.get_logits(), shape=(rows * cols,) + # ) + # self.scores[n_past + n_tokens - 1, :].reshape(-1)[::] = logits + # NOTE: Now that sampling is done inside the sampler, logits are only needed for logprobs which requires logits_all + pass # Update n_tokens self.n_tokens += n_tokens From 84c092063e8f222758dd3d60bdb2d1d342ac292e Mon Sep 17 00:00:00 2001 From: Jonathan Jordan Date: Thu, 19 Sep 2024 05:09:01 +0200 Subject: [PATCH 326/561] feat: Add loading sharded GGUF files from HuggingFace with Llama.from_pretrained(additional_files=[...]) . Closes #1341 Co-authored-by: Andrei --- llama_cpp/llama.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 70623d813c..3505dcfa16 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2227,6 +2227,7 @@ def from_pretrained( cls, repo_id: str, filename: Optional[str], + additional_files: Optional[List] = None, local_dir: Optional[Union[str, os.PathLike[str]]] = None, local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", cache_dir: Optional[Union[str, os.PathLike[str]]] = None, @@ -2239,6 +2240,7 @@ def from_pretrained( Args: repo_id: The model repo id. filename: A filename or glob pattern to match the model file in the repo. + additional_files: A list of filenames or glob patterns to match additional model files in the repo. local_dir: The local directory to save the model to. local_dir_use_symlinks: Whether to use symlinks when downloading the model. **kwargs: Additional keyword arguments to pass to the Llama constructor. @@ -2269,6 +2271,7 @@ def from_pretrained( rel_path = Path(file).relative_to(repo_id) file_list.append(str(rel_path)) + # find the only/first shard file: matching_files = [file for file in file_list if fnmatch.fnmatch(file, filename)] # type: ignore if len(matching_files) == 0: @@ -2298,6 +2301,35 @@ def from_pretrained( cache_dir=cache_dir, ) + if additional_files: + for additonal_file_name in additional_files: + # find the additional shard file: + matching_additional_files = [file for file in file_list if fnmatch.fnmatch(file, additonal_file_name)] + + if len(matching_additional_files) == 0: + raise ValueError( + f"No file found in {repo_id} that match {additonal_file_name}\n\n" + f"Available Files:\n{json.dumps(file_list)}" + ) + + if len(matching_additional_files) > 1: + raise ValueError( + f"Multiple files found in {repo_id} matching {additonal_file_name}\n\n" + f"Available Files:\n{json.dumps(files)}" + ) + + (matching_additional_file,) = matching_additional_files + + # download the additional file + hf_hub_download( + repo_id=repo_id, + filename=matching_additional_file, + subfolder=subfolder, + local_dir=local_dir, + local_dir_use_symlinks=local_dir_use_symlinks, + cache_dir=cache_dir, + ) + if local_dir is None: model_path = hf_hub_download( repo_id=repo_id, @@ -2311,6 +2343,7 @@ def from_pretrained( else: model_path = os.path.join(local_dir, filename) + # loading the first file of a sharded GGUF loads all remaining shard files in the subfolder return cls( model_path=model_path, **kwargs, From 47d7a621335d4f1bc7b96e4d9a39d663c32db22b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 20 Sep 2024 18:02:11 -0400 Subject: [PATCH 327/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 64c6af3195..6026da52d6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 64c6af3195c3cd4aa3328a1282d29cd2635c34c9 +Subproject commit 6026da52d6942b253df835070619775d849d0258 From 6c44a3f36b089239cb6396bb408116aad262c702 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 20 Sep 2024 18:03:37 -0400 Subject: [PATCH 328/561] feat: Add option to configure n_ubatch --- llama_cpp/llama.py | 5 +++++ llama_cpp/server/model.py | 1 + llama_cpp/server/settings.py | 3 +++ 3 files changed, 9 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 3505dcfa16..ae712dedba 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -75,6 +75,7 @@ def __init__( seed: int = llama_cpp.LLAMA_DEFAULT_SEED, n_ctx: int = 512, n_batch: int = 512, + n_ubatch: int = 512, n_threads: Optional[int] = None, n_threads_batch: Optional[int] = None, rope_scaling_type: Optional[ @@ -156,6 +157,7 @@ def __init__( seed: RNG seed, -1 for random n_ctx: Text context, 0 = from model n_batch: Prompt processing maximum batch size + n_ubatch: Physical batch size n_threads: Number of threads to use for generation n_threads_batch: Number of threads to use for batch processing rope_scaling_type: RoPE scaling type, from `enum llama_rope_scaling_type`. ref: https://github.com/ggerganov/llama.cpp/pull/2054 @@ -309,6 +311,7 @@ def __init__( self.context_params = llama_cpp.llama_context_default_params() self.context_params.n_ctx = n_ctx self.context_params.n_batch = self.n_batch + self.context_params.n_ubatch = min(self.n_batch, n_ubatch) self.context_params.n_threads = self.n_threads self.context_params.n_threads_batch = self.n_threads_batch self.context_params.rope_scaling_type = ( @@ -380,6 +383,7 @@ def __init__( self.n_batch = min(n_ctx, n_batch) self.context_params.n_ctx = self._model.n_ctx_train() self.context_params.n_batch = self.n_batch + self.context_params.n_ubatch = min(self.n_batch, n_ubatch) self._ctx = self._stack.enter_context( contextlib.closing( @@ -2071,6 +2075,7 @@ def __getstate__(self): seed=self.context_params.seed, n_ctx=self.context_params.n_ctx, n_batch=self.n_batch, + n_ubatch=self.context_params.n_ubatch, n_threads=self.context_params.n_threads, n_threads_batch=self.context_params.n_threads_batch, rope_scaling_type=self.context_params.rope_scaling_type, diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 901998094f..c6716f9196 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -249,6 +249,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: seed=settings.seed, n_ctx=settings.n_ctx, n_batch=settings.n_batch, + n_ubatch=settings.n_ubatch, n_threads=settings.n_threads, n_threads_batch=settings.n_threads_batch, rope_scaling_type=settings.rope_scaling_type, diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index b206558133..13c9512419 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -70,6 +70,9 @@ class ModelSettings(BaseSettings): n_batch: int = Field( default=512, ge=1, description="The batch size to use per eval." ) + n_ubatch: int = Field( + default=512, ge=1, description="The physical batch size used by llama.cpp" + ) n_threads: int = Field( default=max(multiprocessing.cpu_count() // 2, 1), ge=1, From 49b1e7379f5bb308acdba470e561f280008d8f15 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Sat, 21 Sep 2024 00:04:34 +0200 Subject: [PATCH 329/561] docs: Add cuda 12.5 to README.md (#1750) Typo fix Co-authored-by: Andrei --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 12b4b127d1..dbaec50775 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Where `` is one of the following: - `cu122`: CUDA 12.2 - `cu123`: CUDA 12.3 - `cu124`: CUDA 12.4 -- `cu124`: CUDA 12.5 +- `cu125`: CUDA 12.5 For example, to install the CUDA 12.1 wheel: From 1324c0c50c53e7987ff2ad1ba6d2bbe3fb13af69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 18:04:59 -0400 Subject: [PATCH 330/561] chore(deps): bump actions/cache from 3 to 4 (#1751) Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei --- .github/workflows/build-wheels-cuda.yaml | 2 +- .github/workflows/test.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index b1e5fc1a9c..647b171e84 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -72,7 +72,7 @@ jobs: - name: VS Integration Cache id: vs-integration-cache if: runner.os == 'Windows' - uses: actions/cache@v4.0.2 + uses: actions/cache@v4 with: path: ./MSBuildExtensions key: cuda-${{ matrix.cuda }}-vs-integration diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f24f5b7e18..8f619cf1f4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -46,7 +46,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Restore model cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} @@ -86,7 +86,7 @@ jobs: cache: 'pip' - name: Restore model cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} @@ -129,7 +129,7 @@ jobs: cache: 'pip' - name: Restore model cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} @@ -168,7 +168,7 @@ jobs: python-version: "3.9" - name: Restore model cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} From 4744551b3b777d9889ee0fdd25409101d57d59f5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 21 Sep 2024 20:53:16 -0400 Subject: [PATCH 331/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6026da52d6..d09770cae7 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6026da52d6942b253df835070619775d849d0258 +Subproject commit d09770cae71b416c032ec143dda530f7413c4038 From 926b4143e865bc6ffd405a952639099214385139 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 25 Sep 2024 13:23:18 -0400 Subject: [PATCH 332/561] feat: Update llama.cpp --- llama_cpp/_logger.py | 7 +++++++ vendor/llama.cpp | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/llama_cpp/_logger.py b/llama_cpp/_logger.py index 157af692fe..787b3f108e 100644 --- a/llama_cpp/_logger.py +++ b/llama_cpp/_logger.py @@ -10,6 +10,7 @@ # GGML_LOG_LEVEL_WARN = 2, # GGML_LOG_LEVEL_ERROR = 3, # GGML_LOG_LEVEL_DEBUG = 4, +# GGML_LOG_LEVEL_CONT = 5, // continue previous log # }; GGML_LOG_LEVEL_TO_LOGGING_LEVEL = { 0: logging.CRITICAL, @@ -17,10 +18,12 @@ 2: logging.WARNING, 3: logging.ERROR, 4: logging.DEBUG, + 5: logging.DEBUG, } logger = logging.getLogger("llama-cpp-python") +_last_log_level = GGML_LOG_LEVEL_TO_LOGGING_LEVEL[0] # typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); @llama_cpp.llama_log_callback @@ -29,8 +32,12 @@ def llama_log_callback( text: bytes, user_data: ctypes.c_void_p, ): + # TODO: Correctly implement continue previous log + global _last_log_level + log_level = GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level] if level != 5 else _last_log_level if logger.level <= GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level]: print(text.decode("utf-8"), end="", flush=True, file=sys.stderr) + _last_log_level = log_level llama_cpp.llama_log_set(llama_log_callback, ctypes.c_void_p(0)) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index d09770cae7..ea9c32be71 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit d09770cae71b416c032ec143dda530f7413c4038 +Subproject commit ea9c32be71b91b42ecc538bd902e93cbb5fb36cb From b3dfb420e79fcdb326bc3bac861f73a152380946 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 25 Sep 2024 14:05:08 -0400 Subject: [PATCH 333/561] chore: Bump version --- CHANGELOG.md | 13 +++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f01b1b9bfb..14f68a4563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ea9c32be71b91b42ecc538bd902e93cbb5fb36cb +- feat: Enable detokenizing special tokens with special=True by @benniekiss in #1596 +- feat(ci): Speed up CI workflows using uv, add support for CUDA 12.5 wheels by @Smartappli in e529940f45d42ed8aa31334123b8d66bc67b0e78 +- feat: Add loading sharded GGUF files from HuggingFace with Llama.from_pretrained(additional_files=[...]) by @Gnurro in 84c092063e8f222758dd3d60bdb2d1d342ac292e +- feat: Add option to configure n_ubatch by @abetlen in 6c44a3f36b089239cb6396bb408116aad262c702 +- feat: Update sampling API for llama.cpp. Sampling now uses sampler chain by @abetlen in f8fcb3ea3424bcfba3a5437626a994771a02324b +- fix: Don't store scores internally unless logits_all=True. Reduces memory requirements for large context by @abetlen in 29afcfdff5e75d7df4c13bad0122c98661d251ab +- fix: Fix memory allocation of ndarray in by @xu-song in #1704 +- fix: Use system message in og qwen format by @abetlen in 98eb092d3c6e7c142c4ba2faaca6c091718abbb3 + + ## [0.2.90] - feat: Update llama.cpp to ggerganov/llama.cpp@1d1ccce67613674c75c9c7e3fa4c1e24e428ba48 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index db1f1d0fa4..70fde20412 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.2.90" +__version__ = "0.3.0" From 8e07db0f87f2e0cd543f1449a5139b7026a19e9f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 25 Sep 2024 14:17:04 -0400 Subject: [PATCH 334/561] fix: install build dependency --- .github/workflows/publish.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 67816ea03b..bb76f53941 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -13,34 +13,36 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" - + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.9" - + - name: Install dependencies (Linux/MacOS) if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install build shell: bash - name: Install dependencies (Windows) if: runner.os == 'Windows' env: - RUST_LOG: trace + RUST_LOG: trace run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose + python -m uv pip install build shell: cmd - + - name: Build source distribution run: | python -m build --sdist - + - name: Publish distribution to PyPI # TODO: move to tag based releases # if: startsWith(github.ref, 'refs/tags') From 65222bc4931e8d5cf637a8d5f966da2f46897837 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 25 Sep 2024 14:27:19 -0400 Subject: [PATCH 335/561] fix: install build dependency --- .github/workflows/build-and-release.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 3f85a30fe8..bfa97decdd 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -92,7 +92,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: "recursive" - + - uses: actions/setup-python@v5 with: python-version: "3.9" @@ -103,6 +103,7 @@ jobs: python -m pip install --upgrade pip python -m pip install uv RUST_LOG=trace python -m uv pip install -e .[all] --verbose + python -m uv pip install build shell: bash - name: Install dependencies (Windows) @@ -113,12 +114,13 @@ jobs: python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose + python -m uv pip install build shell: cmd - + - name: Build source distribution run: | python -m build --sdist - + - uses: actions/upload-artifact@v4 with: name: sdist From 9992c5084a3df2f533e265d10f81d4269b97a1e6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 26 Sep 2024 00:34:21 -0400 Subject: [PATCH 336/561] fix: Fix speculative decoding --- llama_cpp/llama.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index ae712dedba..2ae65a83db 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -807,8 +807,10 @@ def sample( grammar=grammar, ) + ridx = idx - self.n_tokens if idx is not None else -1 + assert self.ctx is not None - token = self._sampler.sample(self._ctx, -1) + token = self._sampler.sample(self._ctx, ridx) if tmp_sampler: self._sampler = None return token From 11d9562fd63473dbf4ec91d942ca5f197deec462 Mon Sep 17 00:00:00 2001 From: Xu Song Date: Thu, 26 Sep 2024 12:37:17 +0800 Subject: [PATCH 337/561] misc: Rename all_text to remaining_text (#1658) Co-authored-by: Andrei --- llama_cpp/llama.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2ae65a83db..4f22fae842 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1519,15 +1519,15 @@ def logit_bias_processor( if stream: remaining_tokens = completion_tokens[returned_tokens:] - all_text = self.detokenize( + remaining_text = self.detokenize( remaining_tokens, prev_tokens=prompt_tokens + completion_tokens[:returned_tokens], ) - any_stop = [s for s in stop_sequences if s in all_text] + any_stop = [s for s in stop_sequences if s in remaining_text] if len(any_stop) > 0: - end = min(all_text.index(stop) for stop in any_stop) + end = min(remaining_text.index(stop) for stop in any_stop) else: - end = len(all_text) + end = len(remaining_text) token_end_position = 0 for token in remaining_tokens: From e975dabf74b3ad85689c9a07719cbb181313139b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 26 Sep 2024 01:11:02 -0400 Subject: [PATCH 338/561] fix: Additional fixes for speculative decoding --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 4f22fae842..babb30cf09 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -930,7 +930,7 @@ def generate( sample_idx += 1 if stopping_criteria is not None and stopping_criteria( - self._input_ids, self._scores[-1, :] + self._input_ids[: sample_idx], self._scores[sample_idx - self.n_tokens, :] ): return tokens_or_none = yield token From dca0c9a09cea39a8454dd897253be540e82f369d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 26 Sep 2024 10:09:22 -0400 Subject: [PATCH 339/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ea9c32be71..95bc82fbc0 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ea9c32be71b91b42ecc538bd902e93cbb5fb36cb +Subproject commit 95bc82fbc0df6d48cf66c857a4dda3d044f45ca2 From 01c76078eea77f12fb01aff605859afc372bf598 Mon Sep 17 00:00:00 2001 From: Andrei Date: Thu, 26 Sep 2024 11:23:00 -0400 Subject: [PATCH 340/561] feat: Expose libggml in internal APIs (#1761) * Expose libggml and refactor ctypes extension * Only expose libggml * Use ctypes_extensions module for libllama and libllava --- llama_cpp/_ctypes_extensions.py | 131 ++++++++++++++++++++++++++++++ llama_cpp/_ggml.py | 12 +++ llama_cpp/llama_cpp.py | 140 ++++---------------------------- llama_cpp/llava_cpp.py | 113 +++----------------------- 4 files changed, 171 insertions(+), 225 deletions(-) create mode 100644 llama_cpp/_ctypes_extensions.py create mode 100644 llama_cpp/_ggml.py diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py new file mode 100644 index 0000000000..e88ed387df --- /dev/null +++ b/llama_cpp/_ctypes_extensions.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import sys +import os +import ctypes +import functools +import pathlib + +from typing import ( + Any, + Callable, + List, + Union, + Optional, + TYPE_CHECKING, + TypeVar, + Generic, +) +from typing_extensions import TypeAlias + + +# Load the library +def load_shared_library(lib_base_name: str, base_path: pathlib.Path): + """Platform independent shared library loader""" + # Searching for the library in the current directory under the name "libllama" (default name + # for llamacpp) and "llama" (default name for this repo) + lib_paths: List[pathlib.Path] = [] + # Determine the file extension based on the platform + if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): + lib_paths += [ + base_path / f"lib{lib_base_name}.so", + ] + elif sys.platform == "darwin": + lib_paths += [ + base_path / f"lib{lib_base_name}.so", + base_path / f"lib{lib_base_name}.dylib", + ] + elif sys.platform == "win32": + lib_paths += [ + base_path / f"{lib_base_name}.dll", + base_path / f"lib{lib_base_name}.dll", + ] + else: + raise RuntimeError("Unsupported platform") + + cdll_args = dict() # type: ignore + + # Add the library directory to the DLL search path on Windows (if needed) + if sys.platform == "win32": + os.add_dll_directory(str(base_path)) + os.environ["PATH"] = str(base_path) + os.pathsep + os.environ["PATH"] + + if sys.platform == "win32" and sys.version_info >= (3, 8): + os.add_dll_directory(str(base_path)) + if "CUDA_PATH" in os.environ: + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin")) + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib")) + if "HIP_PATH" in os.environ: + os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) + os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) + cdll_args["winmode"] = ctypes.RTLD_GLOBAL + + # Try to load the shared library, handling potential errors + for lib_path in lib_paths: + if lib_path.exists(): + try: + return ctypes.CDLL(str(lib_path), **cdll_args) # type: ignore + except Exception as e: + raise RuntimeError(f"Failed to load shared library '{lib_path}': {e}") + + raise FileNotFoundError( + f"Shared library with base name '{lib_base_name}' not found" + ) + + +# ctypes sane type hint helpers +# +# - Generic Pointer and Array types +# - PointerOrRef type with a type hinted byref function +# +# NOTE: Only use these for static type checking not for runtime checks +# no good will come of that + +if TYPE_CHECKING: + CtypesCData = TypeVar("CtypesCData", bound=ctypes._CData) # type: ignore + + CtypesArray: TypeAlias = ctypes.Array[CtypesCData] # type: ignore + + CtypesPointer: TypeAlias = ctypes._Pointer[CtypesCData] # type: ignore + + CtypesVoidPointer: TypeAlias = ctypes.c_void_p + + class CtypesRef(Generic[CtypesCData]): + pass + + CtypesPointerOrRef: TypeAlias = Union[ + CtypesPointer[CtypesCData], CtypesRef[CtypesCData] + ] + + CtypesFuncPointer: TypeAlias = ctypes._FuncPointer # type: ignore + +F = TypeVar("F", bound=Callable[..., Any]) + + +def ctypes_function_for_shared_library(lib: ctypes.CDLL): + """Decorator for defining ctypes functions with type hints""" + + def ctypes_function( + name: str, argtypes: List[Any], restype: Any, enabled: bool = True + ): + def decorator(f: F) -> F: + if enabled: + func = getattr(lib, name) + func.argtypes = argtypes + func.restype = restype + functools.wraps(f)(func) + return func + else: + return f + + return decorator + + return ctypes_function + + +def _byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCData]: + """Type-annotated version of ctypes.byref""" + ... + + +byref = _byref if TYPE_CHECKING else ctypes.byref diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py new file mode 100644 index 0000000000..5bee8a93b5 --- /dev/null +++ b/llama_cpp/_ggml.py @@ -0,0 +1,12 @@ +"""Internal module use at your own risk + +This module provides a minimal interface for working with ggml tensors from llama-cpp-python +""" +import os +import pathlib + +import llama_cpp._ctypes_extensions as ctypes_ext + +libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" +libggml = ctypes_ext.load_shared_library("ggml", libggml_base_path) + diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 6b82753e1d..de427970c2 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1,151 +1,45 @@ from __future__ import annotations -import sys import os import ctypes -import functools import pathlib from typing import ( - Any, Callable, - List, Union, NewType, Optional, TYPE_CHECKING, - TypeVar, - Generic, ) -from typing_extensions import TypeAlias +from llama_cpp._ctypes_extensions import ( + load_shared_library, + byref, + ctypes_function_for_shared_library, +) -# Load the library -def _load_shared_library(lib_base_name: str): - # Construct the paths to the possible shared library names - _base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" - # Searching for the library in the current directory under the name "libllama" (default name - # for llamacpp) and "llama" (default name for this repo) - _lib_paths: List[pathlib.Path] = [] - # Determine the file extension based on the platform - if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): - _lib_paths += [ - _base_path / f"lib{lib_base_name}.so", - ] - elif sys.platform == "darwin": - _lib_paths += [ - _base_path / f"lib{lib_base_name}.so", - _base_path / f"lib{lib_base_name}.dylib", - ] - elif sys.platform == "win32": - _lib_paths += [ - _base_path / f"{lib_base_name}.dll", - _base_path / f"lib{lib_base_name}.dll", - ] - else: - raise RuntimeError("Unsupported platform") - - if "LLAMA_CPP_LIB" in os.environ: - lib_base_name = os.environ["LLAMA_CPP_LIB"] - _lib = pathlib.Path(lib_base_name) - _base_path = _lib.parent.resolve() - _lib_paths = [_lib.resolve()] - - cdll_args = dict() # type: ignore - - # Add the library directory to the DLL search path on Windows (if needed) - if sys.platform == "win32": - os.add_dll_directory(str(_base_path)) - os.environ["PATH"] = str(_base_path) + os.pathsep + os.environ["PATH"] - - if sys.platform == "win32" and sys.version_info >= (3, 8): - os.add_dll_directory(str(_base_path)) - if "CUDA_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib")) - if "HIP_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) - cdll_args["winmode"] = ctypes.RTLD_GLOBAL - - # Try to load the shared library, handling potential errors - for _lib_path in _lib_paths: - if _lib_path.exists(): - try: - return ctypes.CDLL(str(_lib_path), **cdll_args) # type: ignore - except Exception as e: - raise RuntimeError(f"Failed to load shared library '{_lib_path}': {e}") - - raise FileNotFoundError( - f"Shared library with base name '{lib_base_name}' not found" +if TYPE_CHECKING: + from llama_cpp._ctypes_extensions import ( + CtypesCData, + CtypesArray, + CtypesPointer, + CtypesVoidPointer, + CtypesRef, + CtypesPointerOrRef, + CtypesFuncPointer, ) # Specify the base name of the shared library to load _lib_base_name = "llama" - +_override_base_path = os.environ.get("LLAMA_CPP_LIB_PATH") +_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _override_base_path is None else pathlib.Path(_override_base_path) # Load the library -_lib = _load_shared_library(_lib_base_name) - - -# ctypes sane type hint helpers -# -# - Generic Pointer and Array types -# - PointerOrRef type with a type hinted byref function -# -# NOTE: Only use these for static type checking not for runtime checks -# no good will come of that - -if TYPE_CHECKING: - CtypesCData = TypeVar("CtypesCData", bound=ctypes._CData) # type: ignore - - CtypesArray: TypeAlias = ctypes.Array[CtypesCData] # type: ignore - - CtypesPointer: TypeAlias = ctypes._Pointer[CtypesCData] # type: ignore - - CtypesVoidPointer: TypeAlias = ctypes.c_void_p - - class CtypesRef(Generic[CtypesCData]): - pass - - CtypesPointerOrRef: TypeAlias = Union[ - CtypesPointer[CtypesCData], CtypesRef[CtypesCData] - ] - - CtypesFuncPointer: TypeAlias = ctypes._FuncPointer # type: ignore - -F = TypeVar("F", bound=Callable[..., Any]) - - -def ctypes_function_for_shared_library(lib: ctypes.CDLL): - def ctypes_function( - name: str, argtypes: List[Any], restype: Any, enabled: bool = True - ): - def decorator(f: F) -> F: - if enabled: - func = getattr(lib, name) - func.argtypes = argtypes - func.restype = restype - functools.wraps(f)(func) - return func - else: - return f - - return decorator - - return ctypes_function - +_lib = load_shared_library(_lib_base_name, _base_path) ctypes_function = ctypes_function_for_shared_library(_lib) -def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCData]: - """Type-annotated version of ctypes.byref""" - ... - - -byref = ctypes.byref # type: ignore - # from ggml.h # // NOTE: always add types at the end of the enum to keep backward compatibility # enum ggml_type { diff --git a/llama_cpp/llava_cpp.py b/llama_cpp/llava_cpp.py index 7d97dc0fd0..d9dfaf5fd0 100644 --- a/llama_cpp/llava_cpp.py +++ b/llama_cpp/llava_cpp.py @@ -1,9 +1,6 @@ from __future__ import annotations -import sys import os -import ctypes -import functools from ctypes import ( c_bool, c_char_p, @@ -17,121 +14,32 @@ ) import pathlib from typing import ( - List, Union, NewType, Optional, - TypeVar, - Callable, - Any, TYPE_CHECKING, - Generic, ) -from typing_extensions import TypeAlias import llama_cpp.llama_cpp as llama_cpp +from llama_cpp._ctypes_extensions import ( + load_shared_library, + ctypes_function_for_shared_library, +) -# Load the library -def _load_shared_library(lib_base_name: str): - # Construct the paths to the possible shared library names - _base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" - # Searching for the library in the current directory under the name "libllama" (default name - # for llamacpp) and "llama" (default name for this repo) - _lib_paths: List[pathlib.Path] = [] - # Determine the file extension based on the platform - if sys.platform.startswith("linux"): - _lib_paths += [ - _base_path / f"lib{lib_base_name}.so", - ] - elif sys.platform == "darwin": - _lib_paths += [ - _base_path / f"lib{lib_base_name}.so", - _base_path / f"lib{lib_base_name}.dylib", - ] - elif sys.platform == "win32": - _lib_paths += [ - _base_path / f"{lib_base_name}.dll", - _base_path / f"lib{lib_base_name}.dll", - ] - else: - raise RuntimeError("Unsupported platform") - - if "LLAVA_CPP_LIB" in os.environ: - lib_base_name = os.environ["LLAVA_CPP_LIB"] - _lib = pathlib.Path(lib_base_name) - _base_path = _lib.parent.resolve() - _lib_paths = [_lib.resolve()] - - cdll_args = dict() # type: ignore - # Add the library directory to the DLL search path on Windows (if needed) - if sys.platform == "win32" and sys.version_info >= (3, 8): - os.add_dll_directory(str(_base_path)) - if "CUDA_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib")) - cdll_args["winmode"] = ctypes.RTLD_GLOBAL - - # Try to load the shared library, handling potential errors - for _lib_path in _lib_paths: - if _lib_path.exists(): - try: - return ctypes.CDLL(str(_lib_path), **cdll_args) # type: ignore - except Exception as e: - raise RuntimeError(f"Failed to load shared library '{_lib_path}': {e}") - - raise FileNotFoundError( - f"Shared library with base name '{lib_base_name}' not found" +if TYPE_CHECKING: + from llama_cpp._ctypes_extensions import ( + CtypesArray, ) # Specify the base name of the shared library to load _libllava_base_name = "llava" +_libllava_override_path = os.environ.get("LLAVA_CPP_LIB") +_libllava_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _libllava_override_path is None else pathlib.Path() # Load the library -_libllava = _load_shared_library(_libllava_base_name) - -# ctypes helper - -if TYPE_CHECKING: - CtypesCData = TypeVar("CtypesCData", bound=ctypes._CData) # type: ignore - - CtypesArray: TypeAlias = ctypes.Array[CtypesCData] # type: ignore - - CtypesPointer: TypeAlias = ctypes._Pointer[CtypesCData] # type: ignore - - CtypesVoidPointer: TypeAlias = ctypes.c_void_p - - class CtypesRef(Generic[CtypesCData]): - pass - - CtypesPointerOrRef: TypeAlias = Union[ - CtypesPointer[CtypesCData], CtypesRef[CtypesCData] - ] - - CtypesFuncPointer: TypeAlias = ctypes._FuncPointer # type: ignore - -F = TypeVar("F", bound=Callable[..., Any]) - - -def ctypes_function_for_shared_library(lib: ctypes.CDLL): - def ctypes_function( - name: str, argtypes: List[Any], restype: Any, enabled: bool = True - ): - def decorator(f: F) -> F: - if enabled: - func = getattr(lib, name) - func.argtypes = argtypes - func.restype = restype - functools.wraps(f)(func) - return func - else: - return f - - return decorator - - return ctypes_function - +_libllava = load_shared_library(_libllava_base_name, _libllava_base_path) ctypes_function = ctypes_function_for_shared_library(_libllava) @@ -247,3 +155,4 @@ def clip_model_load( @ctypes_function("clip_free", [clip_ctx_p_ctypes], None) def clip_free(ctx: clip_ctx_p, /): ... + From 57e70bb5693fb0330b11c50f707adb88a6be7b85 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 29 Sep 2024 15:23:24 -0400 Subject: [PATCH 341/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 16 ++++++++++++---- vendor/llama.cpp | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index de427970c2..97c9691363 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -220,6 +220,7 @@ # LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, # LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, # LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, +# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -247,6 +248,7 @@ LLAMA_VOCAB_PRE_TYPE_BLOOM = 23 LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 +LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 # // note: these values should be synchronized with ggml_rope @@ -404,12 +406,14 @@ # LLAMA_POOLING_TYPE_MEAN = 1, # LLAMA_POOLING_TYPE_CLS = 2, # LLAMA_POOLING_TYPE_LAST = 3, +# LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph # }; LLAMA_POOLING_TYPE_UNSPECIFIED = -1 LLAMA_POOLING_TYPE_NONE = 0 LLAMA_POOLING_TYPE_MEAN = 1 LLAMA_POOLING_TYPE_CLS = 2 LLAMA_POOLING_TYPE_LAST = 3 +LLAMA_POOLING_TYPE_RANK = 4 # enum llama_attention_type { # LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1, @@ -420,10 +424,11 @@ LLAMA_ATTENTION_TYPE_CAUSAL = 0 LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1 + # enum llama_split_mode { -# LLAMA_SPLIT_MODE_NONE = 0, // single GPU -# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs -# LLAMA_SPLIT_MODE_ROW = 2, // split rows across GPUs +# LLAMA_SPLIT_MODE_NONE = 0, // single GPU +# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs +# LLAMA_SPLIT_MODE_ROW = 2, // split rows across GPUs # }; LLAMA_SPLIT_MODE_NONE = 0 LLAMA_SPLIT_MODE_LAYER = 1 @@ -2520,7 +2525,8 @@ def llama_get_embeddings_ith( # // Get the embeddings for a sequence id # // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE -# // shape: [n_embd] (1-dimensional) +# // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[1] with the rank of the sequence +# // otherwise: float[n_embd] (1-dimensional) # LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); @ctypes_function( "llama_get_embeddings_seq", @@ -2672,6 +2678,8 @@ def llama_token_eot(model: llama_model_p, /) -> int: # // # // Tokenization # // +# // The API is thread-safe. +# // # /// @details Convert the provided text into tokens. diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 95bc82fbc0..c919d5db39 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 95bc82fbc0df6d48cf66c857a4dda3d044f45ca2 +Subproject commit c919d5db39c8a7fcb64737f008e4b105ee0acd20 From 7c4aead82d349469bbbe7d8c0f4678825873c039 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 29 Sep 2024 15:26:54 -0400 Subject: [PATCH 342/561] chore: Bump version --- CHANGELOG.md | 7 +++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14f68a4563..c68e96b87b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.1] + +- feat: Update llama.cpp to ggerganov/llama.cpp@c919d5db39c8a7fcb64737f008e4b105ee0acd20 +- feat: Expose libggml in internal APIs by @abetlen in #1761 +- fix: Fix speculative decoding by @abetlen in 9992c5084a3df2f533e265d10f81d4269b97a1e6 and e975dabf74b3ad85689c9a07719cbb181313139b +- misc: Rename all_text to remaining_text by @xu-song in #1658 + ## [0.3.0] - feat: Update llama.cpp to ggerganov/llama.cpp@ea9c32be71b91b42ecc538bd902e93cbb5fb36cb diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 70fde20412..c11b879d6d 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.0" +__version__ = "0.3.1" From 7403e002b8e033c0a34e93fba2b311e2118487fe Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 22 Oct 2024 09:33:48 -0400 Subject: [PATCH 343/561] feat: Update llama.cpp --- CMakeLists.txt | 18 ++++- llama_cpp/llama_cpp.py | 152 ++++++++++++++++++++++++++++++++--------- vendor/llama.cpp | 2 +- 3 files changed, 139 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6b35ed6c3..9af5b2180e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,9 @@ if (LLAMA_BUILD) set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) set(CMAKE_SKIP_RPATH FALSE) + # Enable building of the common library + set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build llama.cpp common library" FORCE) + # Building llama if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") # Need to disable these llama.cpp flags on Apple x86_64, @@ -106,7 +109,7 @@ if (LLAMA_BUILD) # Building llava add_subdirectory(vendor/llama.cpp/examples/llava) set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") - # Set CUDA_ARCHITECTURES to OFF on windows + # Set CUDA_ARCHITECTURES to OFF on Windows if (WIN32) set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) endif() @@ -121,5 +124,18 @@ if (LLAMA_BUILD) DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib ) endif() + + # Fix for llava build: Add include directory for llama.h + # Move these commands after the add_subdirectory call + target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) + + if (BUILD_SHARED_LIBS) + target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) + endif() + + target_include_directories(llama-llava-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + target_include_directories(llama-minicpmv-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) endif() endif() diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 97c9691363..66feed8967 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -464,6 +464,8 @@ class llama_token_data(ctypes.Structure): # typedef struct llama_token_data_array { +# // TODO: consider SoA +# // NOTE: this pointer can be modified by the samplers # llama_token_data * data; # size_t size; # int64_t selected; // this is the index in the data array (i.e. not the token id) @@ -507,8 +509,11 @@ class llama_token_data_array(ctypes.Structure): # // - token : the token ids of the input (used when embd is NULL) # // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) # // - pos : the positions of the respective token in the sequence +# // (if set to NULL, the token position will be tracked automatically by llama_decode) # // - seq_id : the sequence to which the respective token belongs +# // (if set to NULL, the sequence ID will be assumed to be 0) # // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output +# // (if set to NULL, only the logits for last token will be returned) # // # typedef struct llama_batch { # int32_t n_tokens; @@ -519,16 +524,6 @@ class llama_token_data_array(ctypes.Structure): # int32_t * n_seq_id; # llama_seq_id ** seq_id; # int8_t * logits; // TODO: rename this to "output" - - -# // NOTE: helpers for smooth API transition - can be deprecated in the future -# // for future-proof code, use the above fields instead and ignore everything below -# // -# // pos[i] = all_pos_0 + i*all_pos_1 -# // -# llama_pos all_pos_0; // used if pos == NULL -# llama_pos all_pos_1; // used if pos == NULL -# llama_seq_id all_seq_id; // used if seq_id == NULL # } llama_batch; class llama_batch(ctypes.Structure): """Input data for llama_decode @@ -563,9 +558,6 @@ class llama_batch(ctypes.Structure): ("n_seq_id", ctypes.POINTER(ctypes.c_int32)), ("seq_id", ctypes.POINTER(ctypes.POINTER(llama_seq_id))), ("logits", ctypes.POINTER(ctypes.c_int8)), - ("all_pos_0", llama_pos), - ("all_pos_1", llama_pos), - ("all_seq_id", llama_seq_id), ] @@ -1170,6 +1162,12 @@ def llama_supports_gpu_offload() -> bool: ... +# LLAMA_API bool llama_supports_rpc (void); +@ctypes_function("llama_supports_rpc", [], ctypes.c_bool) +def llama_supports_rpc() -> bool: + ... + + # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) def llama_n_ctx(ctx: llama_context_p, /) -> int: @@ -2255,30 +2253,26 @@ def llama_state_seq_load_file( # // -# // Return batch for single sequence of tokens starting at pos_0 +# // Return batch for single sequence of tokens +# // The sequence ID will be fixed to 0 +# // The position of the tokens will be tracked automatically by llama_decode # // # // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it # // # LLAMA_API struct llama_batch llama_batch_get_one( # llama_token * tokens, -# int32_t n_tokens, -# llama_pos pos_0, -# llama_seq_id seq_id); +# int32_t n_tokens); @ctypes_function( "llama_batch_get_one", [ llama_token_p, - ctypes.c_int, - llama_pos, - llama_seq_id, + ctypes.c_int32, ], llama_batch, ) def llama_batch_get_one( tokens: CtypesArray[llama_token], n_tokens: Union[ctypes.c_int, int], - pos_0: Union[llama_pos, int], - seq_id: llama_seq_id, /, ) -> llama_batch: """Return batch for single sequence of tokens starting at pos_0 @@ -2616,6 +2610,13 @@ def llama_token_eos(model: llama_model_p, /) -> int: ... +# LLAMA_API llama_token llama_token_eot(const struct llama_model * model); // end-of-turn +@ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) +def llama_token_eot(model: llama_model_p, /) -> int: + """end-of-turn""" + ... + + # LLAMA_API llama_token llama_token_cls(const struct llama_model * model); // classification @ctypes_function("llama_token_cls", [llama_model_p_ctypes], llama_token) def llama_token_cls(model: llama_model_p, /) -> int: @@ -2650,30 +2651,54 @@ def llama_add_eos_token(model: llama_model_p, /) -> bool: # // Codellama infill tokens -# LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix +# DEPRECATED(LLAMA_API llama_token llama_token_prefix(const struct llama_model * model), "use llama_token_fim_pre instead"); @ctypes_function("llama_token_prefix", [llama_model_p_ctypes], llama_token) def llama_token_prefix(model: llama_model_p) -> int: """codellama infill tokens""" ... -# LLAMA_API llama_token llama_token_middle(const struct llama_model * model); // Beginning of infill middle +# DEPRECATED(LLAMA_API llama_token llama_token_middle(const struct llama_model * model), "use llama_token_fim_mid instead"); @ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) def llama_token_middle(model: llama_model_p, /) -> int: ... -# LLAMA_API llama_token llama_token_suffix(const struct llama_model * model); // Beginning of infill suffix +# DEPRECATED(LLAMA_API llama_token llama_token_suffix(const struct llama_model * model), "use llama_token_fim_suf instead"); @ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) def llama_token_suffix(model: llama_model_p, /) -> int: ... -# LLAMA_API llama_token llama_token_eot (const struct llama_model * model); // End of infill middle -@ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_token_fim_pre(const struct llama_model * model); +@ctypes_function("llama_token_fim_pre", [llama_model_p_ctypes], llama_token) +def llama_token_fim_pre(model: llama_model_p, /) -> int: + ... + +# LLAMA_API llama_token llama_token_fim_suf(const struct llama_model * model); +@ctypes_function("llama_token_fim_suf", [llama_model_p_ctypes], llama_token) +def llama_token_fim_suf(model: llama_model_p, /) -> int: ... +# LLAMA_API llama_token llama_token_fim_mid(const struct llama_model * model); +@ctypes_function("llama_token_fim_mid", [llama_model_p_ctypes], llama_token) +def llama_token_fim_mid(model: llama_model_p, /) -> int: + ... + +# LLAMA_API llama_token llama_token_fim_pad(const struct llama_model * model); +@ctypes_function("llama_token_fim_pad", [llama_model_p_ctypes], llama_token) +def llama_token_fim_pad(model: llama_model_p, /) -> int: + ... + +# LLAMA_API llama_token llama_token_fim_rep(const struct llama_model * model); +@ctypes_function("llama_token_fim_rep", [llama_model_p_ctypes], llama_token) +def llama_token_fim_rep(model: llama_model_p, /) -> int: + ... + +# LLAMA_API llama_token llama_token_fim_sep(const struct llama_model * model); +@ctypes_function("llama_token_fim_sep", [llama_model_p_ctypes], llama_token) +def llama_token_fim_sep(model: llama_model_p, /) -> int: + ... # // # // Tokenization @@ -2786,6 +2811,23 @@ def llama_token_to_piece( ... +# # // check if token0 is contained as a prefix in token1 +# # LLAMA_API bool llama_token_is_prefix( +# # const struct llama_model * model, +# # llama_token token0, +# # llama_token token1); +# @ctypes_function( +# "llama_token_is_prefix", +# [llama_model_p_ctypes, llama_token, llama_token], +# ctypes.c_bool, +# ) +# def llama_token_is_prefix( +# model: llama_model_p, token0: Union[llama_token, int], token1: Union[llama_token, int], / +# ) -> bool: +# """Check if token0 is contained as a prefix in token1""" +# ... + + # /// @details Convert the provided tokens into text (inverse of llama_tokenize()). # /// @param text The char pointer must be large enough to hold the resulting text. # /// @return Returns the number of chars/bytes on success, no more than text_len_max. @@ -3099,20 +3141,22 @@ def llama_sampler_chain_remove( # // available samplers: # -# LLAMA_API struct llama_sampler * llama_sampler_init_greedy (void); +# LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void); @ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes) def llama_sampler_init_greedy() -> llama_sampler_p: ... -# LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed); +# LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed); @ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes) def llama_sampler_init_dist(seed: int) -> llama_sampler_p: ... # /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. -# LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void); +# /// NOTE: Avoid using on the full vocabulary as the sorting can become slow. For example, apply top-k or top-p sampling first. +# DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void), +# "will be removed in the future (see https://github.com/ggerganov/llama.cpp/pull/9896#discussion_r1800920915)"); @ctypes_function("llama_sampler_init_softmax", [], llama_sampler_p_ctypes) def llama_sampler_init_softmax() -> llama_sampler_p: ... @@ -3188,6 +3232,19 @@ def llama_sampler_init_temp_ext( ... +# /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 +# LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed); +@ctypes_function( + "llama_sampler_init_xtc", + [ctypes.c_float, ctypes.c_float, ctypes.c_size_t, ctypes.c_uint32], + llama_sampler_p_ctypes, +) +def llama_sampler_init_xtc( + p: float, t: float, min_keep: int, seed: int, / +) -> llama_sampler_p: + ... + + # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. # /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. # /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. @@ -3301,6 +3358,39 @@ def llama_sampler_init_logit_bias( ... +# // this sampler is meant to be used for fill-in-the-middle infilling +# // it's supposed to be used after top_k + top_p sampling +# // +# // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG +# // 2. combine probs of tokens that have the same prefix +# // +# // example: +# // +# // - before: +# // "hel": 0.5 +# // "hell": 0.2 +# // "hello": 0.1 +# // "dummy": 0.1 +# // +# // - after: +# // "hel": 0.8 +# // "dummy": 0.1 +# // +# // 3. discard non-EOG tokens with low prob +# // 4. if no tokens are left -> pick EOT +# // +# LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_model * model); +@ctypes_function( + "llama_sampler_init_infill", + [llama_model_p_ctypes], + llama_sampler_p_ctypes, +) +def llama_sampler_init_infill(model: llama_model_p, /) -> llama_sampler_p: + """This sampler is meant to be used for fill-in-the-middle infilling. + """ + ... + + # // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise # LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl); @ctypes_function( diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c919d5db39..c421ac072d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c919d5db39c8a7fcb64737f008e4b105ee0acd20 +Subproject commit c421ac072d46172ab18924e1e8be53680b54ed3b From e712cffc0c22d5406c6fe02a1769992f2181f229 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 31 Oct 2024 13:50:18 -0400 Subject: [PATCH 344/561] feat: Update llama.cpp --- llama_cpp/_internals.py | 14 ------------- llama_cpp/llama.py | 1 - llama_cpp/llama_cpp.py | 46 +++++++++++++++++++++++++++++++---------- vendor/llama.cpp | 2 +- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 0aff348442..994d5f1497 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -362,13 +362,6 @@ def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: i self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep ) - def sample_tail_free( - self, candidates: "_LlamaTokenDataArray", z: float, min_keep: int - ): - llama_cpp.llama_sample_tail_free( - self.ctx, llama_cpp.byref(candidates.candidates), z, min_keep - ) - def sample_typical( self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int ): @@ -685,9 +678,6 @@ def sample( ctx_main.sample_top_k( token_data_array, self.params.top_k, min_keep=min_keep ) - ctx_main.sample_tail_free( - token_data_array, self.params.tfs_z, min_keep=min_keep - ) ctx_main.sample_typical( token_data_array, self.params.typical_p, min_keep=min_keep ) @@ -776,10 +766,6 @@ def add_min_p(self, p: float, min_keep: int): sampler = llama_cpp.llama_sampler_init_min_p(p, min_keep) self._add_sampler(sampler) - def add_tail_free(self, z: float, min_keep: int): - sampler = llama_cpp.llama_sampler_init_tail_free(z, min_keep) - self._add_sampler(sampler) - def add_typical(self, p: float, min_keep: int): sampler = llama_cpp.llama_sampler_init_typical(p, min_keep) self._add_sampler(sampler) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index babb30cf09..d15a88b00e 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -745,7 +745,6 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): n_probs = 0 min_keep = max(1, n_probs) sampler.add_top_k(top_k) - sampler.add_tail_free(tfs_z, min_keep) sampler.add_typical(typical_p, min_keep) sampler.add_top_p(top_p, min_keep) sampler.add_min_p(min_p, min_keep) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 66feed8967..457c6dddb4 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -3191,17 +3191,6 @@ def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ... -# /// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/. -# LLAMA_API struct llama_sampler * llama_sampler_init_tail_free (float z, size_t min_keep); -@ctypes_function( - "llama_sampler_init_tail_free", - [ctypes.c_float, ctypes.c_size_t], - llama_sampler_p_ctypes, -) -def llama_sampler_init_tail_free(z: float, min_keep: int) -> llama_sampler_p: - ... - - # /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. # LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep); @ctypes_function( @@ -3343,6 +3332,41 @@ def llama_sampler_init_penalties( ... +# /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 +# LLAMA_API struct llama_sampler * llama_sampler_init_dry( +# const struct llama_model * model, +# float dry_multiplier, +# float dry_base, +# int32_t dry_allowed_length, +# int32_t dry_penalty_last_n, +# const char ** seq_breakers, +# size_t num_breakers); +@ctypes_function( + "llama_sampler_init_dry", + [ + llama_model_p_ctypes, + ctypes.c_float, + ctypes.c_float, + ctypes.c_int32, + ctypes.c_int32, + ctypes.POINTER(ctypes.c_char_p), + ctypes.c_size_t, + ], + llama_sampler_p_ctypes, +) +def llama_sampler_init_dry( + model: llama_model_p, + dry_multiplier: float, + dry_base: float, + dry_allowed_length: int, + dry_penalty_last_n: int, + seq_breakers: CtypesArray[bytes], + num_breakers: int, + /, +) -> llama_sampler_p: + ... + + # LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( # int32_t n_vocab, # int32_t n_logit_bias, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c421ac072d..0a683e8088 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c421ac072d46172ab18924e1e8be53680b54ed3b +Subproject commit 0a683e8088d849626e7471f9e2ed381f7dbdf2e9 From cafa33e1a8646258f2e44dd95782f4903d6944c0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Nov 2024 18:00:30 -0500 Subject: [PATCH 345/561] feat: Update llama.cpp --- CMakeLists.txt | 16 +++++++++++++++- vendor/llama.cpp | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9af5b2180e..7d31ffd6c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,7 +75,21 @@ if (LLAMA_BUILD) add_subdirectory(vendor/llama.cpp) llama_cpp_python_install_target(llama) llama_cpp_python_install_target(ggml) - + llama_cpp_python_install_target(ggml-cpu) + llama_cpp_python_install_target(ggml-base) + + if (GGML_METAL) + llama_cpp_python_install_target(ggml-metal) + endif() + + if (GGML_CUDA) + llama_cpp_python_install_target(ggml-cuda) + endif() + + if (GGML_VULKAN) + llama_cpp_python_install_target(ggml-vulkan) + endif() + # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563 if (WIN32) install( diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 0a683e8088..74d73dc85c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 0a683e8088d849626e7471f9e2ed381f7dbdf2e9 +Subproject commit 74d73dc85cc2057446bf63cc37ff649ae7cebd80 From d1cb50b49fafad171bca623f346cbc5438376fb3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Nov 2024 19:26:16 -0500 Subject: [PATCH 346/561] Add missing ggml dependency --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d31ffd6c0..f8c55d339c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,7 @@ if (LLAMA_BUILD) llama_cpp_python_install_target(ggml) llama_cpp_python_install_target(ggml-cpu) llama_cpp_python_install_target(ggml-base) + llama_cpp_python_install_target(ggml-amx) if (GGML_METAL) llama_cpp_python_install_target(ggml-metal) From 2796f4e62f810780d89a3485a0a122a9e014e8ae Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Nov 2024 19:31:12 -0500 Subject: [PATCH 347/561] Add all missing ggml dependencies --- CMakeLists.txt | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f8c55d339c..c9f3c2b361 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,10 @@ option(LLAMA_BUILD "Build llama.cpp shared library and install alongside python option(LLAVA_BUILD "Build llava shared library and install alongside python package" ON) function(llama_cpp_python_install_target target) + if(NOT TARGET ${target}) + return() + endif() + install( TARGETS ${target} LIBRARY DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib @@ -75,21 +79,21 @@ if (LLAMA_BUILD) add_subdirectory(vendor/llama.cpp) llama_cpp_python_install_target(llama) llama_cpp_python_install_target(ggml) - llama_cpp_python_install_target(ggml-cpu) - llama_cpp_python_install_target(ggml-base) - llama_cpp_python_install_target(ggml-amx) - - if (GGML_METAL) - llama_cpp_python_install_target(ggml-metal) - endif() - if (GGML_CUDA) - llama_cpp_python_install_target(ggml-cuda) - endif() + llama_cpp_python_install_target(ggml-base) - if (GGML_VULKAN) - llama_cpp_python_install_target(ggml-vulkan) - endif() + llama_cpp_python_install_target(ggml-amx) + llama_cpp_python_install_target(ggml-blas) + llama_cpp_python_install_target(ggml-can) + llama_cpp_python_install_target(ggml-cpu) + llama_cpp_python_install_target(ggml-cuda) + llama_cpp_python_install_target(ggml-hip) + llama_cpp_python_install_target(ggml-kompute) + llama_cpp_python_install_target(ggml-metal) + llama_cpp_python_install_target(ggml-musa) + llama_cpp_python_install_target(ggml-rpc) + llama_cpp_python_install_target(ggml-sycl) + llama_cpp_python_install_target(ggml-vulkan) # Workaround for Windows + CUDA https://github.com/abetlen/llama-cpp-python/issues/563 if (WIN32) From 7ecdd944624cbd49e4af0a5ce1aa402607d58dcc Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 15 Nov 2024 19:55:09 -0500 Subject: [PATCH 348/561] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c68e96b87b..8a194fecef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.2] + +- feat: Update llama.cpp to ggerganov/llama.cpp@74d73dc85cc2057446bf63cc37ff649ae7cebd80 + ## [0.3.1] - feat: Update llama.cpp to ggerganov/llama.cpp@c919d5db39c8a7fcb64737f008e4b105ee0acd20 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index c11b879d6d..cda38a841f 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.1" +__version__ = "0.3.2" From f3fb90b114835cc50c4816787d56bac2fe1180c3 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 28 Nov 2024 18:27:55 -0500 Subject: [PATCH 349/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 22 ++++++++++++---------- vendor/llama.cpp | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 457c6dddb4..bb2ba1993c 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -615,6 +615,9 @@ class llama_model_kv_override(ctypes.Structure): # struct llama_model_params { +# // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) +# ggml_backend_dev_t * devices; + # int32_t n_gpu_layers; // number of layers to store in VRAM # enum llama_split_mode split_mode; // how to split the model across multiple GPUs @@ -680,6 +683,7 @@ class llama_model_params(ctypes.Structure): check_tensors: bool _fields_ = [ + ("devices", ctypes.c_void_p), # NOTE: unnused ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), @@ -1898,6 +1902,14 @@ def llama_kv_cache_update(ctx: llama_context_p, /): ... +# // Check if the context supports KV cache shifting +# LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx); +@ctypes_function("llama_kv_cache_can_shift", [llama_context_p_ctypes], ctypes.c_bool) +def llama_kv_cache_can_shift(ctx: llama_context_p, /) -> bool: + """Check if the context supports KV cache shifting""" + ... + + # // # // State / sessions # // @@ -3621,13 +3633,3 @@ def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... -# LLAMA_API void llama_perf_dump_yaml(FILE * stream, const struct llama_context * ctx); -@ctypes_function( - "llama_perf_dump_yaml", - [ctypes.POINTER(ctypes.c_void_p), llama_context_p_ctypes], - None, -) -def llama_perf_dump_yaml( - stream: ctypes.POINTER(ctypes.c_void_p), ctx: llama_context_p, / -): - ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 74d73dc85c..dc22344088 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 74d73dc85cc2057446bf63cc37ff649ae7cebd80 +Subproject commit dc22344088a7ee81a1e4f096459b03a72f24ccdc From 7ba257e5dac58f8796a12260776f27fd989a1182 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 02:33:55 -0500 Subject: [PATCH 350/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 32 ++++++++++++++++++++++++++++++++ vendor/llama.cpp | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index bb2ba1993c..24663146c6 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -221,6 +221,7 @@ # LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, # LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, # LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, +# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -249,6 +250,7 @@ LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 +LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 # // note: these values should be synchronized with ggml_rope @@ -392,12 +394,14 @@ # LLAMA_ROPE_SCALING_TYPE_NONE = 0, # LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, # LLAMA_ROPE_SCALING_TYPE_YARN = 2, +# LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, # LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN, # }; LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1 LLAMA_ROPE_SCALING_TYPE_NONE = 0 LLAMA_ROPE_SCALING_TYPE_LINEAR = 1 LLAMA_ROPE_SCALING_TYPE_YARN = 2 +LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3 LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN # enum llama_pooling_type { @@ -2933,6 +2937,34 @@ def llama_chat_apply_template( ... +# // Get list of built-in chat templates +# LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len); +@ctypes_function( + "llama_chat_builtin_templates", + [ + ctypes.POINTER(ctypes.c_char_p), + ctypes.c_size_t, + ], + ctypes.c_int32, +) +def llama_chat_builtin_templates( + output: CtypesArray[bytes], + len: Union[ctypes.c_size_t, int], + /, +) -> int: + """Get list of built-in chat templates. + + Args: + output: Output buffer to store template names. + len: Length of the output buffer. + + Returns: + Number of templates available. + Returns a negative number on error. + """ + ... + + # // # // Sampling API # // diff --git a/vendor/llama.cpp b/vendor/llama.cpp index dc22344088..7736837d62 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit dc22344088a7ee81a1e4f096459b03a72f24ccdc +Subproject commit 7736837d62efed1dbebfe579472fca041eda12d6 From 9d06e36c00aea381dd995abc800ce9727da13a82 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 02:49:20 -0500 Subject: [PATCH 351/561] fix(ci): Explicitly install arm64 python version --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f619cf1f4..01b2f4c405 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -126,6 +126,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + architecture: "arm64" cache: 'pip' - name: Restore model cache @@ -166,6 +167,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.9" + architecture: "arm64" - name: Restore model cache uses: actions/cache@v4 From fb0b8fefd41dc8c467b0e353638f7345254274b5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 02:53:06 -0500 Subject: [PATCH 352/561] fix(ci): Explicitly set cmake osx architecture --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c9f3c2b361..584e54d39e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,8 @@ if (LLAMA_BUILD) set(GGML_AVX2 "Off" CACHE BOOL "ggml: enable AVX2" FORCE) set(GGML_FMA "Off" CACHE BOOL "gml: enable FMA" FORCE) set(GGML_F16C "Off" CACHE BOOL "gml: enable F16C" FORCE) + + set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING "Build architecture for OS X" FORCE) endif() if (APPLE) From 72ed7b8855a168a454026b98c7f86a452524a7d1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 02:55:57 -0500 Subject: [PATCH 353/561] fix(ci): Explicitly test on arm64 macos runner --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 01b2f4c405..445554e6ba 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -113,7 +113,7 @@ jobs: build-macos: needs: download-model - runs-on: macos-latest + runs-on: macos-14-arm64 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -157,7 +157,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-latest + runs-on: macos-14-arm64 steps: - uses: actions/checkout@v4 with: From 8988aaf7f64a4ffbe30fb1b94d786848ea1efd3c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:02:04 -0500 Subject: [PATCH 354/561] fix(ci): Use macos-14 runner --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 445554e6ba..f5a08f6cbe 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -113,7 +113,7 @@ jobs: build-macos: needs: download-model - runs-on: macos-14-arm64 + runs-on: macos-14 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -157,7 +157,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-14-arm64 + runs-on: macos-14 steps: - uses: actions/checkout@v4 with: From f11a781d86c914ab12dc620edf64c049c6fb0ee7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:04:48 -0500 Subject: [PATCH 355/561] fix(ci): Use macos-13 runner --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f5a08f6cbe..1631d63de3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -113,7 +113,7 @@ jobs: build-macos: needs: download-model - runs-on: macos-14 + runs-on: macos-13 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -157,7 +157,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-14 + runs-on: macos-13 steps: - uses: actions/checkout@v4 with: From 9a09fc78487b08423f4e2c9f02b7bc308dc0e2e7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:19:36 -0500 Subject: [PATCH 356/561] fix(ci): Debug print python system architecture --- .github/workflows/test.yaml | 49 ++++++++++++------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1631d63de3..ee8680a9bb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -51,19 +51,11 @@ jobs: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose shell: bash - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - shell: cmd - name: Test with pytest run: | python -m pytest @@ -90,22 +82,13 @@ jobs: with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - - - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' - run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - shell: bash - name: Install dependencies (Windows) - if: runner.os == 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose - shell: cmd + shell: cmd - name: Test with pytest run: | @@ -129,6 +112,12 @@ jobs: architecture: "arm64" cache: 'pip' + - name: System Info + run: | + uname -a + sysctl -n machdep.cpu.brand_string + python3 -c "import platform; print(platform.machine(), platform.architecture())" + - name: Restore model cache uses: actions/cache@v4 with: @@ -136,21 +125,13 @@ jobs: key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv python -m uv pip install -e .[all] --verbose + CMAKE_ARGS="-DLLAMA_METAL=off" python -m uv pip install .[all] --verbose shell: bash - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - shell: cmd - - name: Test with pytest run: | python -m pytest @@ -169,25 +150,25 @@ jobs: python-version: "3.9" architecture: "arm64" + - name: System Info + run: | + uname -a + sysctl -n machdep.cpu.brand_string + python3 -c "import platform; print(platform.machine(), platform.architecture())" + - name: Restore model cache uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} - - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' + - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install uv CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose shell: bash - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: | - python -m pip install --upgrade pip - CMAKE_ARGS="-DGGML_METAL=on" python -m pip install .[all] --verbose - name: Test with pytest run: | python -m pytest From a412ba5539be6a5499acfcb0a2c234665fcbda11 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:38:12 -0500 Subject: [PATCH 357/561] fix(ci): Update config --- .github/workflows/test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ee8680a9bb..f8b7893f8f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -109,7 +109,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - architecture: "arm64" cache: 'pip' - name: System Info @@ -138,7 +137,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-13 + runs-on: macos-14 steps: - uses: actions/checkout@v4 with: From df05096fc56347efe14ee550f065a1b15933c322 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:42:33 -0500 Subject: [PATCH 358/561] fix(ci): Install with regular pip --- .github/workflows/test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f8b7893f8f..9332e0830b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -164,8 +164,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install uv - CMAKE_ARGS="-DLLAMA_METAL=on" python -m uv pip install .[all] --verbose + CMAKE_ARGS="-DLLAMA_METAL=on" python -m pip install .[all] --verbose shell: bash - name: Test with pytest From 1cd3f2cc6aceaa3167c943eb33b4076ae7a9ba53 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:52:54 -0500 Subject: [PATCH 359/561] fix(ci): gg --- .github/workflows/test.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9332e0830b..a4d56110a5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -110,6 +110,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' + architecture: "arm64" - name: System Info run: | @@ -137,7 +138,7 @@ jobs: build-macos-metal: needs: download-model - runs-on: macos-14 + runs-on: macos-13 steps: - uses: actions/checkout@v4 with: @@ -163,10 +164,10 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - CMAKE_ARGS="-DLLAMA_METAL=on" python -m pip install .[all] --verbose + python3 -m pip install --upgrade pip + CMAKE_ARGS="-DLLAMA_METAL=on" python3 -m pip install .[all] --verbose shell: bash - name: Test with pytest run: | - python -m pytest + python3 -m pytest From b34f20046b9fc058ed8581291a724c6723b06fe4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 03:59:51 -0500 Subject: [PATCH 360/561] fix(ci): Use python3 --- .github/workflows/test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a4d56110a5..7f47749596 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -126,15 +126,15 @@ jobs: - name: Install dependencies (Linux/MacOS) run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - CMAKE_ARGS="-DLLAMA_METAL=off" python -m uv pip install .[all] --verbose + python3 -m pip install --upgrade pip + python3 -m pip install uv + python3 -m uv pip install -e .[all] --verbose + CMAKE_ARGS="-DLLAMA_METAL=off" python3 -m uv pip install .[all] --verbose shell: bash - name: Test with pytest run: | - python -m pytest + python3 -m pytest build-macos-metal: needs: download-model From d8cc231fd00b67528e7e4563c2b4c514acbcd8f9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 04:06:07 -0500 Subject: [PATCH 361/561] fix(ci): Use default architecture chosen by action --- .github/workflows/test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7f47749596..95f6e5a272 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -110,7 +110,6 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - architecture: "arm64" - name: System Info run: | @@ -148,7 +147,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.9" - architecture: "arm64" - name: System Info run: | From d5d50990d016333982ff350c9c151efe5732d76b Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 04:14:59 -0500 Subject: [PATCH 362/561] fix(ci): Update CMakeLists.txt for macos --- CMakeLists.txt | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 584e54d39e..64a0304a16 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,20 +62,35 @@ if (LLAMA_BUILD) # Enable building of the common library set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build llama.cpp common library" FORCE) - # Building llama - if (APPLE AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64") - # Need to disable these llama.cpp flags on Apple x86_64, - # otherwise users may encounter invalid instruction errors - set(GGML_AVX "Off" CACHE BOOL "ggml: enable AVX" FORCE) - set(GGML_AVX2 "Off" CACHE BOOL "ggml: enable AVX2" FORCE) - set(GGML_FMA "Off" CACHE BOOL "gml: enable FMA" FORCE) - set(GGML_F16C "Off" CACHE BOOL "gml: enable F16C" FORCE) - - set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING "Build architecture for OS X" FORCE) - endif() - + # Architecture detection and settings for Apple platforms if (APPLE) - set(GGML_METAL_EMBED_LIBRARY "On" CACHE BOOL "llama: embed metal library" FORCE) + # Get the target architecture + execute_process( + COMMAND uname -m + OUTPUT_VARIABLE HOST_ARCH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + # If CMAKE_OSX_ARCHITECTURES is not set, use the host architecture + if(NOT CMAKE_OSX_ARCHITECTURES) + set(CMAKE_OSX_ARCHITECTURES ${HOST_ARCH} CACHE STRING "Build architecture for macOS" FORCE) + endif() + + message(STATUS "Host architecture: ${HOST_ARCH}") + message(STATUS "Target architecture: ${CMAKE_OSX_ARCHITECTURES}") + + # Configure based on target architecture + if(CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64") + # Intel Mac settings + set(GGML_AVX "OFF" CACHE BOOL "ggml: enable AVX" FORCE) + set(GGML_AVX2 "OFF" CACHE BOOL "ggml: enable AVX2" FORCE) + set(GGML_FMA "OFF" CACHE BOOL "ggml: enable FMA" FORCE) + set(GGML_F16C "OFF" CACHE BOOL "ggml: enable F16C" FORCE) + endif() + + # Metal settings (enable for both architectures) + set(GGML_METAL "ON" CACHE BOOL "ggml: enable Metal" FORCE) + set(GGML_METAL_EMBED_LIBRARY "ON" CACHE BOOL "ggml: embed metal library" FORCE) endif() add_subdirectory(vendor/llama.cpp) @@ -130,7 +145,7 @@ if (LLAMA_BUILD) # Building llava add_subdirectory(vendor/llama.cpp/examples/llava) set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") - # Set CUDA_ARCHITECTURES to OFF on Windows + if (WIN32) set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) endif() From 4f17ae5af073f3741be7235f86e2b183db62a260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=CE=9BBL=C3=98=20=E1=84=83=CE=9E?= Date: Fri, 6 Dec 2024 04:31:01 -0500 Subject: [PATCH 363/561] fix(ci): Remove cuda version 12.5.0 incompatibility with VS (#1838) Co-authored-by: Andrei --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 647b171e84..bde6268285 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -22,7 +22,7 @@ jobs: $matrix = @{ 'os' = @('ubuntu-latest', 'windows-2019') 'pyver' = @("3.9", "3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.0") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") 'releasetag' = @("basic") } From 991d9cde1b81dfdda4d3117b508c16c6fe7dbdca Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 04:32:19 -0500 Subject: [PATCH 364/561] fix(ci): Remove CUDA 12.5 from index --- .github/workflows/generate-index-from-release.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 8d80e7e47f..51961088d7 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -44,7 +44,6 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' - ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu125$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 From 27953036b1bb1da9e4b71b875f66ba8f1aefbf6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 04:32:37 -0500 Subject: [PATCH 365/561] chore(deps): bump pypa/cibuildwheel from 2.21.1 to 2.22.0 (#1844) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.21.1 to 2.22.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.21.1...v2.22.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 4 ++-- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index bfa97decdd..1d549ed68e 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -42,7 +42,7 @@ jobs: shell: cmd - name: Build wheels - uses: pypa/cibuildwheel@v2.21.1 + uses: pypa/cibuildwheel@v2.22.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" @@ -69,7 +69,7 @@ jobs: platforms: linux/arm64 - name: Build wheels - uses: pypa/cibuildwheel@v2.21.1 + uses: pypa/cibuildwheel@v2.22.0 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 8e6fb5cb78..01da3cef51 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -43,7 +43,7 @@ jobs: shell: cmd - name: Build wheels - uses: pypa/cibuildwheel@v2.21.1 + uses: pypa/cibuildwheel@v2.22.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" From 2523472c3eccb9ab9277117cc4ff705212b6888a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 04:36:22 -0500 Subject: [PATCH 366/561] fix: Fix pickling of Llama class by setting seed from _seed member. Closes #1769 --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index d15a88b00e..ce1a266493 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2073,7 +2073,7 @@ def __getstate__(self): use_mlock=self.model_params.use_mlock, kv_overrides=self.kv_overrides, # Context Params - seed=self.context_params.seed, + seed=self._seed, n_ctx=self.context_params.n_ctx, n_batch=self.n_batch, n_ubatch=self.context_params.n_ubatch, From ddac04c78e504196207292aeca4c3adccc655a88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2024 04:39:27 -0500 Subject: [PATCH 367/561] chore(deps): bump conda-incubator/setup-miniconda from 3.0.4 to 3.1.0 (#1821) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.4 to 3.1.0. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.4...v3.1.0) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index bde6268285..3d9c5d9d4d 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -59,7 +59,7 @@ jobs: cache: 'pip' - name: Setup Mamba - uses: conda-incubator/setup-miniconda@v3.0.4 + uses: conda-incubator/setup-miniconda@v3.1.0 with: activate-environment: "build" python-version: ${{ matrix.pyver }} From fa04cdc497e2c72888cccf6268836fe3291c3d42 Mon Sep 17 00:00:00 2001 From: ddh0 Date: Fri, 6 Dec 2024 03:40:47 -0600 Subject: [PATCH 368/561] fix logit-bias type hint (#1802) `Optional[Dict[str, float]]` --> `Optional[Dict[int, float]]` --- llama_cpp/llama.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index ce1a266493..2fd7ff1939 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1141,7 +1141,7 @@ def _create_completion( stopping_criteria: Optional[StoppingCriteriaList] = None, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, + logit_bias: Optional[Dict[int, float]] = None, ) -> Union[ Iterator[CreateCompletionResponse], Iterator[CreateCompletionStreamResponse] ]: @@ -1761,7 +1761,7 @@ def create_completion( stopping_criteria: Optional[StoppingCriteriaList] = None, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, + logit_bias: Optional[Dict[int, float]] = None, ) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]: """Generate text from a prompt. @@ -1858,7 +1858,7 @@ def __call__( stopping_criteria: Optional[StoppingCriteriaList] = None, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, + logit_bias: Optional[Dict[int, float]] = None, ) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]: """Generate text from a prompt. @@ -1951,7 +1951,7 @@ def create_chat_completion( model: Optional[str] = None, logits_processor: Optional[LogitsProcessorList] = None, grammar: Optional[LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, + logit_bias: Optional[Dict[int, float]] = None, logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, ) -> Union[ From 38fbd2968d3e937c9f4157a54372b02fa527233b Mon Sep 17 00:00:00 2001 From: Rich Dougherty Date: Fri, 6 Dec 2024 22:41:52 +1300 Subject: [PATCH 369/561] docs: Remove ref to llama_eval in llama_cpp.py docs (#1819) The llama_eval function was removed in 0.2.14 (0d37ce5) but the docs still reference it. Updated to match the llama.h docs from upstream. --- llama_cpp/llama_cpp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 24663146c6..cd59bb4304 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -772,7 +772,7 @@ class llama_context_params(ctypes.Structure): cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval type_k (int): data type for K cache type_v (int): data type for V cache - logits_all (bool): the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) + logits_all (bool): the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU flash_attn (bool): whether to use flash attention @@ -2469,10 +2469,10 @@ def llama_synchronize(ctx: llama_context_p, /): "llama_get_logits", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float) ) def llama_get_logits(ctx: llama_context_p, /) -> CtypesArray[ctypes.c_float]: - """Token logits obtained from the last call to llama_eval() - The logits for the last token are stored in the last row - Logits for which llama_batch.logits[i] == 0 are undefined - Rows: n_tokens provided with llama_batch + """Token logits obtained from the last call to llama_decode() + The logits for which llama_batch.logits[i] != 0 are stored contiguously + in the order they have appeared in the batch. + Rows: number of tokens for which llama_batch.logits[i] != 0 Cols: n_vocab Returns: From 4192210232ee20ceb028ebd0ee68a6b964f5ffc1 Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Fri, 6 Dec 2024 10:47:38 +0100 Subject: [PATCH 370/561] fix: make content not required in ChatCompletionRequestAssistantMessage (#1807) --- llama_cpp/llama_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index bbb58afc35..57836ee76f 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -214,7 +214,7 @@ class ChatCompletionRequestAssistantMessageFunctionCall(TypedDict): class ChatCompletionRequestAssistantMessage(TypedDict): role: Literal["assistant"] - content: Optional[str] + content: NotRequired[str] tool_calls: NotRequired[ChatCompletionMessageToolCalls] function_call: NotRequired[ ChatCompletionRequestAssistantMessageFunctionCall From 77a12a39663bbe5a5d22a7cc3d5d4f1c591c2851 Mon Sep 17 00:00:00 2001 From: Olivier DEBAUCHE Date: Fri, 6 Dec 2024 11:01:17 +0100 Subject: [PATCH 371/561] fix: Re-add suport for CUDA 12.5, add CUDA 12.6 (#1775) Add support for Cuda 12.6.1 Update version of Cuda 12.5.0 to 12.5.1 Co-authored-by: Andrei --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 3d9c5d9d4d..8624ec6d9e 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -22,7 +22,7 @@ jobs: $matrix = @{ 'os' = @('ubuntu-latest', 'windows-2019') 'pyver' = @("3.9", "3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1", "12.6.1") 'releasetag' = @("basic") } From 073b7e421e419ec389d8c064bbdae26c482ec17c Mon Sep 17 00:00:00 2001 From: "Ignaz \"Ian\" Kraft" Date: Fri, 6 Dec 2024 11:02:34 +0100 Subject: [PATCH 372/561] fix: added missing exit_stack.close() to /v1/chat/completions (#1796) * fix: added missing exit_stack.close() to /v1/chat/completions * fix: added missing exit_stack.close() to /v1/completions --- llama_cpp/server/app.py | 23 ++++++++++++++++------- llama_cpp/server/errors.py | 6 ++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index cd3255176f..9b8dce418f 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -314,10 +314,14 @@ async def create_completion( else: kwargs["logits_processor"].extend(_min_tokens_logits_processor) - iterator_or_completion: Union[ - llama_cpp.CreateCompletionResponse, - Iterator[llama_cpp.CreateCompletionStreamResponse], - ] = await run_in_threadpool(llama, **kwargs) + try: + iterator_or_completion: Union[ + llama_cpp.CreateCompletionResponse, + Iterator[llama_cpp.CreateCompletionStreamResponse], + ] = await run_in_threadpool(llama, **kwargs) + except Exception as err: + exit_stack.close() + raise err if isinstance(iterator_or_completion, Iterator): # EAFP: It's easier to ask for forgiveness than permission @@ -344,6 +348,7 @@ def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: ping_message_factory=_ping_message_factory, ) else: + exit_stack.close() return iterator_or_completion @@ -508,9 +513,13 @@ async def create_chat_completion( else: kwargs["logits_processor"].extend(_min_tokens_logits_processor) - iterator_or_completion: Union[ - llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] - ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) + try: + iterator_or_completion: Union[ + llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] + ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) + except Exception as err: + exit_stack.close() + raise err if isinstance(iterator_or_completion, Iterator): # EAFP: It's easier to ask for forgiveness than permission diff --git a/llama_cpp/server/errors.py b/llama_cpp/server/errors.py index fbf9fd80d5..d0eda5664b 100644 --- a/llama_cpp/server/errors.py +++ b/llama_cpp/server/errors.py @@ -134,8 +134,6 @@ def error_message_wrapper( ] = None, ) -> Tuple[int, ErrorResponse]: """Wraps error message in OpenAI style error response""" - print(f"Exception: {str(error)}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) if body is not None and isinstance( body, ( @@ -149,6 +147,10 @@ def error_message_wrapper( if match is not None: return callback(body, match) + # Only print the trace on unexpected exceptions + print(f"Exception: {str(error)}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + # Wrap other errors as internal server error return 500, ErrorResponse( message=str(error), From 9bd0c95424aad6a80a7305761b465006af3dde0d Mon Sep 17 00:00:00 2001 From: Graeme Power Date: Fri, 6 Dec 2024 10:03:41 +0000 Subject: [PATCH 373/561] fix: Avoid thread starvation on many concurrent requests by making use of asyncio to lock llama_proxy context (#1798) * fix: make use of asyncio to lock llama_proxy context * fix: use aclose instead of close for AsyncExitStack * fix: don't call exit stack close in stream iterator as it will be called by finally from on_complete anyway * fix: use anyio.Lock instead of asyncio.Lock --------- Co-authored-by: Andrei --- llama_cpp/server/app.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 9b8dce418f..f7c0284756 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -5,7 +5,7 @@ import typing import contextlib -from threading import Lock +from anyio import Lock from functools import partial from typing import Iterator, List, Optional, Union, Dict @@ -70,14 +70,14 @@ def set_llama_proxy(model_settings: List[ModelSettings]): _llama_proxy = LlamaProxy(models=model_settings) -def get_llama_proxy(): +async def get_llama_proxy(): # NOTE: This double lock allows the currently streaming llama model to # check if any other requests are pending in the same thread and cancel # the stream if so. - llama_outer_lock.acquire() + await llama_outer_lock.acquire() release_outer_lock = True try: - llama_inner_lock.acquire() + await llama_inner_lock.acquire() try: llama_outer_lock.release() release_outer_lock = False @@ -159,7 +159,7 @@ async def get_event_publisher( request: Request, inner_send_chan: MemoryObjectSendStream[typing.Any], iterator: Iterator[typing.Any], - on_complete: typing.Optional[typing.Callable[[], None]] = None, + on_complete: typing.Optional[typing.Callable[[], typing.Awaitable[None]]] = None, ): server_settings = next(get_server_settings()) interrupt_requests = ( @@ -182,7 +182,7 @@ async def get_event_publisher( raise e finally: if on_complete: - on_complete() + await on_complete() def _logit_bias_tokens_to_input_ids( @@ -267,10 +267,8 @@ async def create_completion( request: Request, body: CreateCompletionRequest, ) -> llama_cpp.Completion: - exit_stack = contextlib.ExitStack() - llama_proxy = await run_in_threadpool( - lambda: exit_stack.enter_context(contextlib.contextmanager(get_llama_proxy)()) - ) + exit_stack = contextlib.AsyncExitStack() + llama_proxy = await exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) if llama_proxy is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, @@ -332,7 +330,6 @@ async def create_completion( def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: yield first_response yield from iterator_or_completion - exit_stack.close() send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( @@ -342,13 +339,13 @@ def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: request=request, inner_send_chan=send_chan, iterator=iterator(), - on_complete=exit_stack.close, + on_complete=exit_stack.aclose, ), sep="\n", ping_message_factory=_ping_message_factory, ) else: - exit_stack.close() + await exit_stack.aclose() return iterator_or_completion @@ -477,10 +474,8 @@ async def create_chat_completion( # where the dependency is cleaned up before a StreamingResponse # is complete. # https://github.com/tiangolo/fastapi/issues/11143 - exit_stack = contextlib.ExitStack() - llama_proxy = await run_in_threadpool( - lambda: exit_stack.enter_context(contextlib.contextmanager(get_llama_proxy)()) - ) + exit_stack = contextlib.AsyncExitStack() + llama_proxy = exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) if llama_proxy is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, @@ -530,7 +525,6 @@ async def create_chat_completion( def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: yield first_response yield from iterator_or_completion - exit_stack.close() send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( @@ -540,13 +534,13 @@ def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: request=request, inner_send_chan=send_chan, iterator=iterator(), - on_complete=exit_stack.close, + on_complete=exit_stack.aclose, ), sep="\n", ping_message_factory=_ping_message_factory, ) else: - exit_stack.close() + await exit_stack.aclose() return iterator_or_completion From 1ea615424b7647a56946ce4fc3ea72b6ebaa867c Mon Sep 17 00:00:00 2001 From: Florents Tselai Date: Fri, 6 Dec 2024 12:11:50 +0200 Subject: [PATCH 374/561] fix(docs): Update development instructions (#1833) --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dbaec50775..e004565809 100644 --- a/README.md +++ b/README.md @@ -752,15 +752,29 @@ pip install --upgrade pip pip install -e . # if you want to use the fastapi / openapi server -pip install -e .[server] +pip install -e '.[server]' # to install all optional dependencies -pip install -e .[all] +pip install -e '.[all]' # to clear the local build cache make clean ``` +Now try running the tests + +```bash +pytest +``` + +There's a `Makefile` available with useful targets. +A typical workflow would look like this: + +```bash +make build +make test +``` + You can also test out specific commits of `llama.cpp` by checking out the desired commit in the `vendor/llama.cpp` submodule and then running `make clean` and `pip install -e .` again. Any changes in the `llama.h` API will require changes to the `llama_cpp/llama_cpp.py` file to match the new API (additional changes may be required elsewhere). From d610477a0659fd7f56c5712adccebde4135a250c Mon Sep 17 00:00:00 2001 From: Luke Stanley <306671+lukestanley@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:22:26 +0000 Subject: [PATCH 375/561] fix(examples): Refactor Batching notebook to use new sampler chain API (#1793) - Replaced deprecated llama_sample_token with llama_sampler_sample - Updated llama_token_to_piece signature to include lstrip and special arguments - Changed llama_load_model_from_file to use positional argument for params (currently does not accept a keyword argument for it) --- examples/notebooks/Batching.ipynb | 557 ++++++++---------------------- 1 file changed, 146 insertions(+), 411 deletions(-) diff --git a/examples/notebooks/Batching.ipynb b/examples/notebooks/Batching.ipynb index 73b28c7444..be7fe9b522 100644 --- a/examples/notebooks/Batching.ipynb +++ b/examples/notebooks/Batching.ipynb @@ -6,6 +6,7 @@ "metadata": {}, "outputs": [], "source": [ + "import ctypes\n", "import llama_cpp" ] }, @@ -13,18 +14,7 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ggml_init_cublas: GGML_CUDA_FORCE_MMQ: no\n", - "ggml_init_cublas: CUDA_USE_TENSOR_CORES: yes\n", - "ggml_init_cublas: found 1 CUDA devices:\n", - " Device 0: NVIDIA GeForce RTX 2060, compute capability 7.5\n" - ] - } - ], + "outputs": [], "source": [ "llama_cpp.llama_backend_init(numa=False)" ] @@ -38,354 +28,94 @@ "name": "stderr", "output_type": "stream", "text": [ - "llama_model_loader: loaded meta data with 16 key-value pairs and 291 tensors from ../../models/mistral-7b-v0.1-GGUF/ggml-model-Q4_K.gguf (version GGUF V2)\n", - "llama_model_loader: - tensor 0: token_embd.weight q4_K [ 4096, 32000, 1, 1 ]\n", - "llama_model_loader: - tensor 1: output_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 2: output.weight q6_K [ 4096, 32000, 1, 1 ]\n", - "llama_model_loader: - tensor 3: blk.0.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 4: blk.0.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 5: blk.0.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 6: blk.0.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 7: blk.0.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 8: blk.0.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 9: blk.0.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 10: blk.0.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 11: blk.0.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 12: blk.1.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 13: blk.1.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 14: blk.1.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 15: blk.1.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 16: blk.1.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 17: blk.1.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 18: blk.1.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 19: blk.1.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 20: blk.1.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 21: blk.2.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 22: blk.2.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 23: blk.2.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 24: blk.2.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 25: blk.2.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 26: blk.2.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 27: blk.2.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 28: blk.2.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 29: blk.2.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 30: blk.3.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 31: blk.3.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 32: blk.3.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 33: blk.3.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 34: blk.3.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 35: blk.3.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 36: blk.3.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 37: blk.3.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 38: blk.3.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 39: blk.4.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 40: blk.4.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 41: blk.4.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 42: blk.4.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 43: blk.4.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 44: blk.4.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 45: blk.4.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 46: blk.4.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 47: blk.4.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 48: blk.5.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 49: blk.5.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 50: blk.5.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 51: blk.5.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 52: blk.5.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 53: blk.5.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 54: blk.5.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 55: blk.5.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 56: blk.5.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 57: blk.6.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 58: blk.6.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 59: blk.6.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 60: blk.6.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 61: blk.6.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 62: blk.6.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 63: blk.6.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 64: blk.6.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 65: blk.6.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 66: blk.7.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 67: blk.7.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 68: blk.7.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 69: blk.7.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 70: blk.7.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 71: blk.7.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 72: blk.7.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 73: blk.7.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 74: blk.7.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 75: blk.8.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 76: blk.8.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 77: blk.8.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 78: blk.8.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 79: blk.8.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 80: blk.8.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 81: blk.8.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 82: blk.8.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 83: blk.8.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 84: blk.9.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 85: blk.9.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 86: blk.9.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 87: blk.9.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 88: blk.9.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 89: blk.9.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 90: blk.9.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 91: blk.9.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 92: blk.9.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 93: blk.10.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 94: blk.10.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 95: blk.10.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 96: blk.10.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 97: blk.10.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 98: blk.10.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 99: blk.10.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 100: blk.10.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 101: blk.10.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 102: blk.11.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 103: blk.11.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 104: blk.11.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 105: blk.11.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 106: blk.11.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 107: blk.11.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 108: blk.11.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 109: blk.11.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 110: blk.11.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 111: blk.12.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 112: blk.12.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 113: blk.12.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 114: blk.12.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 115: blk.12.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 116: blk.12.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 117: blk.12.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 118: blk.12.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 119: blk.12.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 120: blk.13.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 121: blk.13.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 122: blk.13.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 123: blk.13.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 124: blk.13.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 125: blk.13.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 126: blk.13.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 127: blk.13.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 128: blk.13.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 129: blk.14.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 130: blk.14.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 131: blk.14.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 132: blk.14.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 133: blk.14.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 134: blk.14.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 135: blk.14.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 136: blk.14.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 137: blk.14.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 138: blk.15.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 139: blk.15.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 140: blk.15.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 141: blk.15.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 142: blk.15.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 143: blk.15.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 144: blk.15.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 145: blk.15.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 146: blk.15.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 147: blk.16.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 148: blk.16.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 149: blk.16.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 150: blk.16.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 151: blk.16.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 152: blk.16.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 153: blk.16.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 154: blk.16.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 155: blk.16.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 156: blk.17.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 157: blk.17.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 158: blk.17.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 159: blk.17.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 160: blk.17.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 161: blk.17.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 162: blk.17.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 163: blk.17.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 164: blk.17.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 165: blk.18.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 166: blk.18.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 167: blk.18.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 168: blk.18.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 169: blk.18.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 170: blk.18.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 171: blk.18.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 172: blk.18.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 173: blk.18.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 174: blk.19.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 175: blk.19.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 176: blk.19.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 177: blk.19.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 178: blk.19.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 179: blk.19.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 180: blk.19.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 181: blk.19.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 182: blk.19.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 183: blk.20.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 184: blk.20.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 185: blk.20.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 186: blk.20.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 187: blk.20.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 188: blk.20.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 189: blk.20.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 190: blk.20.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 191: blk.20.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 192: blk.21.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 193: blk.21.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 194: blk.21.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 195: blk.21.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 196: blk.21.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 197: blk.21.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 198: blk.21.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 199: blk.21.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 200: blk.21.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 201: blk.22.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 202: blk.22.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 203: blk.22.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 204: blk.22.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 205: blk.22.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 206: blk.22.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 207: blk.22.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 208: blk.22.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 209: blk.22.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 210: blk.23.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 211: blk.23.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 212: blk.23.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 213: blk.23.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 214: blk.23.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 215: blk.23.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 216: blk.23.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 217: blk.23.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 218: blk.23.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 219: blk.24.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 220: blk.24.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 221: blk.24.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 222: blk.24.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 223: blk.24.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 224: blk.24.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 225: blk.24.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 226: blk.24.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 227: blk.24.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 228: blk.25.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 229: blk.25.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 230: blk.25.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 231: blk.25.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 232: blk.25.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 233: blk.25.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 234: blk.25.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 235: blk.25.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 236: blk.25.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 237: blk.26.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 238: blk.26.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 239: blk.26.attn_v.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 240: blk.26.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 241: blk.26.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 242: blk.26.ffn_down.weight q4_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 243: blk.26.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 244: blk.26.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 245: blk.26.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 246: blk.27.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 247: blk.27.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 248: blk.27.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 249: blk.27.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 250: blk.27.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 251: blk.27.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 252: blk.27.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 253: blk.27.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 254: blk.27.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 255: blk.28.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 256: blk.28.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 257: blk.28.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 258: blk.28.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 259: blk.28.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 260: blk.28.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 261: blk.28.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 262: blk.28.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 263: blk.28.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 264: blk.29.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 265: blk.29.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 266: blk.29.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 267: blk.29.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 268: blk.29.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 269: blk.29.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 270: blk.29.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 271: blk.29.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 272: blk.29.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 273: blk.30.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 274: blk.30.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 275: blk.30.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 276: blk.30.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 277: blk.30.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 278: blk.30.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 279: blk.30.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 280: blk.30.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 281: blk.30.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 282: blk.31.attn_q.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 283: blk.31.attn_k.weight q4_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 284: blk.31.attn_v.weight q6_K [ 4096, 1024, 1, 1 ]\n", - "llama_model_loader: - tensor 285: blk.31.attn_output.weight q4_K [ 4096, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 286: blk.31.ffn_gate.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 287: blk.31.ffn_down.weight q6_K [ 14336, 4096, 1, 1 ]\n", - "llama_model_loader: - tensor 288: blk.31.ffn_up.weight q4_K [ 4096, 14336, 1, 1 ]\n", - "llama_model_loader: - tensor 289: blk.31.attn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - tensor 290: blk.31.ffn_norm.weight f32 [ 4096, 1, 1, 1 ]\n", - "llama_model_loader: - kv 0: general.architecture str \n", - "llama_model_loader: - kv 1: general.name str \n", - "llama_model_loader: - kv 2: llama.context_length u32 \n", - "llama_model_loader: - kv 3: llama.embedding_length u32 \n", - "llama_model_loader: - kv 4: llama.block_count u32 \n", - "llama_model_loader: - kv 5: llama.feed_forward_length u32 \n", - "llama_model_loader: - kv 6: llama.rope.dimension_count u32 \n", - "llama_model_loader: - kv 7: llama.attention.head_count u32 \n", - "llama_model_loader: - kv 8: llama.attention.head_count_kv u32 \n", - "llama_model_loader: - kv 9: llama.attention.layer_norm_rms_epsilon f32 \n", - "llama_model_loader: - kv 10: general.file_type u32 \n", - "llama_model_loader: - kv 11: tokenizer.ggml.model str \n", - "llama_model_loader: - kv 12: tokenizer.ggml.tokens arr \n", - "llama_model_loader: - kv 13: tokenizer.ggml.scores arr \n", - "llama_model_loader: - kv 14: tokenizer.ggml.token_type arr \n", - "llama_model_loader: - kv 15: general.quantization_version u32 \n", + "llama_model_loader: loaded meta data with 20 key-value pairs and 291 tensors from /workspaces/llama-cpp-python/mistral-7b-v0.1.Q2_K.gguf (version GGUF V2)\n", + "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", + "llama_model_loader: - kv 0: general.architecture str = llama\n", + "llama_model_loader: - kv 1: general.name str = mistralai_mistral-7b-v0.1\n", + "llama_model_loader: - kv 2: llama.context_length u32 = 32768\n", + "llama_model_loader: - kv 3: llama.embedding_length u32 = 4096\n", + "llama_model_loader: - kv 4: llama.block_count u32 = 32\n", + "llama_model_loader: - kv 5: llama.feed_forward_length u32 = 14336\n", + "llama_model_loader: - kv 6: llama.rope.dimension_count u32 = 128\n", + "llama_model_loader: - kv 7: llama.attention.head_count u32 = 32\n", + "llama_model_loader: - kv 8: llama.attention.head_count_kv u32 = 8\n", + "llama_model_loader: - kv 9: llama.attention.layer_norm_rms_epsilon f32 = 0.000010\n", + "llama_model_loader: - kv 10: llama.rope.freq_base f32 = 10000.000000\n", + "llama_model_loader: - kv 11: general.file_type u32 = 10\n", + "llama_model_loader: - kv 12: tokenizer.ggml.model str = llama\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "llama_model_loader: - kv 13: tokenizer.ggml.tokens arr[str,32000] = [\"\", \"\", \"\", \"<0x00>\", \"<...\n", + "llama_model_loader: - kv 14: tokenizer.ggml.scores arr[f32,32000] = [0.000000, 0.000000, 0.000000, 0.0000...\n", + "llama_model_loader: - kv 15: tokenizer.ggml.token_type arr[i32,32000] = [2, 3, 3, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n", + "llama_model_loader: - kv 16: tokenizer.ggml.bos_token_id u32 = 1\n", + "llama_model_loader: - kv 17: tokenizer.ggml.eos_token_id u32 = 2\n", + "llama_model_loader: - kv 18: tokenizer.ggml.unknown_token_id u32 = 0\n", + "llama_model_loader: - kv 19: general.quantization_version u32 = 2\n", "llama_model_loader: - type f32: 65 tensors\n", - "llama_model_loader: - type q4_K: 193 tensors\n", - "llama_model_loader: - type q6_K: 33 tensors\n", - "llm_load_vocab: special tokens definition check successful ( 259/32000 ).\n", + "llama_model_loader: - type q2_K: 65 tensors\n", + "llama_model_loader: - type q3_K: 160 tensors\n", + "llama_model_loader: - type q6_K: 1 tensors\n", + "llm_load_vocab: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\n", + "llm_load_vocab: special tokens cache size = 3\n", + "llm_load_vocab: token to piece cache size = 0.1637 MB\n", "llm_load_print_meta: format = GGUF V2\n", "llm_load_print_meta: arch = llama\n", "llm_load_print_meta: vocab type = SPM\n", "llm_load_print_meta: n_vocab = 32000\n", "llm_load_print_meta: n_merges = 0\n", - "llm_load_print_meta: n_ctx_train = 4096\n", + "llm_load_print_meta: vocab_only = 0\n", + "llm_load_print_meta: n_ctx_train = 32768\n", "llm_load_print_meta: n_embd = 4096\n", + "llm_load_print_meta: n_layer = 32\n", "llm_load_print_meta: n_head = 32\n", "llm_load_print_meta: n_head_kv = 8\n", - "llm_load_print_meta: n_layer = 32\n", "llm_load_print_meta: n_rot = 128\n", + "llm_load_print_meta: n_swa = 0\n", + "llm_load_print_meta: n_embd_head_k = 128\n", + "llm_load_print_meta: n_embd_head_v = 128\n", "llm_load_print_meta: n_gqa = 4\n", + "llm_load_print_meta: n_embd_k_gqa = 1024\n", + "llm_load_print_meta: n_embd_v_gqa = 1024\n", "llm_load_print_meta: f_norm_eps = 0.0e+00\n", "llm_load_print_meta: f_norm_rms_eps = 1.0e-05\n", "llm_load_print_meta: f_clamp_kqv = 0.0e+00\n", "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n", + "llm_load_print_meta: f_logit_scale = 0.0e+00\n", "llm_load_print_meta: n_ff = 14336\n", + "llm_load_print_meta: n_expert = 0\n", + "llm_load_print_meta: n_expert_used = 0\n", + "llm_load_print_meta: causal attn = 1\n", + "llm_load_print_meta: pooling type = 0\n", + "llm_load_print_meta: rope type = 0\n", + "llm_load_print_meta: rope scaling = linear\n", "llm_load_print_meta: freq_base_train = 10000.0\n", "llm_load_print_meta: freq_scale_train = 1\n", + "llm_load_print_meta: n_ctx_orig_yarn = 32768\n", + "llm_load_print_meta: rope_finetuned = unknown\n", + "llm_load_print_meta: ssm_d_conv = 0\n", + "llm_load_print_meta: ssm_d_inner = 0\n", + "llm_load_print_meta: ssm_d_state = 0\n", + "llm_load_print_meta: ssm_dt_rank = 0\n", + "llm_load_print_meta: ssm_dt_b_c_rms = 0\n", "llm_load_print_meta: model type = 7B\n", - "llm_load_print_meta: model ftype = mostly Q4_K - Medium\n", + "llm_load_print_meta: model ftype = Q2_K - Medium\n", "llm_load_print_meta: model params = 7.24 B\n", - "llm_load_print_meta: model size = 4.07 GiB (4.83 BPW) \n", - "llm_load_print_meta: general.name = LLaMA v2\n", - "llm_load_print_meta: BOS token = 1 ''\n", - "llm_load_print_meta: EOS token = 2 ''\n", - "llm_load_print_meta: UNK token = 0 ''\n", - "llm_load_print_meta: LF token = 13 '<0x0A>'\n", - "llm_load_tensors: ggml ctx size = 0.10 MB\n", - "llm_load_tensors: using CUDA for GPU acceleration\n", - "llm_load_tensors: mem required = 70.41 MB\n", - "llm_load_tensors: offloading 32 repeating layers to GPU\n", - "llm_load_tensors: offloading non-repeating layers to GPU\n", - "llm_load_tensors: offloaded 35/35 layers to GPU\n", - "llm_load_tensors: VRAM used: 4095.05 MB\n", - ".................................................................................................\n" + "llm_load_print_meta: model size = 2.87 GiB (3.41 BPW) \n", + "llm_load_print_meta: general.name = mistralai_mistral-7b-v0.1\n", + "llm_load_print_meta: BOS token = 1 ''\n", + "llm_load_print_meta: EOS token = 2 ''\n", + "llm_load_print_meta: UNK token = 0 ''\n", + "llm_load_print_meta: LF token = 13 '<0x0A>'\n", + "llm_load_print_meta: EOG token = 2 ''\n", + "llm_load_print_meta: max token length = 48\n", + "llm_load_tensors: ggml ctx size = 0.14 MiB\n", + "llm_load_tensors: CPU buffer size = 2939.57 MiB\n", + "..................................................................................................\n" ] } ], @@ -393,7 +123,7 @@ "params = llama_cpp.llama_model_default_params()\n", "params.n_gpu_layers = 35\n", "model = llama_cpp.llama_load_model_from_file(\n", - " b\"../../models/mistral-7b-v0.1-GGUF/ggml-model-Q4_K.gguf\", params=params\n", + " b\"/workspaces/llama-cpp-python/mistral-7b-v0.1.Q2_K.gguf\", params\n", ") # Update this to whatever" ] }, @@ -406,7 +136,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[1, 1014, 2936, 9060, 285, 1142]\n", + "[1, 415, 2936, 9060, 285, 1142]\n", "58\n" ] } @@ -436,17 +166,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "llama_new_context_with_model: n_ctx = 58\n", + "llama_new_context_with_model: n_ctx = 64\n", + "llama_new_context_with_model: n_batch = 32\n", + "llama_new_context_with_model: n_ubatch = 32\n", + "llama_new_context_with_model: flash_attn = 0\n", "llama_new_context_with_model: freq_base = 10000.0\n", "llama_new_context_with_model: freq_scale = 1\n", - "llama_kv_cache_init: offloading v cache to GPU\n", - "llama_kv_cache_init: offloading k cache to GPU\n", - "llama_kv_cache_init: VRAM kv self = 7.25 MB\n", - "llama_new_context_with_model: kv self size = 7.25 MB\n", - "llama_build_graph: non-view tensors processed: 740/740\n", - "llama_new_context_with_model: compute buffer total size = 10.63 MB\n", - "llama_new_context_with_model: VRAM scratch buffer: 4.51 MB\n", - "llama_new_context_with_model: total VRAM used: 4106.81 MB (model: 4095.05 MB, context: 11.76 MB)\n" + "llama_kv_cache_init: CPU KV buffer size = 8.00 MiB\n", + "llama_new_context_with_model: KV self size = 8.00 MiB, K (f16): 4.00 MiB, V (f16): 4.00 MiB\n", + "llama_new_context_with_model: CPU output buffer size = 0.12 MiB\n", + "llama_new_context_with_model: CPU compute buffer size = 5.01 MiB\n", + "llama_new_context_with_model: graph nodes = 1030\n", + "llama_new_context_with_model: graph splits = 1\n" ] } ], @@ -476,7 +207,7 @@ "metadata": {}, "outputs": [], "source": [ - "import ctypes\n", + "\n", "\n", "batch.n_tokens = tokens_len\n", "for i in range(tokens_len):\n", @@ -502,10 +233,35 @@ " llama_cpp.llama_kv_cache_seq_cp(ctx, 0, i, 0, batch.n_tokens)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": 9, "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Initialize sampler chain with default parameters\n", + "sparams = llama_cpp.llama_sampler_chain_default_params()\n", + "sampler_chain = llama_cpp.llama_sampler_chain_init(sparams)\n", + "\n", + "# Add top_k, top_p, temperature, and final distribution-based sampler\n", + "llama_cpp.llama_sampler_chain_add(sampler_chain, llama_cpp.llama_sampler_init_top_k(40))\n", + "llama_cpp.llama_sampler_chain_add(sampler_chain, llama_cpp.llama_sampler_init_top_p(0.9, 1))\n", + "llama_cpp.llama_sampler_chain_add(sampler_chain, llama_cpp.llama_sampler_init_temp(0.4))\n", + "llama_cpp.llama_sampler_chain_add(sampler_chain, llama_cpp.llama_sampler_init_dist(1234)) # Final \"dist\" sampler" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -514,61 +270,59 @@ "7\n", "[' j', ' jumped']\n", "8\n", - "[' jumps', ' jumped over']\n", + "[' j over', ' jumped over']\n", "9\n", - "[' jumps over', ' jumped over the']\n", + "[' j over the', ' jumped over the']\n", "10\n", - "[' jumps over the', ' jumped over the lazy']\n", + "[' j over the lazy', ' jumped over the lazy']\n", "11\n", - "[' jumps over the lazy', ' jumped over the lazy dog']\n", + "[' j over the lazy dog', ' jumped over the lazy dog']\n", "12\n", - "[' jumps over the lazy dog', ' jumped over the lazy dog.']\n", + "[' j over the lazy dog.', ' jumped over the lazy dog\\n']\n", "13\n", - "[' jumps over the lazy dog.', ' jumped over the lazy dog.\\n']\n", + "[' j over the lazy dog. También', ' jumped over the lazy dog\\nGroupLayout']\n", "14\n", - "[' jumps over the lazy dog.\\n', ' jumped over the lazy dog.\\n\\n']\n", + "[' j over the lazy dog. También:', ' jumped over the lazy dog\\nGroupLayouting']\n", "15\n", - "[' jumps over the lazy dog.\\n\\n', ' jumped over the lazy dog.\\n\\nThe']\n", + "[' j over the lazy dog. También: is', ' jumped over the lazy dog\\nGroupLayouting is']\n", "16\n", - "[' jumps over the lazy dog.\\n\\nI', ' jumped over the lazy dog.\\n\\nThe quick']\n", + "[' j over the lazy dog. También: is a', ' jumped over the lazy dog\\nGroupLayouting is a']\n", "17\n", - "[' jumps over the lazy dog.\\n\\nI’', ' jumped over the lazy dog.\\n\\nThe quick brown']\n", + "[' j over the lazy dog. También: is a technique', ' jumped over the lazy dog\\nGroupLayouting is a common']\n", "18\n", - "[' jumps over the lazy dog.\\n\\nI’m', ' jumped over the lazy dog.\\n\\nThe quick brown f']\n", + "[' j over the lazy dog. También: is a technique practice', ' jumped over the lazy dog\\nGroupLayouting is a common practice']\n", "19\n", - "[' jumps over the lazy dog.\\n\\nI’m not', ' jumped over the lazy dog.\\n\\nThe quick brown fox']\n", + "[' j over the lazy dog. También: is a technique practice in', ' jumped over the lazy dog\\nGroupLayouting is a common practice in']\n", "20\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped']\n", + "[' j over the lazy dog. También: is a technique practice in the', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the']\n", "21\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over']\n", + "[' j over the lazy dog. También: is a technique practice in the real', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media']\n", "22\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the']\n", + "[' j over the lazy dog. También: is a technique practice in the real-', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry']\n", "23\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy']\n", + "[' j over the lazy dog. También: is a technique practice in the real-.', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry.']\n", "24\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However']\n", "25\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We,', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However,']\n", "26\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there']\n", "27\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\n']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has']\n", "28\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been']\n", "29\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little']\n", "30\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in the', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick brown']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little research', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little emp']\n", "31\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in the English', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick brown f']\n", + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little researchirical', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little empirical']\n", "32\n", - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in the English language', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick brown fox']\n" + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little researchirical research', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little empirical research']\n" ] } ], "source": [ - "import ctypes\n", - "\n", "streams = [\"\"] * n_parallel\n", "i_batch = [batch.n_tokens - 1] * n_parallel\n", "\n", @@ -581,36 +335,17 @@ " if i_batch[i] < 0:\n", " continue\n", "\n", - " n_vocab = llama_cpp.llama_n_vocab(model)\n", - " logits = llama_cpp.llama_get_logits_ith(ctx, i_batch[i])\n", - "\n", - " candidates = (llama_cpp.llama_token_data * n_vocab)()\n", - "\n", - " for token_id in range(n_vocab):\n", - " candidates[token_id].id = token_id\n", - " candidates[token_id].logit = logits[token_id]\n", - " candidates[token_id].p = 0.0\n", - "\n", - " candidates_p = llama_cpp.llama_token_data_array(\n", - " candidates, len(candidates), False\n", - " )\n", - "\n", - " top_k = 40\n", - " top_p = 0.9\n", - " temp = 0.4\n", - "\n", - " llama_cpp.llama_sample_top_k(ctx, ctypes.byref(candidates_p), top_k, 1)\n", - " llama_cpp.llama_sample_top_p(ctx, ctypes.byref(candidates_p), top_p, 1)\n", - " llama_cpp.llama_sample_temp(ctx, ctypes.byref(candidates_p), temp)\n", - "\n", - " new_token_id = llama_cpp.llama_sample_token(ctx, ctypes.byref(candidates_p))\n", + " # Sample the next token using the sampler chain\n", + " new_token_id = llama_cpp.llama_sampler_sample(sampler_chain, ctx, -1)\n", "\n", " if new_token_id == llama_cpp.llama_token_eos(ctx) or n_cur == n_len:\n", " i_batch[i] = -1\n", " continue\n", "\n", " buf = (ctypes.c_char * 32)()\n", - " outlen = llama_cpp.llama_token_to_piece(model, new_token_id, buf, len(buf))\n", + " \n", + " # Convert token ID to text\n", + " outlen = llama_cpp.llama_token_to_piece(model, new_token_id, buf, len(buf), 0, False)\n", " streams[i] += bytes(buf[:outlen]).decode(\"utf-8\")\n", "\n", " batch.token[batch.n_tokens] = new_token_id\n", @@ -637,14 +372,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[' jumps over the lazy dog.\\n\\nI’m not sure if that’s the most famous sentence in the English language', ' jumped over the lazy dog.\\n\\nThe quick brown fox jumped over the lazy dog.\\n\\nThe quick brown fox']\n" + "[' j over the lazy dog. También: is a technique practice in the real-. We, when is been little researchirical research', ' jumped over the lazy dog\\nGroupLayouting is a common practice in the media industry. However, there has been little empirical research']\n" ] } ], @@ -654,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -663,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -672,7 +407,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -681,7 +416,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -705,7 +440,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -719,7 +454,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.5+" + "version": "3.12.1" }, "orig_nbformat": 4 }, From 4f0ec659f9519d3009873f6adf6aa8af5ab32d8b Mon Sep 17 00:00:00 2001 From: Adam Jones Date: Fri, 6 Dec 2024 12:35:46 +0000 Subject: [PATCH 376/561] fix: chat API logprobs format (#1788) * fix: chat API logprobs format * Fix optional properties --- Makefile | 2 +- llama_cpp/llama_chat_format.py | 53 ++++++++++++++++++++++++---------- llama_cpp/llama_types.py | 22 ++++++++++++-- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index be9b55a3e7..52b17b61a7 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ docker: docker build -t llama-cpp-python:latest -f docker/simple/Dockerfile . run-server: - uvicorn --factory llama.server:app --host ${HOST} --port ${PORT} + python llama_cpp/server --model ${MODEL} clean: - cd vendor/llama.cpp && make clean diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index dfb0af65e6..17575c7008 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -259,6 +259,31 @@ def to_chat_handler(self) -> LlamaChatCompletionHandler: return chat_formatter_to_chat_completion_handler(self) +def _convert_text_completion_logprobs_to_chat( + logprobs: Optional[llama_types.CompletionLogprobs], +) -> llama_types.ChatCompletionLogprobs: + if logprobs is None: + return None + + return { + "content": [ + { + "token": token, + "bytes": None, + "logprob": logprob, + "top_logprobs": [ + { + "token": top_token, + "logprob": top_logprob, + "bytes": None, + } + for top_token, top_logprob in top_logprobs.items() + ], + } for (token, logprob, top_logprobs) in zip(logprobs["tokens"], logprobs["token_logprobs"], logprobs["top_logprobs"]) + ], + "refusal": None, + } + def _convert_text_completion_to_chat( completion: llama_types.Completion, ) -> llama_types.ChatCompletion: @@ -275,7 +300,7 @@ def _convert_text_completion_to_chat( "role": "assistant", "content": completion["choices"][0]["text"], }, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "finish_reason": completion["choices"][0]["finish_reason"], } ], @@ -319,7 +344,7 @@ def _convert_text_completion_chunks_to_chat( if chunk["choices"][0]["finish_reason"] is None else {} ), - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "finish_reason": chunk["choices"][0]["finish_reason"], } ], @@ -382,7 +407,7 @@ def _convert_completion_to_chat_function( } ], }, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "finish_reason": "tool_calls", } ], @@ -435,7 +460,7 @@ def _stream_response_to_function_stream( { "index": 0, "finish_reason": None, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": None, "content": None, @@ -472,7 +497,7 @@ def _stream_response_to_function_stream( { "index": 0, "finish_reason": None, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": None, "content": None, @@ -1716,7 +1741,7 @@ def message_to_str(msg: llama_types.ChatCompletionRequestMessage): } ], }, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "finish_reason": "tool_calls", } ], @@ -2128,7 +2153,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": None, "content": None, @@ -2230,7 +2255,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": "assistant", "content": None, @@ -2268,9 +2293,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0][ - "logprobs" - ], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": "assistant", "content": buffer.pop(0), @@ -2293,7 +2316,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": "assistant", "content": ( @@ -2379,7 +2402,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": chunk["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), "delta": { "role": None, "content": None, @@ -2613,7 +2636,7 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "message": { "role": "assistant", "content": None if content == "" else content, @@ -3745,7 +3768,7 @@ def chatml_function_calling( { "finish_reason": "tool_calls", "index": 0, - "logprobs": completion["choices"][0]["logprobs"], + "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), "message": { "role": "assistant", "content": None, diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 57836ee76f..f647822ff5 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -82,10 +82,28 @@ class ChatCompletionFunction(TypedDict): parameters: Dict[str, JsonType] # TODO: make this more specific +class ChatCompletionTopLogprobToken(TypedDict): + token: str + logprob: float + bytes: Optional[List[int]] + + +class ChatCompletionLogprobToken(ChatCompletionTopLogprobToken): + token: str + logprob: float + bytes: Optional[List[int]] + top_logprobs: List[ChatCompletionTopLogprobToken] + + +class ChatCompletionLogprobs(TypedDict): + content: Optional[List[ChatCompletionLogprobToken]] + refusal: Optional[List[ChatCompletionLogprobToken]] + + class ChatCompletionResponseChoice(TypedDict): index: int message: "ChatCompletionResponseMessage" - logprobs: Optional[CompletionLogprobs] + logprobs: Optional[ChatCompletionLogprobs] finish_reason: Optional[str] @@ -134,7 +152,7 @@ class ChatCompletionStreamResponseChoice(TypedDict): ChatCompletionStreamResponseDelta, ChatCompletionStreamResponseDeltaEmpty ] finish_reason: Optional[Literal["stop", "length", "tool_calls", "function_call"]] - logprobs: NotRequired[Optional[CompletionLogprobs]] + logprobs: NotRequired[Optional[ChatCompletionLogprobs]] class CreateChatCompletionStreamResponse(TypedDict): From df136cb3c7d3a7a9a4e30ef2d16c5e42e0e36e43 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 07:36:47 -0500 Subject: [PATCH 377/561] misc: Update development Makefile --- Makefile | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index be9b55a3e7..37ef9b1d60 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,15 @@ build.debug: --config-settings=cmake.args="-DCMAKE_BUILD_TYPE=Debug;-DCMAKE_C_FLAGS='-ggdb -O0';-DCMAKE_CXX_FLAGS='-ggdb -O0'" \ --editable . +build.debug.extra: + python3 -m pip install \ + --verbose \ + --config-settings=cmake.verbose=true \ + --config-settings=logging.level=INFO \ + --config-settings=install.strip=false \ + --config-settings=cmake.args="-DCMAKE_BUILD_TYPE=Debug;-DCMAKE_C_FLAGS='-fsanitize=address -ggdb -O0';-DCMAKE_CXX_FLAGS='-fsanitize=address -ggdb -O0'" \ + --editable . + build.cuda: CMAKE_ARGS="-DGGML_CUDA=on" python3 -m pip install --verbose -e . @@ -46,7 +55,7 @@ build.rpc: CMAKE_ARGS="-DGGML_RPC=on" python3 -m pip install --verbose -e . build.sdist: - python3 -m build --sdist + python3 -m build --sdist --verbose deploy.pypi: python3 -m twine upload dist/* @@ -56,7 +65,7 @@ deploy.gh-docs: mkdocs gh-deploy test: - python3 -m pytest + python3 -m pytest --full-trace -v docker: docker build -t llama-cpp-python:latest -f docker/simple/Dockerfile . @@ -68,11 +77,11 @@ clean: - cd vendor/llama.cpp && make clean - cd vendor/llama.cpp && rm libllama.so - rm -rf _skbuild - - rm llama_cpp/*.so - - rm llama_cpp/*.dylib - - rm llama_cpp/*.metal - - rm llama_cpp/*.dll - - rm llama_cpp/*.lib + - rm llama_cpp/lib/*.so + - rm llama_cpp/lib/*.dylib + - rm llama_cpp/lib/*.metal + - rm llama_cpp/lib/*.dll + - rm llama_cpp/lib/*.lib .PHONY: \ update \ From b9b50e50b9d29b2034c50e1b0ee09dc7c47bbc63 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 6 Dec 2024 07:37:53 -0500 Subject: [PATCH 378/561] misc: Update run server command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 78b11cf11b..26ddf2c7a1 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ docker: docker build -t llama-cpp-python:latest -f docker/simple/Dockerfile . run-server: - python llama_cpp/server --model ${MODEL} + python3 -m llama_cpp.server --model ${MODEL} clean: - cd vendor/llama.cpp && make clean From 5585f8afe1aea74dca7364f18a76f0426381001d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 22:40:19 -0500 Subject: [PATCH 379/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 12 ++++++------ vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index cd59bb4304..44a1461db7 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -344,9 +344,9 @@ # LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors # LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors # LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors -# LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // except 1d tensors -# LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // except 1d tensors -# LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // except 1d tensors +# //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack +# //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack +# //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack # LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors # LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors # @@ -382,9 +382,9 @@ LLAMA_FTYPE_MOSTLY_IQ4_XS = 30 LLAMA_FTYPE_MOSTLY_IQ1_M = 31 LLAMA_FTYPE_MOSTLY_BF16 = 32 -LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 -LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 -LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 +# LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 +# LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 +# LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 LLAMA_FTYPE_MOSTLY_TQ1_0 = 36 LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 LLAMA_FTYPE_GUESSED = 1024 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7736837d62..ce8784bdb1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7736837d62efed1dbebfe579472fca041eda12d6 +Subproject commit ce8784bdb153ff7794dde5a50b0ebfa51baa6171 From 61508c265a4e295d69aaee7533bde5b51278e574 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 22:55:03 -0500 Subject: [PATCH 380/561] Add CUDA 12.5 and 12.6 to generated output wheels --- .github/workflows/generate-index-from-release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 51961088d7..5c7171bc16 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -44,6 +44,8 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 From a9fe0f872527ffc1bf64e125b7996ec4dfa598eb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 22:56:45 -0500 Subject: [PATCH 381/561] chore: Bump version --- CHANGELOG.md | 14 ++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a194fecef..331161a0ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.3] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ce8784bdb153ff7794dde5a50b0ebfa51baa6171 +- fix: chat API logprobs format by @domdomegg in #1788 +- feat: Add support for CUDA 12.6, fix CUDA 12.5 by @Smartappli in #1775 +- fix: Make content not required in ChatCompletionRequestAssistantMessage by @feloy in #1807 +- fix: Fix pickling of Llama class by setting seed from _seed member by @abetlen in 2523472c3eccb9ab9277117cc4ff705212b6888a +- fix: Fix logit-bias type hint by @ddh0 in #1802 +- fix(server): Avoid thread starvation on many concurrent requests by making use of asyncio to lock llama_proxy context by @gjpower in #1798 +- fix(server): Added missing exit_stack.close() to /v1/chat/completions by @Ian321 in #1796 +- fix(examples): Refactor Batching notebook to use new sampler chain API by @lukestanley in #1793 +- fix(docs): Update development instructions by @Florents-Tselai in #1833 +- fix(docs): Remove ref to llama_eval in llama_cpp.py docs by @richdougherty in #1819 + ## [0.3.2] - feat: Update llama.cpp to ggerganov/llama.cpp@74d73dc85cc2057446bf63cc37ff649ae7cebd80 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index cda38a841f..36c0eafc07 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.2" +__version__ = "0.3.3" From ca808028bd16b8327bd84128d48015a4b1304690 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 23:51:49 -0500 Subject: [PATCH 382/561] fix(ci): hotfix for wheels --- .github/workflows/build-wheels-cuda.yaml | 2 +- .github/workflows/build-wheels-metal.yaml | 2 +- .github/workflows/generate-index-from-release.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 8624ec6d9e..3d410148f2 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -22,7 +22,7 @@ jobs: $matrix = @{ 'os' = @('ubuntu-latest', 'windows-2019') 'pyver' = @("3.9", "3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1", "12.6.1") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") #, "12.5.1", "12.6.1") 'releasetag' = @("basic") } diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 01da3cef51..9b97bf2f56 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-12, macos-13, macos-14] + os: [macos-13, macos-14, macos-15] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 5c7171bc16..255ee67d6f 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -44,8 +44,8 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' - ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' - ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + # ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + # ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 From 002f58300fdaf865cf1d305b3721c5d96003c440 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 8 Dec 2024 23:53:30 -0500 Subject: [PATCH 383/561] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 331161a0ce..2a1d3225f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.4] + +- fix(ci): Build wheels for macos 13-15, cuda 12.1-12.4 by @abetlen in ca808028bd16b8327bd84128d48015a4b1304690 + ## [0.3.3] - feat: Update llama.cpp to ggerganov/llama.cpp@ce8784bdb153ff7794dde5a50b0ebfa51baa6171 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 36c0eafc07..dd8f1fc581 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.3" +__version__ = "0.3.4" From ea4d86ac25e0f39370ff3f54550387a02467ab77 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 9 Dec 2024 00:42:15 -0500 Subject: [PATCH 384/561] fix(ci): update macos runner image to non-deprecated version --- .github/workflows/build-and-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 1d549ed68e..7307c85ab6 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-12] + os: [ubuntu-20.04, windows-2019, macos-13] steps: - uses: actions/checkout@v4 From afedfc888462f9a6e809dc9455eb3b663764cc3f Mon Sep 17 00:00:00 2001 From: Graeme Power Date: Mon, 9 Dec 2024 23:47:18 +0000 Subject: [PATCH 385/561] fix: add missing await statements for async exit_stack handling (#1858) --- llama_cpp/server/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index f7c0284756..b6db453b88 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -318,7 +318,7 @@ async def create_completion( Iterator[llama_cpp.CreateCompletionStreamResponse], ] = await run_in_threadpool(llama, **kwargs) except Exception as err: - exit_stack.close() + await exit_stack.aclose() raise err if isinstance(iterator_or_completion, Iterator): @@ -475,7 +475,7 @@ async def create_chat_completion( # is complete. # https://github.com/tiangolo/fastapi/issues/11143 exit_stack = contextlib.AsyncExitStack() - llama_proxy = exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) + llama_proxy = await exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) if llama_proxy is None: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, @@ -513,7 +513,7 @@ async def create_chat_completion( llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) except Exception as err: - exit_stack.close() + await exit_stack.aclose() raise err if isinstance(iterator_or_completion, Iterator): From 801a73a621c6ba8f9a8ffe110766c9e721b09520 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 9 Dec 2024 18:49:27 -0500 Subject: [PATCH 386/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ce8784bdb1..26a8406ba9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ce8784bdb153ff7794dde5a50b0ebfa51baa6171 +Subproject commit 26a8406ba9198eb6fdd8329fa717555b4f77f05f From 803924b02cc8ca7531b70e6497d1ff7b83f75737 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 9 Dec 2024 18:59:22 -0500 Subject: [PATCH 387/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1d3225f4..6f63558933 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.5] + +- feat: Update llama.cpp to ggerganov/llama.cpp@26a8406ba9198eb6fdd8329fa717555b4f77f05f +- fix(ci): Fix release by updating macos runner image to non-deprecated version by @abetlen in afedfc888462f9a6e809dc9455eb3b663764cc3f +- fix(server): add missing await statements for async exit_stack handling by @gjpower in #1858 + ## [0.3.4] - fix(ci): Build wheels for macos 13-15, cuda 12.1-12.4 by @abetlen in ca808028bd16b8327bd84128d48015a4b1304690 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index dd8f1fc581..d7d9be881a 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.4" +__version__ = "0.3.5" From 2bc1d97c9672320828e70dc8293d5f8754682109 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 19 Dec 2024 01:55:12 -0500 Subject: [PATCH 388/561] feat: Update llama.cpp --- llama_cpp/_internals.py | 5 ---- llama_cpp/llama_cpp.py | 54 +++++++++++------------------------------ vendor/llama.cpp | 2 +- 3 files changed, 15 insertions(+), 46 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 994d5f1497..8fa2b447ff 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -805,15 +805,10 @@ def add_penalties( ignore_eos: bool, ): sampler = llama_cpp.llama_sampler_init_penalties( - n_vocab, - special_eos_id, - linefeed_id, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, - penalize_nl, - ignore_eos, ) self._add_sampler(sampler) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 44a1461db7..0481cdbcd4 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -256,13 +256,17 @@ # // note: these values should be synchronized with ggml_rope # // TODO: maybe move this enum to ggml.h (ggml_rope_type) # enum llama_rope_type { -# LLAMA_ROPE_TYPE_NONE = -1, -# LLAMA_ROPE_TYPE_NORM = 0, -# LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, +# LLAMA_ROPE_TYPE_NONE = -1, +# LLAMA_ROPE_TYPE_NORM = 0, +# LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, +# LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE, +# LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION, # }; LLAMA_ROPE_TYPE_NONE = -1 LLAMA_ROPE_TYPE_NORM = 0 LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2 +LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE = 8 +LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION = 24 # enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file @@ -1265,6 +1269,7 @@ def llama_rope_freq_scale_train(model: llama_model_p, /) -> float: # // Functions to access the model's GGUF metadata scalar values # // - The functions return the length of the string on success, or -1 on failure # // - The output string is always null-terminated and cleared on failure +# // - When retrieving a string, an extra byte must be allocated to account for the null terminator # // - GGUF array values are not supported by these functions @@ -1378,18 +1383,6 @@ def llama_model_n_params(model: llama_model_p, /) -> int: ... -# // Get a llama model tensor -# LLAMA_API struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name); -@ctypes_function( - "llama_get_model_tensor", [llama_model_p_ctypes, ctypes.c_char_p], ctypes.c_void_p -) -def llama_get_model_tensor( - model: llama_model_p, name: Union[ctypes.c_char_p, bytes], / -) -> ctypes.c_void_p: - """Get a llama model tensor""" - ... - - # // Returns true if the model contains an encoder that requires llama_encode() call # LLAMA_API bool llama_model_has_encoder(const struct llama_model * model); @ctypes_function("llama_model_has_encoder", [llama_model_p_ctypes], ctypes.c_bool) @@ -3336,41 +3329,22 @@ def llama_sampler_init_grammar( ... +# /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( -# int32_t n_vocab, // llama_n_vocab() -# llama_token special_eos_id, // llama_token_eos() -# llama_token linefeed_id, // llama_token_nl() -# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) -# float penalty_repeat, // 1.0 = disabled -# float penalty_freq, // 0.0 = disabled -# float penalty_present, // 0.0 = disabled -# bool penalize_nl, // consider newlines as a repeatable token -# bool ignore_eos); // ignore the end-of-sequence token +# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) +# float penalty_repeat, // 1.0 = disabled +# float penalty_freq, // 0.0 = disabled +# float penalty_present); // 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", - [ - ctypes.c_int32, - llama_token, - llama_token, - ctypes.c_int32, - ctypes.c_float, - ctypes.c_float, - ctypes.c_float, - ctypes.c_bool, - ctypes.c_bool, - ], + [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], llama_sampler_p_ctypes, ) def llama_sampler_init_penalties( - n_vocab: int, - special_eos_id: int, - linefeed_id: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, - penalize_nl: bool, - ignore_eos: bool, /, ) -> llama_sampler_p: ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 26a8406ba9..7909e8588d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 26a8406ba9198eb6fdd8329fa717555b4f77f05f +Subproject commit 7909e8588ddf70820adf1f325490eb3f67b32875 From c9dfad4b6fdebc85f8f7914e8e46c6d9299ff4b4 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Mon, 30 Dec 2024 13:43:46 -0500 Subject: [PATCH 389/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7909e8588d..716bd6dec3 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7909e8588ddf70820adf1f325490eb3f67b32875 +Subproject commit 716bd6dec3e044e5c325386b5b0483392b24cefe From 1d5f5340e49d90bc097c1f75605a4dd7408c2378 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 8 Jan 2025 16:33:53 -0500 Subject: [PATCH 390/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 31 +++++++++++++++++++++++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 0481cdbcd4..16c6b9d733 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -222,6 +222,7 @@ # LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, # LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, # LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -251,6 +252,7 @@ LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 +LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28 # // note: these values should be synchronized with ggml_rope @@ -1090,9 +1092,10 @@ def llama_backend_free(): ... -# LLAMA_API struct llama_model * llama_load_model_from_file( +# DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( # const char * path_model, -# struct llama_model_params params); +# struct llama_model_params params), +# "use llama_model_load_from_file instead"); @ctypes_function( "llama_load_model_from_file", [ctypes.c_char_p, llama_model_params], @@ -1104,6 +1107,20 @@ def llama_load_model_from_file( ... +# LLAMA_API struct llama_model * llama_model_load_from_file( +# const char * path_model, +# struct llama_model_params params); +@ctypes_function( + "llama_model_load_from_file", + [ctypes.c_char_p, llama_model_params], + llama_model_p_ctypes, +) +def llama_model_load_from_file( + path_model: bytes, params: llama_model_params, / +) -> Optional[llama_model_p]: + ... + + # LLAMA_API void llama_free_model(struct llama_model * model); @ctypes_function( "llama_free_model", @@ -1114,6 +1131,16 @@ def llama_free_model(model: llama_model_p, /): ... +# LLAMA_API void llama_model_free(struct llama_model * model); +@ctypes_function( + "llama_model_free", + [llama_model_p_ctypes], + None, +) +def llama_model_free(model: llama_model_p, /): + ... + + # LLAMA_API struct llama_context * llama_new_context_with_model( # struct llama_model * model, # struct llama_context_params params); diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 716bd6dec3..f7cd13301c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 716bd6dec3e044e5c325386b5b0483392b24cefe +Subproject commit f7cd13301c2a88f97073fd119072b4cc92c08df1 From e8f14ceff9b3279c67368d85f9457bb27168d126 Mon Sep 17 00:00:00 2001 From: Graeme Power Date: Wed, 8 Jan 2025 21:46:43 +0000 Subject: [PATCH 391/561] fix: streaming resource lock (#1879) * fix: correct issue with handling lock during streaming move locking for streaming into get_event_publisher call so it is locked and unlocked in the correct task for the streaming reponse * fix: simplify exit stack management for create_chat_completion and create_completion * fix: correct missing `async with` and format code * fix: remove unnecessary explicit use of AsyncExitStack fix: correct type hints for body_model --------- Co-authored-by: Andrei --- llama_cpp/server/app.py | 224 ++++++++++++++++++---------------------- 1 file changed, 103 insertions(+), 121 deletions(-) diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index b6db453b88..5120f24162 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -7,7 +7,7 @@ from anyio import Lock from functools import partial -from typing import Iterator, List, Optional, Union, Dict +from typing import List, Optional, Union, Dict import llama_cpp @@ -155,34 +155,71 @@ def create_app( return app +def prepare_request_resources( + body: CreateCompletionRequest | CreateChatCompletionRequest, + llama_proxy: LlamaProxy, + body_model: str | None, + kwargs, +) -> llama_cpp.Llama: + if llama_proxy is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Service is not available", + ) + llama = llama_proxy(body_model) + if body.logit_bias is not None: + kwargs["logit_bias"] = ( + _logit_bias_tokens_to_input_ids(llama, body.logit_bias) + if body.logit_bias_type == "tokens" + else body.logit_bias + ) + + if body.grammar is not None: + kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) + + if body.min_tokens > 0: + _min_tokens_logits_processor = llama_cpp.LogitsProcessorList( + [llama_cpp.MinTokensLogitsProcessor(body.min_tokens, llama.token_eos())] + ) + if "logits_processor" not in kwargs: + kwargs["logits_processor"] = _min_tokens_logits_processor + else: + kwargs["logits_processor"].extend(_min_tokens_logits_processor) + return llama + + async def get_event_publisher( request: Request, inner_send_chan: MemoryObjectSendStream[typing.Any], - iterator: Iterator[typing.Any], - on_complete: typing.Optional[typing.Callable[[], typing.Awaitable[None]]] = None, + body: CreateCompletionRequest | CreateChatCompletionRequest, + body_model: str | None, + llama_call, + kwargs, ): server_settings = next(get_server_settings()) interrupt_requests = ( server_settings.interrupt_requests if server_settings else False ) - async with inner_send_chan: - try: - async for chunk in iterate_in_threadpool(iterator): - await inner_send_chan.send(dict(data=json.dumps(chunk))) - if await request.is_disconnected(): - raise anyio.get_cancelled_exc_class()() - if interrupt_requests and llama_outer_lock.locked(): - await inner_send_chan.send(dict(data="[DONE]")) - raise anyio.get_cancelled_exc_class()() - await inner_send_chan.send(dict(data="[DONE]")) - except anyio.get_cancelled_exc_class() as e: - print("disconnected") - with anyio.move_on_after(1, shield=True): - print(f"Disconnected from client (via refresh/close) {request.client}") - raise e - finally: - if on_complete: - await on_complete() + async with contextlib.asynccontextmanager(get_llama_proxy)() as llama_proxy: + llama = prepare_request_resources(body, llama_proxy, body_model, kwargs) + async with inner_send_chan: + try: + iterator = await run_in_threadpool(llama_call, llama, **kwargs) + async for chunk in iterate_in_threadpool(iterator): + await inner_send_chan.send(dict(data=json.dumps(chunk))) + if await request.is_disconnected(): + raise anyio.get_cancelled_exc_class()() + if interrupt_requests and llama_outer_lock.locked(): + await inner_send_chan.send(dict(data="[DONE]")) + raise anyio.get_cancelled_exc_class()() + await inner_send_chan.send(dict(data="[DONE]")) + except anyio.get_cancelled_exc_class() as e: + print("disconnected") + with anyio.move_on_after(1, shield=True): + print( + f"Disconnected from client (via refresh/close) {request.client}" + ) + raise e def _logit_bias_tokens_to_input_ids( @@ -267,18 +304,11 @@ async def create_completion( request: Request, body: CreateCompletionRequest, ) -> llama_cpp.Completion: - exit_stack = contextlib.AsyncExitStack() - llama_proxy = await exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) - if llama_proxy is None: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Service is not available", - ) if isinstance(body.prompt, list): assert len(body.prompt) <= 1 body.prompt = body.prompt[0] if len(body.prompt) > 0 else "" - llama = llama_proxy( + body_model = ( body.model if request.url.path != "/v1/engines/copilot-codex/completions" else "copilot-codex" @@ -293,44 +323,8 @@ async def create_completion( } kwargs = body.model_dump(exclude=exclude) - if body.logit_bias is not None: - kwargs["logit_bias"] = ( - _logit_bias_tokens_to_input_ids(llama, body.logit_bias) - if body.logit_bias_type == "tokens" - else body.logit_bias - ) - - if body.grammar is not None: - kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) - - if body.min_tokens > 0: - _min_tokens_logits_processor = llama_cpp.LogitsProcessorList( - [llama_cpp.MinTokensLogitsProcessor(body.min_tokens, llama.token_eos())] - ) - if "logits_processor" not in kwargs: - kwargs["logits_processor"] = _min_tokens_logits_processor - else: - kwargs["logits_processor"].extend(_min_tokens_logits_processor) - - try: - iterator_or_completion: Union[ - llama_cpp.CreateCompletionResponse, - Iterator[llama_cpp.CreateCompletionStreamResponse], - ] = await run_in_threadpool(llama, **kwargs) - except Exception as err: - await exit_stack.aclose() - raise err - - if isinstance(iterator_or_completion, Iterator): - # EAFP: It's easier to ask for forgiveness than permission - first_response = await run_in_threadpool(next, iterator_or_completion) - - # If no exception was raised from first_response, we can assume that - # the iterator is valid and we can use it to stream the response. - def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: - yield first_response - yield from iterator_or_completion - + # handle streaming request + if kwargs.get("stream", False): send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( recv_chan, @@ -338,15 +332,29 @@ def iterator() -> Iterator[llama_cpp.CreateCompletionStreamResponse]: get_event_publisher, request=request, inner_send_chan=send_chan, - iterator=iterator(), - on_complete=exit_stack.aclose, + body=body, + body_model=body_model, + llama_call=llama_cpp.Llama.__call__, + kwargs=kwargs, ), sep="\n", ping_message_factory=_ping_message_factory, ) - else: - await exit_stack.aclose() - return iterator_or_completion + + # handle regular request + async with contextlib.asynccontextmanager(get_llama_proxy)() as llama_proxy: + llama = prepare_request_resources(body, llama_proxy, body_model, kwargs) + + if await request.is_disconnected(): + print( + f"Disconnected from client (via refresh/close) before llm invoked {request.client}" + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Client closed request", + ) + + return await run_in_threadpool(llama, **kwargs) @router.post( @@ -474,13 +482,8 @@ async def create_chat_completion( # where the dependency is cleaned up before a StreamingResponse # is complete. # https://github.com/tiangolo/fastapi/issues/11143 - exit_stack = contextlib.AsyncExitStack() - llama_proxy = await exit_stack.enter_async_context(contextlib.asynccontextmanager(get_llama_proxy)()) - if llama_proxy is None: - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Service is not available", - ) + + body_model = body.model exclude = { "n", "logit_bias_type", @@ -488,44 +491,9 @@ async def create_chat_completion( "min_tokens", } kwargs = body.model_dump(exclude=exclude) - llama = llama_proxy(body.model) - if body.logit_bias is not None: - kwargs["logit_bias"] = ( - _logit_bias_tokens_to_input_ids(llama, body.logit_bias) - if body.logit_bias_type == "tokens" - else body.logit_bias - ) - - if body.grammar is not None: - kwargs["grammar"] = llama_cpp.LlamaGrammar.from_string(body.grammar) - - if body.min_tokens > 0: - _min_tokens_logits_processor = llama_cpp.LogitsProcessorList( - [llama_cpp.MinTokensLogitsProcessor(body.min_tokens, llama.token_eos())] - ) - if "logits_processor" not in kwargs: - kwargs["logits_processor"] = _min_tokens_logits_processor - else: - kwargs["logits_processor"].extend(_min_tokens_logits_processor) - - try: - iterator_or_completion: Union[ - llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk] - ] = await run_in_threadpool(llama.create_chat_completion, **kwargs) - except Exception as err: - await exit_stack.aclose() - raise err - - if isinstance(iterator_or_completion, Iterator): - # EAFP: It's easier to ask for forgiveness than permission - first_response = await run_in_threadpool(next, iterator_or_completion) - - # If no exception was raised from first_response, we can assume that - # the iterator is valid and we can use it to stream the response. - def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: - yield first_response - yield from iterator_or_completion + # handle streaming request + if kwargs.get("stream", False): send_chan, recv_chan = anyio.create_memory_object_stream(10) return EventSourceResponse( recv_chan, @@ -533,15 +501,29 @@ def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]: get_event_publisher, request=request, inner_send_chan=send_chan, - iterator=iterator(), - on_complete=exit_stack.aclose, + body=body, + body_model=body_model, + llama_call=llama_cpp.Llama.create_chat_completion, + kwargs=kwargs, ), sep="\n", ping_message_factory=_ping_message_factory, ) - else: - await exit_stack.aclose() - return iterator_or_completion + + # handle regular request + async with contextlib.asynccontextmanager(get_llama_proxy)() as llama_proxy: + llama = prepare_request_resources(body, llama_proxy, body_model, kwargs) + + if await request.is_disconnected(): + print( + f"Disconnected from client (via refresh/close) before llm invoked {request.client}" + ) + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Client closed request", + ) + + return await run_in_threadpool(llama.create_chat_completion, **kwargs) @router.get( From 0580cf273debf4a7f2efcdfd5ef092ff5cedf9b0 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 8 Jan 2025 16:53:54 -0500 Subject: [PATCH 392/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f63558933..b4f49c9045 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.6] + +- feat: Update llama.cpp to ggerganov/llama.cpp@f7cd13301c2a88f97073fd119072b4cc92c08df1 +- fix(server): streaming resource lock by @gjpower in #1879 + ## [0.3.5] - feat: Update llama.cpp to ggerganov/llama.cpp@26a8406ba9198eb6fdd8329fa717555b4f77f05f diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index d7d9be881a..8c6118fb42 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.5" +__version__ = "0.3.6" From 80be68ac36b458e02f5e6af8bddd005a2f680e39 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 28 Jan 2025 21:38:05 -0500 Subject: [PATCH 393/561] feat: Update llama.cpp --- llama_cpp/_internals.py | 181 +++++----- llama_cpp/llama.py | 16 +- llama_cpp/llama_cpp.py | 707 +++++++++++++++++++++++++++++----------- vendor/llama.cpp | 2 +- 4 files changed, 626 insertions(+), 280 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 8fa2b447ff..343581dce0 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -55,7 +55,13 @@ def __init__( if model is None: raise ValueError(f"Failed to load model from file: {path_model}") + vocab = llama_cpp.llama_model_get_vocab(model) + + if vocab is None: + raise ValueError(f"Failed to get vocab from model: {path_model}") + self.model = model + self.vocab = vocab def free_model(): if self.model is None: @@ -75,7 +81,7 @@ def vocab_type(self) -> int: return llama_cpp.llama_vocab_type(self.model) def n_vocab(self) -> int: - return llama_cpp.llama_n_vocab(self.model) + return llama_cpp.llama_n_vocab(self.vocab) def n_ctx_train(self) -> int: return llama_cpp.llama_n_ctx_train(self.model) @@ -84,7 +90,7 @@ def n_embd(self) -> int: return llama_cpp.llama_n_embd(self.model) def rope_freq_scale_train(self) -> float: - return llama_cpp.llama_rope_freq_scale_train(self.model) + return llama_cpp.llama_model_rope_freq_scale_train(self.model) def desc(self) -> str: buf = ctypes.create_string_buffer(1024) @@ -98,53 +104,53 @@ def n_params(self) -> int: return llama_cpp.llama_model_n_params(self.model) def get_tensor(self, name: str) -> ctypes.c_void_p: - return llama_cpp.llama_get_model_tensor(self.model, name.encode("utf-8")) + raise NotImplementedError("get_tensor is not implemented in llama.cpp") # Vocab def token_get_text(self, token: int) -> str: - return llama_cpp.llama_token_get_text(self.model, token).decode("utf-8") + return llama_cpp.llama_token_get_text(self.vocab, token).decode("utf-8") def token_get_score(self, token: int) -> float: - return llama_cpp.llama_token_get_score(self.model, token) + return llama_cpp.llama_token_get_score(self.vocab, token) def token_get_attr(self, token: int) -> int: - return llama_cpp.llama_token_get_attr(self.model, token) + return llama_cpp.llama_token_get_attr(self.vocab, token) # Special tokens def token_bos(self) -> int: - return llama_cpp.llama_token_bos(self.model) + return llama_cpp.llama_token_bos(self.vocab) def token_eos(self) -> int: - return llama_cpp.llama_token_eos(self.model) + return llama_cpp.llama_token_eos(self.vocab) def token_cls(self) -> int: - return llama_cpp.llama_token_cls(self.model) + return llama_cpp.llama_token_cls(self.vocab) def token_sep(self) -> int: - return llama_cpp.llama_token_sep(self.model) + return llama_cpp.llama_token_sep(self.vocab) def token_nl(self) -> int: - return llama_cpp.llama_token_nl(self.model) + return llama_cpp.llama_token_nl(self.vocab) def token_prefix(self) -> int: - return llama_cpp.llama_token_prefix(self.model) + raise NotImplementedError("token_prefix is not implemented in llama.cpp") def token_middle(self) -> int: - return llama_cpp.llama_token_middle(self.model) + raise NotImplementedError("token_middle is not implemented in llama.cpp") def token_suffix(self) -> int: - return llama_cpp.llama_token_suffix(self.model) + raise NotImplementedError("token_suffix is not implemented in llama.cpp") def token_eot(self) -> int: - return llama_cpp.llama_token_eot(self.model) + return llama_cpp.llama_token_eot(self.vocab) def add_bos_token(self) -> bool: - return llama_cpp.llama_add_bos_token(self.model) + return llama_cpp.llama_add_bos_token(self.vocab) def add_eos_token(self) -> bool: - return llama_cpp.llama_add_eos_token(self.model) + return llama_cpp.llama_add_eos_token(self.vocab) # Tokenization @@ -152,13 +158,13 @@ def tokenize(self, text: bytes, add_bos: bool, special: bool): n_ctx = self.n_ctx_train() tokens = (llama_cpp.llama_token * n_ctx)() n_tokens = llama_cpp.llama_tokenize( - self.model, text, len(text), tokens, n_ctx, add_bos, special + self.vocab, text, len(text), tokens, n_ctx, add_bos, special ) if n_tokens < 0: n_tokens = abs(n_tokens) tokens = (llama_cpp.llama_token * n_tokens)() n_tokens = llama_cpp.llama_tokenize( - self.model, text, len(text), tokens, n_tokens, add_bos, special + self.vocab, text, len(text), tokens, n_tokens, add_bos, special ) if n_tokens < 0: raise RuntimeError( @@ -168,7 +174,7 @@ def tokenize(self, text: bytes, add_bos: bool, special: bool): def token_to_piece(self, token: int, special: bool = False) -> bytes: buf = ctypes.create_string_buffer(32) - llama_cpp.llama_token_to_piece(self.model, token, buf, 32, 0, special) + llama_cpp.llama_token_to_piece(self.vocab, token, buf, 32, 0, special) return bytes(buf) def detokenize(self, tokens: List[int], special: bool = False) -> bytes: @@ -177,7 +183,7 @@ def detokenize(self, tokens: List[int], special: bool = False) -> bytes: buffer = (ctypes.c_char * size)() for token in tokens: n = llama_cpp.llama_token_to_piece( - self.model, llama_cpp.llama_token(token), buffer, size, 0, special + self.vocab, llama_cpp.llama_token(token), buffer, size, 0, special ) assert n <= size output += bytes(buffer[:n]) @@ -320,7 +326,8 @@ def get_embeddings(self): def set_rng_seed(self, seed: int): # TODO: Fix - llama_cpp.llama_set_rng_seed(self.ctx, seed) + # llama_cpp.llama_set_rng_seed(self.ctx, seed) + raise NotImplementedError("set_rng_seed is not implemented in llama.cpp") def sample_repetition_penalties( self, @@ -331,55 +338,63 @@ def sample_repetition_penalties( penalty_freq: float, penalty_present: float, ): - llama_cpp.llama_sample_repetition_penalties( - self.ctx, - llama_cpp.byref(candidates.candidates), - last_tokens_data, - penalty_last_n, - penalty_repeat, - penalty_freq, - penalty_present, - ) + # llama_cpp.llama_sample_repetition_penalties( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # last_tokens_data, + # penalty_last_n, + # penalty_repeat, + # penalty_freq, + # penalty_present, + # ) + raise NotImplementedError("sample_repetition_penalties is not implemented in llama.cpp") def sample_softmax(self, candidates: "_LlamaTokenDataArray"): - llama_cpp.llama_sample_softmax( - self.ctx, - llama_cpp.byref(candidates.candidates), - ) + # llama_cpp.llama_sample_softmax( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # ) + raise NotImplementedError("sample_softmax is not implemented in llama.cpp") def sample_top_k(self, candidates: "_LlamaTokenDataArray", k: int, min_keep: int): - llama_cpp.llama_sample_top_k( - self.ctx, llama_cpp.byref(candidates.candidates), k, min_keep - ) + # llama_cpp.llama_sample_top_k( + # self.ctx, llama_cpp.byref(candidates.candidates), k, min_keep + # ) + raise NotImplementedError("sample_top_k is not implemented in llama.cpp") def sample_top_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - llama_cpp.llama_sample_top_p( - self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep - ) + # llama_cpp.llama_sample_top_p( + # self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep + # ) + raise NotImplementedError("sample_top_p is not implemented in llama.cpp") def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - llama_cpp.llama_sample_min_p( - self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep - ) + # llama_cpp.llama_sample_min_p( + # self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep + # ) + raise NotImplementedError("sample_min_p is not implemented in llama.cpp") def sample_typical( self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int ): - llama_cpp.llama_sample_typical( - self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep - ) + # llama_cpp.llama_sample_typical( + # self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep + # ) + raise NotImplementedError("sample_typical is not implemented in llama.cpp") def sample_temp(self, candidates: "_LlamaTokenDataArray", temp: float): - llama_cpp.llama_sample_temp( - self.ctx, llama_cpp.byref(candidates.candidates), temp - ) + # llama_cpp.llama_sample_temp( + # self.ctx, llama_cpp.byref(candidates.candidates), temp + # ) + raise NotImplementedError("sample_temp is not implemented in llama.cpp") def sample_grammar(self, candidates: "_LlamaTokenDataArray", grammar: LlamaGrammar): - llama_cpp.llama_sample_grammar( - self.ctx, - llama_cpp.byref(candidates.candidates), - grammar.grammar, - ) + # llama_cpp.llama_sample_grammar( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # grammar.grammar, + # ) + raise NotImplementedError("sample_grammar is not implemented in llama.cpp") def sample_token_mirostat( self, @@ -389,14 +404,15 @@ def sample_token_mirostat( m: int, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - return llama_cpp.llama_sample_token_mirostat( - self.ctx, - llama_cpp.byref(candidates.candidates), - tau, - eta, - m, - mu, - ) + raise NotImplementedError("sample_token_mirostat is not implemented in llama.cpp") + # return llama_cpp.llama_sample_token_mirostat( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # tau, + # eta, + # m, + # mu, + # ) def sample_token_mirostat_v2( self, @@ -405,29 +421,33 @@ def sample_token_mirostat_v2( eta: float, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - return llama_cpp.llama_sample_token_mirostat_v2( - self.ctx, - llama_cpp.byref(candidates.candidates), - tau, - eta, - mu, - ) + raise NotImplementedError("sample_token_mirostat_v2 is not implemented in llama.cpp") + # return llama_cpp.llama_sample_token_mirostat_v2( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # tau, + # eta, + # mu, + # ) def sample_token_greedy(self, candidates: "_LlamaTokenDataArray") -> int: - return llama_cpp.llama_sample_token_greedy( - self.ctx, - llama_cpp.byref(candidates.candidates), - ) + raise NotImplementedError("sample_token_greedy is not implemented in llama.cpp") + # return llama_cpp.llama_sample_token_greedy( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # ) def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: - return llama_cpp.llama_sample_token( - self.ctx, - llama_cpp.byref(candidates.candidates), - ) + raise NotImplementedError("sample_token is not implemented in llama.cpp") + # return llama_cpp.llama_sample_token( + # self.ctx, + # llama_cpp.byref(candidates.candidates), + # ) # Grammar def grammar_accept_token(self, grammar: LlamaGrammar, token: int): - llama_cpp.llama_grammar_accept_token(grammar.grammar, self.ctx, token) + raise NotImplementedError("grammar_accept_token is not implemented in llama.cpp") + # llama_cpp.llama_grammar_accept_token(grammar.grammar, self.ctx, token) def reset_timings(self): llama_cpp.llama_perf_context_reset(self.ctx) @@ -788,7 +808,7 @@ def add_mirostat_v2(self, seed: int, tau: float, eta: float): def add_grammar(self, model: LlamaModel, grammar: LlamaGrammar): sampler = llama_cpp.llama_sampler_init_grammar( - model.model, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8") + model.vocab, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8") ) self._add_sampler(sampler) @@ -842,6 +862,7 @@ def get_seed(self) -> int: def sample(self, ctx: LlamaContext, idx: int) -> int: assert self.sampler is not None + assert ctx.ctx is not None return llama_cpp.llama_sampler_sample(self.sampler, ctx.ctx, idx) def close(self): diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2fd7ff1939..11fc744a01 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -406,10 +406,10 @@ def __init__( ) ) - self._lora_adapter: Optional[llama_cpp.llama_lora_adapter_p] = None + self._lora_adapter: Optional[llama_cpp.llama_adapter_lora_p] = None if self.lora_path: - self._lora_adapter = llama_cpp.llama_lora_adapter_init( + self._lora_adapter = llama_cpp.llama_adapter_lora_init( self._model.model, self.lora_path.encode("utf-8"), ) @@ -421,12 +421,12 @@ def __init__( def free_lora_adapter(): if self._lora_adapter is None: return - llama_cpp.llama_lora_adapter_free(self._lora_adapter) + llama_cpp.llama_adapter_lora_free(self._lora_adapter) self._lora_adapter = None self._stack.callback(free_lora_adapter) - if llama_cpp.llama_lora_adapter_set( + if llama_cpp.llama_set_adapter_lora( self._ctx.ctx, self._lora_adapter, self.lora_scale ): raise RuntimeError( @@ -1152,9 +1152,9 @@ def _create_completion( bos_token_id: int = self.token_bos() cls_token_id: int = self._model.token_cls() sep_token_id: int = self._model.token_sep() - prefix_token_id: int = self._model.token_prefix() - middle_token_id: int = self._model.token_middle() - suffix_token_id: int = self._model.token_suffix() + prefix_token_id: int = 0 # self._model.token_prefix() # TODO: Fix + middle_token_id: int = 0 # self._model.token_middle() # TODO: Fix + suffix_token_id: int = 0 # self._model.token_suffix() # TODO: Fix add_space_prefix: bool = ( self.metadata.get("tokenizer.ggml.add_space_prefix", "true") == "true" ) @@ -1332,7 +1332,7 @@ def logit_bias_processor( logits_processor=logits_processor, grammar=grammar, ): - if llama_cpp.llama_token_is_eog(self._model.model, token): + if llama_cpp.llama_token_is_eog(self._model.vocab, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) finish_reason = "stop" break diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 16c6b9d733..7845f0d1cf 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -149,6 +149,10 @@ # define LLAMA_STATE_SEQ_VERSION 2 LLAMA_STATE_SEQ_VERSION = 2 +# struct llama_vocab; +llama_vocab_p = NewType("llama_vocab_p", int) +llama_vocab_p_ctypes = ctypes.c_void_p + # struct llama_model; llama_model_p = NewType("llama_model_p", int) llama_model_p_ctypes = ctypes.c_void_p @@ -640,9 +644,6 @@ class llama_model_kv_override(ctypes.Structure): # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() # const float * tensor_split; -# // comma separated list of RPC servers to use for offloading -# const char * rpc_servers; - # // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. # // If the provided progress_callback returns true, model loading continues. # // If it returns false, model loading is immediately aborted. @@ -669,7 +670,6 @@ class llama_model_params(ctypes.Structure): split_mode (int): how to split the model across multiple GPUs main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() - rpc_servers (ctypes.c_char_p): comma separated list of RPC servers to use for offloading progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data @@ -683,7 +683,6 @@ class llama_model_params(ctypes.Structure): split_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] - rpc_servers: ctypes.c_char_p progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] @@ -698,7 +697,6 @@ class llama_model_params(ctypes.Structure): ("split_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), - ("rpc_servers", ctypes.c_char_p), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), @@ -975,9 +973,9 @@ class llama_chat_message(ctypes.Structure): # // lora adapter -# struct llama_lora_adapter; -llama_lora_adapter_p = ctypes.c_void_p -llama_lora_adapter_p_ctypes = ctypes.POINTER(ctypes.c_void_p) +# struct llama_adapter_lora; +llama_adapter_lora_p = ctypes.c_void_p +llama_adapter_lora_p_ctypes = ctypes.POINTER(ctypes.c_void_p) # // Helpers for getting default parameters @@ -1059,6 +1057,18 @@ def llama_backend_init(): GGML_NUMA_STRATEGY_COUNT = 5 +# // Call once at the end of the program - currently only used for MPI +# LLAMA_API void llama_backend_free(void); +@ctypes_function( + "llama_backend_free", + [], + None, +) +def llama_backend_free(): + """Call once at the end of the program - currently only used for MPI""" + ... + + # //optional: # LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa); @ctypes_function( @@ -1072,24 +1082,14 @@ def llama_numa_init(numa: int, /): # // Optional: an auto threadpool gets created in ggml if not passed explicitly # LLAMA_API void llama_attach_threadpool( -# struct llama_context * ctx, -# ggml_threadpool_t threadpool, -# ggml_threadpool_t threadpool_batch); +# struct llama_context * ctx, +# ggml_threadpool_t threadpool, +# ggml_threadpool_t threadpool_batch); +# TODO: Add llama_attach_threadpool # LLAMA_API void llama_detach_threadpool(struct llama_context * ctx); - - -# // Call once at the end of the program - currently only used for MPI -# LLAMA_API void llama_backend_free(void); -@ctypes_function( - "llama_backend_free", - [], - None, -) -def llama_backend_free(): - """Call once at the end of the program - currently only used for MPI""" - ... +# TODO: Add llama_detach_threadpool # DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( @@ -1107,6 +1107,9 @@ def llama_load_model_from_file( ... +# // Load the model from a file +# // If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf +# // If the split file name does not follow this pattern, use llama_model_load_from_splits # LLAMA_API struct llama_model * llama_model_load_from_file( # const char * path_model, # struct llama_model_params params); @@ -1118,6 +1121,31 @@ def llama_load_model_from_file( def llama_model_load_from_file( path_model: bytes, params: llama_model_params, / ) -> Optional[llama_model_p]: + """Load the model from a file + + If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf + + If the split file name does not follow this pattern, use llama_model_load_from_splits""" + ... + + +# // Load the model from multiple splits (support custom naming scheme) +# // The paths must be in the correct order +# LLAMA_API struct llama_model * llama_model_load_from_splits( +# const char ** paths, +# size_t n_paths, +# struct llama_model_params params); +@ctypes_function( + "llama_model_load_from_splits", + [ctypes.POINTER(ctypes.c_char_p), ctypes.c_size_t, llama_model_params], + llama_model_p_ctypes, +) +def llama_model_load_from_splits( + paths: List[bytes], n_paths: int, params: llama_model_params, / +) -> Optional[llama_model_p]: + """Load the model from multiple splits (support custom naming scheme) + + The paths must be in the correct order""" ... @@ -1141,9 +1169,24 @@ def llama_model_free(model: llama_model_p, /): ... -# LLAMA_API struct llama_context * llama_new_context_with_model( +# LLAMA_API struct llama_context * llama_init_from_model( # struct llama_model * model, # struct llama_context_params params); +@ctypes_function( + "llama_init_from_model", + [llama_model_p_ctypes, llama_context_params], + llama_context_p_ctypes, +) +def llama_init_from_model( + model: llama_model_p, params: llama_context_params, / +) -> Optional[llama_context_p]: + ... + + +# DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model( +# struct llama_model * model, +# struct llama_context_params params), +# "use llama_init_from_model instead"); @ctypes_function( "llama_new_context_with_model", [llama_model_p_ctypes, llama_context_params], @@ -1231,65 +1274,102 @@ def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... -# LLAMA_API int32_t llama_n_vocab (const struct llama_model * model); -@ctypes_function("llama_n_vocab", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_model_p, /) -> int: - ... -# LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model); +# DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_ctx_train(model: llama_model_p, /) -> int: ... -# LLAMA_API int32_t llama_n_embd (const struct llama_model * model); +# DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_embd(model: llama_model_p, /) -> int: ... -# LLAMA_API int32_t llama_n_layer (const struct llama_model * model); +# DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_layer(model: llama_model_p, /) -> int: ... -# LLAMA_API int32_t llama_n_head (const struct llama_model * model); +# DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); @ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_head(model: llama_model_p, /) -> int: ... -# LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx); +# DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); +@ctypes_function("llama_n_vocab", [llama_vocab_p_ctypes], ctypes.c_int32) +def llama_n_vocab(model: llama_vocab_p, /) -> int: + ... + + +# LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... -# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); +# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) def llama_pooling_type(ctx: llama_context_p, /) -> int: ... -# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); -@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: +# LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); +@ctypes_function("llama_model_get_vocab", [llama_model_p_ctypes], llama_vocab_p_ctypes) +def llama_model_get_vocab(model: llama_model_p, /) -> Optional[llama_vocab_p]: + ... + + +# LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model); +@ctypes_function("llama_model_rope_type", [llama_model_p_ctypes], ctypes.c_int) +def llama_model_rope_type(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model); +@ctypes_function("llama_model_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_ctx_train(model: llama_model_p, /) -> int: ... -# LLAMA_API enum llama_rope_type llama_rope_type (const struct llama_model * model); -@ctypes_function("llama_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_rope_type(model: llama_model_p, /) -> int: +# LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model); +@ctypes_function("llama_model_n_embd", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_embd(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model); +@ctypes_function("llama_model_n_layer", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_layer(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); +@ctypes_function("llama_model_n_head", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_head(model: llama_model_p, /) -> int: ... # // Get the model's RoPE frequency scaling factor -# LLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model); -@ctypes_function("llama_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float) -def llama_rope_freq_scale_train(model: llama_model_p, /) -> float: - """Get the model's RoPE frequency scaling factor""" +# LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); +@ctypes_function("llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float) +def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: + ... + + +# LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); +@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) +def llama_vocab_type(model: llama_model_p, /) -> int: + ... + + +# LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab); +@ctypes_function("llama_vocab_n_tokens", [llama_vocab_p_ctypes], ctypes.c_int32) +def llama_vocab_n_tokens(vocab: llama_vocab_p, /) -> int: ... @@ -1402,6 +1482,14 @@ def llama_model_size(model: llama_model_p, /) -> int: ... +# // Get the default chat template. Returns nullptr if not available +# LLAMA_API const char * llama_model_chat_template(const struct llama_model * model); +@ctypes_function("llama_model_chat_template", [llama_model_p_ctypes], ctypes.c_char_p) +def llama_model_chat_template(model: llama_model_p, /) -> Optional[bytes]: + """Get the default chat template. Returns None if not available""" + ... + + # // Returns the total number of parameters in the model # LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model); @ctypes_function("llama_model_n_params", [llama_model_p_ctypes], ctypes.c_uint64) @@ -1472,37 +1560,48 @@ def llama_model_quantize( # // Load a LoRA adapter from file -# // The loaded adapter will be associated to the given model, and will be free when the model is deleted -# LLAMA_API struct llama_lora_adapter * llama_lora_adapter_init( +# LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init( # struct llama_model * model, # const char * path_lora); @ctypes_function( - "llama_lora_adapter_init", + "llama_adapter_lora_init", [llama_model_p_ctypes, ctypes.c_char_p], - llama_lora_adapter_p_ctypes, + llama_adapter_lora_p_ctypes, ) -def llama_lora_adapter_init( +def llama_adapter_lora_init( model: llama_model_p, path_lora: bytes, / -) -> Optional[llama_lora_adapter_p]: - """Load a LoRA adapter from file - The loaded adapter will be associated to the given model, and will be free when the model is deleted - """ +) -> Optional[llama_adapter_lora_p]: + ... + + +# // Manually free a LoRA adapter +# // Note: loaded adapters will be free when the associated model is deleted +# LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter); +@ctypes_function( + "llama_adapter_lora_free", + [llama_adapter_lora_p_ctypes], + None, +) +def llama_adapter_lora_free(adapter: llama_adapter_lora_p, /): ... +# // The following functions operate on a llama_context, hence the naming: llama_verb_... + + # // Add a loaded LoRA adapter to given context # // This will not modify model's weight -# LLAMA_API int32_t llama_lora_adapter_set( +# LLAMA_API int32_t llama_set_adapter_lora( # struct llama_context * ctx, -# struct llama_lora_adapter * adapter, +# struct llama_adapter_lora * adapter, # float scale); @ctypes_function( - "llama_lora_adapter_set", - [llama_context_p_ctypes, llama_lora_adapter_p_ctypes, ctypes.c_float], + "llama_set_adapter_lora", + [llama_context_p_ctypes, llama_adapter_lora_p_ctypes, ctypes.c_float], ctypes.c_int32, ) -def llama_lora_adapter_set( - ctx: llama_context_p, adapter: llama_lora_adapter_p, scale: float, / +def llama_set_adapter_lora( + ctx: llama_context_p, adapter: llama_adapter_lora_p, scale: float, / ) -> int: """Add a loaded LoRA adapter to given context This will not modify model's weight""" @@ -1511,64 +1610,49 @@ def llama_lora_adapter_set( # // Remove a specific LoRA adapter from given context # // Return -1 if the adapter is not present in the context -# LLAMA_API int32_t llama_lora_adapter_remove( +# LLAMA_API int32_t llama_rm_adapter_lora( # struct llama_context * ctx, -# struct llama_lora_adapter * adapter); +# struct llama_adapter_lora * adapter); @ctypes_function( - "llama_lora_adapter_remove", - [llama_context_p_ctypes, llama_lora_adapter_p_ctypes], + "llama_rm_adapter_lora", + [llama_context_p_ctypes, llama_adapter_lora_p_ctypes], ctypes.c_int32, ) -def llama_lora_adapter_remove( - ctx: llama_context_p, adapter: llama_lora_adapter_p, / +def llama_rm_adapter_lora( + ctx: llama_context_p, adapter: llama_adapter_lora_p, / ) -> int: - """Remove a LoRA adapter from given context + """Remove a specific LoRA adapter from given context Return -1 if the adapter is not present in the context""" ... # // Remove all LoRA adapters from given context -# LLAMA_API void llama_lora_adapter_clear( -# struct llama_context * ctx); +# LLAMA_API void llama_clear_adapter_lora(struct llama_context * ctx); @ctypes_function( - "llama_lora_adapter_clear", + "llama_clear_adapter_lora", [llama_context_p_ctypes], None, ) -def llama_lora_adapter_clear(ctx: llama_context_p, /): +def llama_clear_adapter_lora(ctx: llama_context_p, /): """Remove all LoRA adapters from given context""" ... -# // Manually free a LoRA adapter -# // Note: loaded adapters will be free when the associated model is deleted -# LLAMA_API void llama_lora_adapter_free(struct llama_lora_adapter * adapter); -@ctypes_function( - "llama_lora_adapter_free", - [llama_lora_adapter_p_ctypes], - None, -) -def llama_lora_adapter_free(adapter: llama_lora_adapter_p, /): - """Manually free a LoRA adapter - Note: loaded adapters will be free when the associated model is deleted""" - ... - - # // Apply a loaded control vector to a llama_context, or if data is NULL, clear # // the currently loaded vector. # // n_embd should be the size of a single layer's control, and data should point # // to an n_embd x n_layers buffer starting from layer 1. # // il_start and il_end are the layer range the vector should apply to (both inclusive) # // See llama_control_vector_load in common to load a control vector. -# LLAMA_API int32_t llama_control_vector_apply( -# struct llama_context * lctx, +# LLAMA_API int32_t llama_apply_adapter_cvec( +# struct llama_context * ctx, # const float * data, # size_t len, # int32_t n_embd, # int32_t il_start, # int32_t il_end); @ctypes_function( - "llama_control_vector_apply", + "llama_apply_adapter_cvec", [ llama_context_p_ctypes, ctypes.POINTER(ctypes.c_float), @@ -1579,8 +1663,8 @@ def llama_lora_adapter_free(adapter: llama_lora_adapter_p, /): ], ctypes.c_int32, ) -def llama_control_vector_apply( - lctx: llama_context_p, +def llama_apply_adapter_cvec( + ctx: llama_context_p, data: CtypesPointerOrRef[ctypes.c_float], len: int, n_embd: int, @@ -2577,53 +2661,53 @@ def llama_get_embeddings_seq( # // -# LLAMA_API const char * llama_token_get_text(const struct llama_model * model, llama_token token); +# LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_get_text", [llama_model_p_ctypes, llama_token], ctypes.c_char_p + "llama_vocab_get_text", [llama_vocab_p_ctypes, llama_token], ctypes.c_char_p ) -def llama_token_get_text( - model: llama_model_p, token: Union[llama_token, int], / +def llama_vocab_get_text( + vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> bytes: ... -# LLAMA_API float llama_token_get_score(const struct llama_model * model, llama_token token); +# LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_get_score", [llama_model_p_ctypes, llama_token], ctypes.c_float + "llama_vocab_get_score", [llama_vocab_p_ctypes, llama_token], ctypes.c_float ) -def llama_token_get_score( - model: llama_model_p, token: Union[llama_token, int], / +def llama_vocab_get_score( + vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> float: ... -# LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_model * model, llama_token token); +# LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_get_attr", [llama_model_p_ctypes, llama_token], ctypes.c_int + "llama_vocab_get_attr", [llama_vocab_p_ctypes, llama_token], ctypes.c_int ) -def llama_token_get_attr( - model: llama_model_p, token: Union[llama_token, int], / +def llama_vocab_get_attr( + vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> int: ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) -# LLAMA_API bool llama_token_is_eog(const struct llama_model * model, llama_token token); +# LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_is_eog", [llama_model_p_ctypes, llama_token], ctypes.c_bool + "llama_vocab_is_eog", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool ) -def llama_token_is_eog(model: llama_model_p, token: Union[llama_token, int], /) -> bool: +def llama_vocab_is_eog(vocab: llama_vocab_p, token: Union[llama_token, int], /) -> bool: """Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)""" ... # // Identify if Token Id is a control token or a render-able token -# LLAMA_API bool llama_token_is_control(const struct llama_model * model, llama_token token); +# LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token); @ctypes_function( - "llama_token_is_control", [llama_model_p_ctypes, llama_token], ctypes.c_bool + "llama_vocab_is_control", [llama_vocab_p_ctypes, llama_token], ctypes.c_bool ) -def llama_token_is_control( - model: llama_model_p, token: Union[llama_token, int], / +def llama_vocab_is_control( + vocab: llama_vocab_p, token: Union[llama_token, int], / ) -> bool: """Identify if Token Id is a control token or a render-able token""" ... @@ -2632,110 +2716,335 @@ def llama_token_is_control( # // Special tokens -# LLAMA_API llama_token llama_token_bos(const struct llama_model * model); // beginning-of-sentence -@ctypes_function("llama_token_bos", [llama_model_p_ctypes], llama_token) -def llama_token_bos(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence +@ctypes_function("llama_vocab_bos", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_bos(vocab: llama_vocab_p, /) -> llama_token: """beginning-of-sentence""" ... -# LLAMA_API llama_token llama_token_eos(const struct llama_model * model); // end-of-sentence -@ctypes_function("llama_token_eos", [llama_model_p_ctypes], llama_token) -def llama_token_eos(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence +@ctypes_function("llama_vocab_eos", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_eos(vocab: llama_vocab_p, /) -> llama_token: """end-of-sentence""" ... -# LLAMA_API llama_token llama_token_eot(const struct llama_model * model); // end-of-turn -@ctypes_function("llama_token_eot", [llama_model_p_ctypes], llama_token) -def llama_token_eot(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn +@ctypes_function("llama_vocab_eot", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_eot(vocab: llama_vocab_p, /) -> llama_token: """end-of-turn""" ... -# LLAMA_API llama_token llama_token_cls(const struct llama_model * model); // classification -@ctypes_function("llama_token_cls", [llama_model_p_ctypes], llama_token) -def llama_token_cls(model: llama_model_p, /) -> int: - """classification""" +# LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator +@ctypes_function("llama_vocab_sep", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_sep(vocab: llama_vocab_p, /) -> llama_token: + """sentence separator""" ... -# LLAMA_API llama_token llama_token_sep(const struct llama_model * model); // sentence separator -@ctypes_function("llama_token_sep", [llama_model_p_ctypes], llama_token) -def llama_token_sep(model: llama_model_p, /) -> int: - """sentence separator""" +# LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line +@ctypes_function("llama_vocab_nl", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_nl(vocab: llama_vocab_p, /) -> llama_token: + """next-line""" ... -# LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line -@ctypes_function("llama_token_nl", [llama_model_p_ctypes], llama_token) -def llama_token_nl(model: llama_model_p, /) -> int: - """next-line""" +# LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding +@ctypes_function("llama_vocab_pad", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_pad(vocab: llama_vocab_p, /) -> llama_token: + """padding""" + ... + +# LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_get_add_bos", + [llama_vocab_p_ctypes], + ctypes.c_bool, +) +def llama_vocab_get_add_bos(vocab: llama_vocab_p, /) -> bool: ... -# LLAMA_API bool llama_add_bos_token(const struct llama_model * model); -@ctypes_function("llama_add_bos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_bos_token(model: llama_model_p, /) -> bool: +# LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_get_add_eos", + [llama_vocab_p_ctypes], + ctypes.c_bool, +) +def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: ... -# LLAMA_API bool llama_add_eos_token(const struct llama_model * model); -@ctypes_function("llama_add_eos_token", [llama_model_p_ctypes], ctypes.c_bool) -def llama_add_eos_token(model: llama_model_p, /) -> bool: +# LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_pre", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ... -# // Codellama infill tokens -# DEPRECATED(LLAMA_API llama_token llama_token_prefix(const struct llama_model * model), "use llama_token_fim_pre instead"); -@ctypes_function("llama_token_prefix", [llama_model_p_ctypes], llama_token) -def llama_token_prefix(model: llama_model_p) -> int: - """codellama infill tokens""" +# LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_suf", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ... -# DEPRECATED(LLAMA_API llama_token llama_token_middle(const struct llama_model * model), "use llama_token_fim_mid instead"); -@ctypes_function("llama_token_middle", [llama_model_p_ctypes], llama_token) -def llama_token_middle(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_mid", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ... -# DEPRECATED(LLAMA_API llama_token llama_token_suffix(const struct llama_model * model), "use llama_token_fim_suf instead"); -@ctypes_function("llama_token_suffix", [llama_model_p_ctypes], llama_token) -def llama_token_suffix(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_pad", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_pre(const struct llama_model * model); -@ctypes_function("llama_token_fim_pre", [llama_model_p_ctypes], llama_token) -def llama_token_fim_pre(model: llama_model_p, /) -> int: +# LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_rep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_rep(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_suf(const struct llama_model * model); -@ctypes_function("llama_token_fim_suf", [llama_model_p_ctypes], llama_token) -def llama_token_fim_suf(model: llama_model_p, /) -> int: + +# LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_fim_sep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_mid(const struct llama_model * model); -@ctypes_function("llama_token_fim_mid", [llama_model_p_ctypes], llama_token) -def llama_token_fim_mid(model: llama_model_p, /) -> int: + + +# DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead"); +@ctypes_function( + "llama_token_get_text", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_char_p, +) +def llama_token_get_text( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> bytes: + ... + + +# DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); +@ctypes_function( + "llama_token_get_score", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_float, +) +def llama_token_get_score( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> float: + ... + +# DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); +@ctypes_function( + "llama_token_get_attr", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_int, +) +def llama_token_get_attr( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> int: + ... + +# DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); +@ctypes_function( + "llama_token_is_eog", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_bool, +) +def llama_token_is_eog( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> bool: + ... + +# DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); +@ctypes_function( + "llama_token_is_control", + [llama_vocab_p_ctypes, llama_token], + ctypes.c_bool, +) +def llama_token_is_control( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> bool: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); +@ctypes_function( + "llama_token_bos", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_bos(vocab: llama_vocab_p, /) -> int: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); +@ctypes_function( + "llama_token_eos", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_eos(vocab: llama_vocab_p, /) -> int: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); +@ctypes_function( + "llama_token_eot", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_eot(vocab: llama_vocab_p, /) -> int: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); +@ctypes_function( + "llama_token_cls", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_cls(vocab: llama_vocab_p, /) -> int: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); +@ctypes_function( + "llama_token_sep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_sep(vocab: llama_vocab_p, /) -> int: + ... + + +# DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); +@ctypes_function( + "llama_token_nl", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_nl(vocab: llama_vocab_p, /) -> int: + ... + + +# DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); +@ctypes_function( + "llama_token_pad", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_pad(vocab: llama_vocab_p, /) -> int: + ... + + +# DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); +@ctypes_function( + "llama_add_bos_token", + [llama_vocab_p_ctypes], + ctypes.c_bool, +) +def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: + ... + +# DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); +@ctypes_function( + "llama_add_eos_token", + [llama_vocab_p_ctypes], + ctypes.c_bool, +) +def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: + ... + + +# DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); +@ctypes_function( + "llama_token_fim_pre", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_pad(const struct llama_model * model); -@ctypes_function("llama_token_fim_pad", [llama_model_p_ctypes], llama_token) -def llama_token_fim_pad(model: llama_model_p, /) -> int: +# DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); +@ctypes_function( + "llama_token_fim_suf", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_rep(const struct llama_model * model); -@ctypes_function("llama_token_fim_rep", [llama_model_p_ctypes], llama_token) -def llama_token_fim_rep(model: llama_model_p, /) -> int: +# DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); +@ctypes_function( + "llama_token_fim_mid", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ... -# LLAMA_API llama_token llama_token_fim_sep(const struct llama_model * model); -@ctypes_function("llama_token_fim_sep", [llama_model_p_ctypes], llama_token) -def llama_token_fim_sep(model: llama_model_p, /) -> int: +# DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); +@ctypes_function( + "llama_token_fim_pad", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ... +# DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); +@ctypes_function( + "llama_token_fim_rep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: + ... + +# DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); +@ctypes_function( + "llama_token_fim_sep", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: + ... + +# // CLS is equivalent to BOS +# DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification +# "use llama_vocab_bos instead"); +@ctypes_function( + "llama_vocab_cls", + [llama_vocab_p_ctypes], + llama_token, +) +def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: + ... + + # // # // Tokenization # // @@ -2751,7 +3060,7 @@ def llama_token_fim_sep(model: llama_model_p, /) -> int: # /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated # /// as plaintext. Does not insert a leading space. # LLAMA_API int32_t llama_tokenize( -# const struct llama_model * model, +# const struct llama_vocab * vocab, # const char * text, # int32_t text_len, # llama_token * tokens, @@ -2761,7 +3070,7 @@ def llama_token_fim_sep(model: llama_model_p, /) -> int: @ctypes_function( "llama_tokenize", [ - llama_model_p_ctypes, + llama_vocab_p_ctypes, ctypes.c_char_p, ctypes.c_int32, llama_token_p, @@ -2772,7 +3081,7 @@ def llama_token_fim_sep(model: llama_model_p, /) -> int: ctypes.c_int32, ) def llama_tokenize( - model: llama_model_p, + vocab: llama_vocab_p, text: bytes, text_len: Union[ctypes.c_int, int], tokens: CtypesArray[llama_token], @@ -2784,7 +3093,7 @@ def llama_tokenize( """Convert the provided text into tokens. Args: - model: The model to use for tokenization. + vocab: The vocabulary to use for tokenization. text: The text to tokenize. text_len: The length of the text. tokens: The tokens pointer must be large enough to hold the resulting tokens. @@ -2805,7 +3114,7 @@ def llama_tokenize( # // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix') # // @param special If true, special tokens are rendered in the output. # LLAMA_API int32_t llama_token_to_piece( -# const struct llama_model * model, +# const struct llama_vocab * vocab, # llama_token token, # char * buf, # int32_t length, @@ -2814,7 +3123,7 @@ def llama_tokenize( @ctypes_function( "llama_token_to_piece", [ - llama_model_p_ctypes, + llama_vocab_p_ctypes, llama_token, ctypes.c_char_p, ctypes.c_int32, @@ -2824,7 +3133,7 @@ def llama_tokenize( ctypes.c_int32, ) def llama_token_to_piece( - model: llama_model_p, + vocab: llama_vocab_p, token: Union[llama_token, int], buf: Union[ctypes.c_char_p, bytes, CtypesArray[ctypes.c_char]], length: Union[ctypes.c_int, int], @@ -2838,7 +3147,7 @@ def llama_token_to_piece( User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens. Args: - model: The model to use for tokenization. + vocab: The vocabulary to use for tokenization. token: The token to convert. buf: The buffer to write the token to. length: The length of the buffer. @@ -2930,7 +3239,6 @@ def llama_detokenize( # /// @param length The size of the allocated buffer # /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. # LLAMA_API int32_t llama_chat_apply_template( -# const struct llama_model * model, # const char * tmpl, # const struct llama_chat_message * chat, # size_t n_msg, @@ -2940,20 +3248,37 @@ def llama_detokenize( @ctypes_function( "llama_chat_apply_template", [ - ctypes.c_void_p, - ctypes.c_char_p, - ctypes.POINTER(llama_chat_message), - ctypes.c_size_t, + ctypes.c_char_p, # tmpl + ctypes.POINTER(llama_chat_message), # chat + ctypes.c_size_t, # n_msg + ctypes.c_bool, # add_ass (added) + ctypes.c_char_p, # buf + ctypes.c_int32, # length ], ctypes.c_int32, ) def llama_chat_apply_template( - model: llama_model_p, tmpl: bytes, chat: CtypesArray[llama_chat_message], n_msg: int, + add_ass: bool, # Added parameter + buf: bytes, + length: int, /, ) -> int: + """Apply chat template. + + Args: + tmpl: Template to use. If None, uses model's default + chat: Array of chat messages + n_msg: Number of messages + add_ass: Whether to end prompt with assistant token + buf: Output buffer + length: Buffer length + + Returns: + Number of bytes written, or needed if buffer too small + """ ... @@ -3022,7 +3347,6 @@ def llama_chat_builtin_templates( # // llama_sampler_free(smpl); # // # // TODO: In the future, llama_sampler will be utilized to offload the sampling to the backends (e.g. GPU). -# // TODO: in the future, the entire sampling API that uses llama_model should start using llama_vocab # // # typedef void * llama_sampler_context_t; @@ -3342,16 +3666,16 @@ def llama_sampler_init_mirostat_v2( # LLAMA_API struct llama_sampler * llama_sampler_init_grammar( -# const struct llama_model * model, +# const struct llama_vocab * vocab, # const char * grammar_str, # const char * grammar_root); @ctypes_function( "llama_sampler_init_grammar", - [llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_char_p], + [llama_vocab_p_ctypes, ctypes.c_char_p, ctypes.c_char_p], llama_sampler_p_ctypes, ) def llama_sampler_init_grammar( - model: llama_model_p, grammar_str: bytes, grammar_root: bytes, / + vocab: llama_vocab_p, grammar_str: bytes, grammar_root: bytes, / ) -> llama_sampler_p: ... @@ -3379,7 +3703,8 @@ def llama_sampler_init_penalties( # /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 # LLAMA_API struct llama_sampler * llama_sampler_init_dry( -# const struct llama_model * model, +# const struct llama_vocab * vocab, +# int32_t n_ctx_train, # float dry_multiplier, # float dry_base, # int32_t dry_allowed_length, @@ -3389,7 +3714,8 @@ def llama_sampler_init_penalties( @ctypes_function( "llama_sampler_init_dry", [ - llama_model_p_ctypes, + llama_vocab_p_ctypes, + ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_int32, @@ -3400,7 +3726,8 @@ def llama_sampler_init_penalties( llama_sampler_p_ctypes, ) def llama_sampler_init_dry( - model: llama_model_p, + vocab: llama_vocab_p, + n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, @@ -3448,15 +3775,13 @@ def llama_sampler_init_logit_bias( # // 3. discard non-EOG tokens with low prob # // 4. if no tokens are left -> pick EOT # // -# LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_model * model); +# LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab); @ctypes_function( "llama_sampler_init_infill", - [llama_model_p_ctypes], + [llama_vocab_p_ctypes], llama_sampler_p_ctypes, ) -def llama_sampler_init_infill(model: llama_model_p, /) -> llama_sampler_p: - """This sampler is meant to be used for fill-in-the-middle infilling. - """ +def llama_sampler_init_infill(vocab: llama_vocab_p, /) -> llama_sampler_p: ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f7cd13301c..ef6dada60c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f7cd13301c2a88f97073fd119072b4cc92c08df1 +Subproject commit ef6dada60ca710f4edfbb6fd9e1258685d8ea49d From 0b89fe48ad26ffcff76451bd87642d916a1a3385 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 28 Jan 2025 21:40:53 -0500 Subject: [PATCH 394/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 10 ++++++---- vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 7845f0d1cf..2b9edbb99a 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1483,10 +1483,12 @@ def llama_model_size(model: llama_model_p, /) -> int: # // Get the default chat template. Returns nullptr if not available -# LLAMA_API const char * llama_model_chat_template(const struct llama_model * model); -@ctypes_function("llama_model_chat_template", [llama_model_p_ctypes], ctypes.c_char_p) -def llama_model_chat_template(model: llama_model_p, /) -> Optional[bytes]: - """Get the default chat template. Returns None if not available""" +# // If name is NULL, returns the default chat template +# LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name); +@ctypes_function("llama_model_chat_template", [llama_model_p_ctypes, ctypes.c_char_p], ctypes.c_char_p) +def llama_model_chat_template(model: llama_model_p, name: Optional[bytes], /) -> Optional[bytes]: + """Get the default chat template. Returns None if not available + If name is None, returns the default chat template""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ef6dada60c..794fe23f29 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ef6dada60ca710f4edfbb6fd9e1258685d8ea49d +Subproject commit 794fe23f29fb40104975c91fe19f23798f7c726e From 14879c7819eb023a6a272e0f18aca887ae605c0f Mon Sep 17 00:00:00 2001 From: oobabooga Date: Tue, 28 Jan 2025 23:43:29 -0300 Subject: [PATCH 395/561] fix(ci): Fix the CUDA workflow (#1894) --- .github/workflows/build-wheels-cuda.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 3d410148f2..745b2e602d 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -61,11 +61,9 @@ jobs: - name: Setup Mamba uses: conda-incubator/setup-miniconda@v3.1.0 with: - activate-environment: "build" + activate-environment: "llamacpp" python-version: ${{ matrix.pyver }} - miniforge-variant: Mambaforge miniforge-version: latest - use-mamba: true add-pip-as-python-dependency: true auto-activate-base: false From 4442ff8e0a09feafcf7c91045ebbe0ba6f026feb Mon Sep 17 00:00:00 2001 From: Shaka Huang Date: Wed, 29 Jan 2025 11:57:37 +0800 Subject: [PATCH 396/561] fix: error showing time spent in llama perf context print (#1898) * feat: Sync with llama.cpp Add `no_perf` field to `llama_context_params` to optionally disable performance timing measurements. * fix: Display performance metrics by default --------- Co-authored-by: Andrei --- llama_cpp/llama.py | 4 ++++ llama_cpp/llama_cpp.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 11fc744a01..7e9a6af23d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -94,6 +94,7 @@ def __init__( offload_kqv: bool = True, flash_attn: bool = False, # Sampling Params + no_perf: bool = False, last_n_tokens_size: int = 64, # LoRA Params lora_base: Optional[str] = None, @@ -173,6 +174,7 @@ def __init__( embedding: Embedding mode only. offload_kqv: Offload K, Q, V to GPU. flash_attn: Use flash attention. + no_perf: Measure performance timings. last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque. lora_base: Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model. lora_path: Path to a LoRA file to apply to the model. @@ -351,6 +353,7 @@ def __init__( if type_v is not None: self.context_params.type_v = type_v # Sampling Params + self.context_params.no_perf = no_perf self.last_n_tokens_size = last_n_tokens_size self.cache: Optional[BaseLlamaCache] = None @@ -2093,6 +2096,7 @@ def __getstate__(self): offload_kqv=self.context_params.offload_kqv, flash_attn=self.context_params.flash_attn, # Sampling Params + no_perf=self.context_params.no_perf, last_n_tokens_size=self.last_n_tokens_size, # LoRA Params lora_base=self.lora_base, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2b9edbb99a..205d89a0b5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -780,6 +780,7 @@ class llama_context_params(ctypes.Structure): embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU flash_attn (bool): whether to use flash attention + no_perf (bool): whether to measure performance timings abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback """ @@ -810,6 +811,7 @@ class llama_context_params(ctypes.Structure): embeddings: bool offload_kqv: bool flash_attn: bool + no_perf: bool abort_callback: Callable[[ctypes.c_void_p], bool] abort_callback_data: ctypes.c_void_p @@ -839,6 +841,7 @@ class llama_context_params(ctypes.Structure): ("embeddings", ctypes.c_bool), ("offload_kqv", ctypes.c_bool), ("flash_attn", ctypes.c_bool), + ("no_perf", ctypes.c_bool), ("abort_callback", ggml_abort_callback), ("abort_callback_data", ctypes.c_void_p), ] From 710e19a81284e5af0d5db93cef7a9063b3e8534f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 28 Jan 2025 23:19:04 -0500 Subject: [PATCH 397/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4f49c9045..53365e3689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.7] + +- feat: Update llama.cpp to ggerganov/llama.cpp@794fe23f29fb40104975c91fe19f23798f7c726e +- fix(ci): Fix the CUDA workflow by @oobabooga in #1894 +- fix: error showing time spent in llama perf context print, adds `no_perf` flag to `Llama` class by @shakalaca in #1898 + ## [0.3.6] - feat: Update llama.cpp to ggerganov/llama.cpp@f7cd13301c2a88f97073fd119072b4cc92c08df1 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 8c6118fb42..fc1fcbcf66 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.6" +__version__ = "0.3.7" From 344c106a6961b50f734b95f084fbf02057d4b475 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 12 Mar 2025 04:11:42 -0400 Subject: [PATCH 398/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 74 ++++++++++++++++++++++++++++++++++++++++-- vendor/llama.cpp | 2 +- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 205d89a0b5..f3985ad2fc 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -227,6 +227,7 @@ # LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, # LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, # LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, +# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -257,6 +258,7 @@ LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28 +LLAMA_VOCAB_PRE_TYPE_GPT4O = 29 # // note: these values should be synchronized with ggml_rope @@ -1357,6 +1359,12 @@ def llama_model_n_head(model: llama_model_p, /) -> int: ... +# LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model); +@ctypes_function("llama_model_n_head_kv", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_head_kv(model: llama_model_p, /) -> int: + ... + + # // Get the model's RoPE frequency scaling factor # LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); @ctypes_function("llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float) @@ -3375,8 +3383,8 @@ class llama_sampler_i(ctypes.Structure): # struct llama_sampler { -# struct llama_sampler_i * iface; -# llama_sampler_context_t ctx; +# const struct llama_sampler_i * iface; +# llama_sampler_context_t ctx; # }; class llama_sampler(ctypes.Structure): _fields_ = [ @@ -3410,6 +3418,18 @@ class llama_sampler(ctypes.Structure): # // mirror of llama_sampler_i: +# LLAMA_API struct llama_sampler * llama_sampler_init (const struct llama_sampler_i * iface, llama_sampler_context_t ctx); +@ctypes_function( + "llama_sampler_init", + [ctypes.POINTER(llama_sampler_i), llama_sampler_context_t], + llama_sampler_p_ctypes, +) +def llama_sampler_init( + iface: ctypes.POINTER(llama_sampler_i), ctx: llama_sampler_context_t, / +) -> llama_sampler_p: + ... + + # LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl); @ctypes_function( "llama_sampler_name", @@ -3627,6 +3647,17 @@ def llama_sampler_init_xtc( ... +# /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 +# LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float n); +@ctypes_function( + "llama_sampler_init_top_n_sigma", + [ctypes.c_float], + llama_sampler_p_ctypes, +) +def llama_sampler_init_top_n_sigma(n: float, /) -> llama_sampler_p: + ... + + # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. # /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. # /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. @@ -3685,6 +3716,43 @@ def llama_sampler_init_grammar( ... +# /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 +# /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. +# /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. +# LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( +# const struct llama_vocab * vocab, +# const char * grammar_str, +# const char * grammar_root, +# const char ** trigger_patterns, +# size_t num_trigger_patterns, +# const llama_token * trigger_tokens, +# size_t num_trigger_tokens); +@ctypes_function( + "llama_sampler_init_grammar_lazy_patterns", + [ + llama_vocab_p_ctypes, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.POINTER(ctypes.c_char_p), + ctypes.c_size_t, + ctypes.POINTER(llama_token), + ctypes.c_size_t, + ], + llama_sampler_p_ctypes, +) +def llama_sampler_init_grammar_lazy_patterns( + vocab: llama_vocab_p, + grammar_str: bytes, + grammar_root: bytes, + trigger_patterns: CtypesArray[bytes], + num_trigger_patterns: int, + trigger_tokens: CtypesArray[llama_token], + num_trigger_tokens: int, + /, +) -> llama_sampler_p: + ... + + # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( # int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) @@ -3737,7 +3805,7 @@ def llama_sampler_init_dry( dry_base: float, dry_allowed_length: int, dry_penalty_last_n: int, - seq_breakers: CtypesArray[bytes], + seq_breakers, num_breakers: int, /, ) -> llama_sampler_p: diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 794fe23f29..2c9f833d17 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 794fe23f29fb40104975c91fe19f23798f7c726e +Subproject commit 2c9f833d17bb5b8ea89dec663b072b5420fc5438 From e232fae1a35c0bdaabf230a5d71780c0178e5d8f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 12 Mar 2025 04:44:18 -0400 Subject: [PATCH 399/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 2c9f833d17..7841fc723e 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 2c9f833d17bb5b8ea89dec663b072b5420fc5438 +Subproject commit 7841fc723e059d1fd9640e5c0ef19050fcc7c698 From 37eb5f0a4c2a8706b89ead1406b1577c4602cdec Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 12 Mar 2025 05:30:21 -0400 Subject: [PATCH 400/561] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53365e3689..605370e7de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.8] + +- feat: Update llama.cpp to ggerganov/llama.cpp@7841fc723e059d1fd9640e5c0ef19050fcc7c698 + ## [0.3.7] - feat: Update llama.cpp to ggerganov/llama.cpp@794fe23f29fb40104975c91fe19f23798f7c726e diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index fc1fcbcf66..b1a8b9baa5 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.7" +__version__ = "0.3.8" From 99f2ebfde18912adeb7f714b49c1ddb624df3087 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 11 Apr 2025 03:28:20 -0400 Subject: [PATCH 401/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 268 ++++++++++++++++++++++++++++++++++++++--- vendor/llama.cpp | 2 +- 2 files changed, 255 insertions(+), 15 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index f3985ad2fc..710bd83c85 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -165,6 +165,10 @@ # llama_sampler_p = NewType("llama_sampler_p", int) # llama_sampler_p_ctypes = ctypes.c_void_p +# struct llama_kv_cache; +llama_kv_cache_p = NewType("llama_kv_cache_p", int) +llama_kv_cache_p_ctypes = ctypes.c_void_p + # typedef int32_t llama_pos; llama_pos = ctypes.c_int32 # typedef int32_t llama_token; @@ -228,6 +232,9 @@ # LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, # LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, # LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, +# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, +# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, +# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -259,6 +266,9 @@ LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28 LLAMA_VOCAB_PRE_TYPE_GPT4O = 29 +LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30 +LLAMA_VOCAB_PRE_TYPE_TRILLION = 31 +LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32 # // note: these values should be synchronized with ggml_rope @@ -630,10 +640,19 @@ class llama_model_kv_override(ctypes.Structure): value: Union[int, float, bool, bytes] +# struct llama_model_tensor_buft_override { +# const char * pattern; +# ggml_backend_buffer_type_t buft; +# }; + + # struct llama_model_params { # // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) # ggml_backend_dev_t * devices; +# // NULL-terminated list of buffer types to use for tensors that match a pattern +# const struct llama_model_tensor_buft_override * tensor_buft_overrides; + # int32_t n_gpu_layers; // number of layers to store in VRAM # enum llama_split_mode split_mode; // how to split the model across multiple GPUs @@ -668,6 +687,8 @@ class llama_model_params(ctypes.Structure): """Parameters for llama_model Attributes: + devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) + tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern n_gpu_layers (int): number of layers to store in VRAM split_mode (int): how to split the model across multiple GPUs main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored @@ -681,6 +702,8 @@ class llama_model_params(ctypes.Structure): check_tensors (bool): validate model tensor data""" if TYPE_CHECKING: + devices: CtypesArray[ctypes.c_void_p] # NOTE: unused + tensor_buft_overrides: CtypesArray[llama_model_tensor_buft_override] # NOTE: unused n_gpu_layers: int split_mode: int main_gpu: int @@ -695,6 +718,7 @@ class llama_model_params(ctypes.Structure): _fields_ = [ ("devices", ctypes.c_void_p), # NOTE: unnused + ("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), @@ -1317,7 +1341,18 @@ def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... -# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); +# LLAMA_API struct llama_kv_cache * llama_get_kv_self ( struct llama_context * ctx); +@ctypes_function( + "llama_get_kv_self", + [llama_context_p_ctypes], + llama_kv_cache_p_ctypes, +) +def llama_get_kv_self(ctx: llama_context_p, /) -> Optional[llama_kv_cache_p]: + """Get the KV cache for self-attention""" + ... + + +# LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) def llama_pooling_type(ctx: llama_context_p, /) -> int: ... @@ -1810,7 +1845,19 @@ def llama_kv_cache_view_update(ctx: llama_context_p, view: CtypesPointerOrRef[ll # // Returns the number of tokens in the KV cache (slow, use only for debug) # // If a KV cell has multiple sequences assigned to it, it will be counted multiple times -# LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx); +# LLAMA_API int32_t llama_kv_self_n_tokens(const struct llama_context * ctx); +@ctypes_function( + "llama_kv_self_n_tokens", [llama_context_p_ctypes], ctypes.c_int32 +) +def llama_kv_self_n_tokens(ctx: llama_context_p, /) -> int: + """Returns the number of tokens in the KV cache (slow, use only for debug) + If a KV cell has multiple sequences assigned to it, it will be counted multiple times + """ + ... + + +# DEPRECATED(LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx), +# "use llama_kv_self_n_tokens instead"); @ctypes_function( "llama_get_kv_cache_token_count", [llama_context_p_ctypes], ctypes.c_int32 ) @@ -1822,7 +1869,17 @@ def llama_get_kv_cache_token_count(ctx: llama_context_p, /) -> int: # // Returns the number of used KV cells (i.e. have at least one sequence assigned to them) -# LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx); +# LLAMA_API int32_t llama_kv_self_used_cells(const struct llama_context * ctx); +@ctypes_function( + "llama_kv_self_used_cells", [llama_context_p_ctypes], ctypes.c_int32 +) +def llama_kv_self_used_cells(ctx: llama_context_p, /) -> int: + """Returns the number of used KV cells (i.e. have at least one sequence assigned to them)""" + ... + + +# DEPRECATED(LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx), +# "use llama_kv_self_used_cells instead"); @ctypes_function( "llama_get_kv_cache_used_cells", [llama_context_p_ctypes], ctypes.c_int32 ) @@ -1832,9 +1889,17 @@ def llama_get_kv_cache_used_cells(ctx: llama_context_p, /) -> int: # // Clear the KV cache - both cell info is erased and KV data is zeroed -# LLAMA_API void llama_kv_cache_clear( +# LLAMA_API void llama_kv_self_clear( # struct llama_context * ctx); -@ctypes_function("llama_kv_cache_clear", [llama_context_p_ctypes], None) +@ctypes_function( + "llama_kv_self_clear", [llama_context_p_ctypes], None +) +def llama_kv_self_clear(ctx: llama_context_p, /): + """Clear the KV cache - both cell info is erased and KV data is zeroed""" + ... + +# NOTE: Deprecated +@ctypes_function("llama_kv_self_clear", [llama_context_p_ctypes], None) def llama_kv_cache_clear(ctx: llama_context_p, /): """Clear the KV cache""" ... @@ -1881,14 +1946,41 @@ def llama_kv_cache_seq_rm( # // Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence # // p0 < 0 : [0, p1] # // p1 < 0 : [p0, inf) -# LLAMA_API void llama_kv_cache_seq_cp( +# LLAMA_API void llama_kv_self_seq_cp( # struct llama_context * ctx, # llama_seq_id seq_id_src, # llama_seq_id seq_id_dst, # llama_pos p0, # llama_pos p1); @ctypes_function( - "llama_kv_cache_seq_cp", + "llama_kv_self_seq_cp", + [ + llama_context_p_ctypes, + llama_seq_id, + llama_seq_id, + llama_pos, + llama_pos, + ], + None, +) +def llama_kv_self_seq_cp( + ctx: llama_context_p, + seq_id_src: Union[llama_seq_id, int], + seq_id_dst: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + /, +): + """Copy all tokens that belong to the specified sequence to another sequence + Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... + + +# NOTE: Deprecated +@ctypes_function( + "llama_kv_self_seq_cp", [ llama_context_p_ctypes, llama_seq_id, @@ -1914,17 +2006,27 @@ def llama_kv_cache_seq_cp( # // Removes all tokens that do not belong to the specified sequence -# LLAMA_API void llama_kv_cache_seq_keep( +# LLAMA_API void llama_kv_self_seq_keep( # struct llama_context * ctx, # llama_seq_id seq_id); @ctypes_function( - "llama_kv_cache_seq_keep", [llama_context_p_ctypes, llama_seq_id], None + "llama_kv_self_seq_keep", [llama_context_p_ctypes, llama_seq_id], None +) +def llama_kv_self_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /): + """Removes all tokens that do not belong to the specified sequence""" + ... + + +# NOTE: Deprecated +@ctypes_function( + "llama_kv_self_seq_keep", [llama_context_p_ctypes, llama_seq_id], None ) def llama_kv_cache_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /): """Removes all tokens that do not belong to the specified sequence""" ... + # // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) # // If the KV cache is RoPEd, the KV data is updated accordingly: # // - lazily on next llama_decode() @@ -1938,7 +2040,48 @@ def llama_kv_cache_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, in # llama_pos p1, # llama_pos delta); @ctypes_function( - "llama_kv_cache_seq_add", + "llama_kv_self_seq_add", + [ + llama_context_p_ctypes, + llama_seq_id, + llama_pos, + llama_pos, + llama_pos, + ], + None, +) +def llama_kv_self_seq_add( + ctx: llama_context_p, + seq_id: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + delta: Union[llama_pos, int], + /, +): + """Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) + If the KV cache is RoPEd, the KV data is updated accordingly: + - lazily on next llama_decode() + - explicitly with llama_kv_cache_update() + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... + + +# // NOTE: Deprecated +# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) +# // If the KV cache is RoPEd, the KV data is updated accordingly: +# // - lazily on next llama_decode() +# // - explicitly with llama_kv_cache_update() +# // p0 < 0 : [0, p1] +# // p1 < 0 : [p0, inf) +# LLAMA_API void llama_kv_cache_seq_add( +# struct llama_context * ctx, +# llama_seq_id seq_id, +# llama_pos p0, +# llama_pos p1, +# llama_pos delta); +@ctypes_function( + "llama_kv_self_seq_add", [ llama_context_p_ctypes, llama_seq_id, @@ -1976,7 +2119,44 @@ def llama_kv_cache_seq_add( # llama_pos p1, # int d); @ctypes_function( - "llama_kv_cache_seq_div", + "llama_kv_self_seq_div", + [ + llama_context_p_ctypes, + llama_seq_id, + llama_pos, + llama_pos, + ctypes.c_int, + ], + None, +) +def llama_kv_self_seq_div( + ctx: llama_context_p, + seq_id: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + d: Union[ctypes.c_int, int], + /, +): + """Integer division of the positions by factor of `d > 1` + If the KV cache is RoPEd, the KV data is updated accordingly + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... + + +# // NOTE: Deprecated +# // Integer division of the positions by factor of `d > 1` +# // If the KV cache is RoPEd, the KV data is updated accordingly +# // p0 < 0 : [0, p1] +# // p1 < 0 : [p0, inf) +# LLAMA_API void llama_kv_cache_seq_div( +# struct llama_context * ctx, +# llama_seq_id seq_id, +# llama_pos p0, +# llama_pos p1, +# int d); +@ctypes_function( + "llama_kv_self_seq_div", [ llama_context_p_ctypes, llama_seq_id, @@ -2001,10 +2181,39 @@ def llama_kv_cache_seq_div( ... +# // Returns the largest position present in the KV cache for the specified sequence +# LLAMA_API llama_pos llama_kv_self_seq_pos_max( +# struct llama_context * ctx, +# llama_seq_id seq_id); +@ctypes_function( + "llama_kv_self_seq_pos_max", [llama_context_p_ctypes, llama_seq_id], llama_pos +) +def llama_kv_self_seq_pos_max( + ctx: llama_context_p, seq_id: Union[llama_seq_id, int], / +) -> int: + """Returns the largest position present in the KV cache for the specified sequence""" + ... + + # // Defragment the KV cache # // This will be applied: # // - lazily on next llama_decode() -# // - explicitly with llama_kv_cache_update() +# // - explicitly with llama_kv_self_update() +# LLAMA_API void llama_kv_self_defrag(struct llama_context * ctx); +@ctypes_function("llama_kv_self_defrag", [llama_context_p_ctypes], None) +def llama_kv_self_defrag(ctx: llama_context_p, /): + """Defragment the KV cache + This will be applied: + - lazily on next llama_decode() + - explicitly with llama_kv_cache_update()""" + ... + + +# NOTE: Deprecated +# // Defragment the KV cache +# // This will be applied: +# // - lazily on next llama_decode() +# // - explicitly with llama_kv_self_update() # LLAMA_API void llama_kv_cache_defrag(struct llama_context * ctx); @ctypes_function("llama_kv_cache_defrag", [llama_context_p_ctypes], None) def llama_kv_cache_defrag(ctx: llama_context_p, /): @@ -2017,7 +2226,15 @@ def llama_kv_cache_defrag(ctx: llama_context_p, /): # // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) # LLAMA_API void llama_kv_cache_update(struct llama_context * ctx); -@ctypes_function("llama_kv_cache_update", [llama_context_p_ctypes], None) +@ctypes_function("llama_kv_self_update", [llama_context_p_ctypes], None) +def llama_kv_self_update(ctx: llama_context_p, /): + """Apply the KV cache updates (such as K-shifts, defragmentation, etc.)""" + ... + +# // NOTE: Deprecated +# // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) +# LLAMA_API void llama_kv_cache_update(struct llama_context * ctx); +@ctypes_function("llama_kv_self_update", [llama_context_p_ctypes], None) def llama_kv_cache_update(ctx: llama_context_p, /): """Apply the KV cache updates (such as K-shifts, defragmentation, etc.)""" ... @@ -2025,7 +2242,16 @@ def llama_kv_cache_update(ctx: llama_context_p, /): # // Check if the context supports KV cache shifting # LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx); -@ctypes_function("llama_kv_cache_can_shift", [llama_context_p_ctypes], ctypes.c_bool) +@ctypes_function("llama_kv_self_can_shift", [llama_context_p_ctypes], ctypes.c_bool) +def llama_kv_self_can_shift(ctx: llama_context_p, /) -> bool: + """Check if the context supports KV cache shifting""" + ... + + +# // NOTE: Deprecated +# // Check if the context supports KV cache shifting +# LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx); +@ctypes_function("llama_kv_self_can_shift", [llama_context_p_ctypes], ctypes.c_bool) def llama_kv_cache_can_shift(ctx: llama_context_p, /) -> bool: """Check if the context supports KV cache shifting""" ... @@ -2547,6 +2773,16 @@ def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /): ... +# // Set whether the model is in warmup mode or not +# // If true, all model tensors are activated during llama_decode() to load and cache their weights. +# LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup); +@ctypes_function("llama_set_warmup", [llama_context_p_ctypes, ctypes.c_bool], None) +def llama_set_warmup(ctx: llama_context_p, warmup: bool, /): + """Set whether the model is in warmup mode or not + If true, all model tensors are activated during llama_decode() to load and cache their weights.""" + ... + + # // Set abort callback # LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); @ctypes_function( @@ -3701,6 +3937,10 @@ def llama_sampler_init_mirostat_v2( ... +# /// @details Intializes a GBNF grammar, see grammars/README.md for details. +# /// @param vocab The vocabulary that this grammar will be used with. +# /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. +# /// @param grammar_root The name of the start symbol for the grammar. # LLAMA_API struct llama_sampler * llama_sampler_init_grammar( # const struct llama_vocab * vocab, # const char * grammar_str, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7841fc723e..6bf28f0111 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7841fc723e059d1fd9640e5c0ef19050fcc7c698 +Subproject commit 6bf28f0111ff9f21b3c1b1eace20c590281e7ba6 From 4c6514d365cb4f00910424c2feb8e45e68a870a6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 May 2025 06:52:52 -0400 Subject: [PATCH 402/561] feat: Update llama.cpp --- CMakeLists.txt | 2 +- llama_cpp/llama_cpp.py | 33 +++++++++++++++++++++------------ vendor/llama.cpp | 2 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 64a0304a16..443d1ee532 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,7 +143,7 @@ if (LLAMA_BUILD) endif() # Building llava - add_subdirectory(vendor/llama.cpp/examples/llava) + add_subdirectory(vendor/llama.cpp/tools/mtmd) set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") if (WIN32) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 710bd83c85..63de3a93a4 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -235,6 +235,8 @@ # LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, # LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, # LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, +# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, +# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -252,7 +254,7 @@ LLAMA_VOCAB_PRE_TYPE_DBRX = 13 LLAMA_VOCAB_PRE_TYPE_SMAUG = 14 LLAMA_VOCAB_PRE_TYPE_PORO = 15 -LLAMA_VOCAV_PRE_TYPE_CHATGLM3 = 16 +LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16 LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17 LLAMA_VOCAB_PRE_TYPE_VIKING = 18 LLAMA_VOCAB_PRE_TYPE_JAIS = 19 @@ -269,6 +271,8 @@ LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30 LLAMA_VOCAB_PRE_TYPE_TRILLION = 31 LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32 +LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33 +LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34 # // note: these values should be synchronized with ggml_rope @@ -891,17 +895,18 @@ class llama_context_params(ctypes.Structure): # // model quantization parameters # typedef struct llama_model_quantize_params { -# int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() -# enum llama_ftype ftype; // quantize to this llama_ftype -# enum ggml_type output_tensor_type; // output tensor type -# enum ggml_type token_embedding_type; // token embeddings tensor type -# bool allow_requantize; // allow quantizing non-f32/f16 tensors -# bool quantize_output_tensor; // quantize output.weight -# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored -# bool pure; // quantize all tensors to the default type -# bool keep_split; // quantize to the same number of shards -# void * imatrix; // pointer to importance matrix data -# void * kv_overrides; // pointer to vector containing overrides +# int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() +# enum llama_ftype ftype; // quantize to this llama_ftype +# enum ggml_type output_tensor_type; // output tensor type +# enum ggml_type token_embedding_type; // token embeddings tensor type +# bool allow_requantize; // allow quantizing non-f32/f16 tensors +# bool quantize_output_tensor; // quantize output.weight +# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored +# bool pure; // quantize all tensors to the default type +# bool keep_split; // quantize to the same number of shards +# void * imatrix; // pointer to importance matrix data +# void * kv_overrides; // pointer to vector containing overrides +# void * tensor_types; // pointer to vector containing tensor types # } llama_model_quantize_params; class llama_model_quantize_params(ctypes.Structure): """Parameters for llama_model_quantize @@ -918,6 +923,7 @@ class llama_model_quantize_params(ctypes.Structure): keep_split (bool): quantize to the same number of shards imatrix (ctypes.c_void_p): pointer to importance matrix data kv_overrides (ctypes.c_void_p): pointer to vector containing overrides + tensor_types (ctypes.c_void_p): pointer to vector containing tensor types """ if TYPE_CHECKING: @@ -932,6 +938,7 @@ class llama_model_quantize_params(ctypes.Structure): keep_split: bool imatrix: ctypes.c_void_p kv_overrides: ctypes.c_void_p + tensor_types: ctypes.c_void_p _fields_ = [ ("nthread", ctypes.c_int32), @@ -945,6 +952,7 @@ class llama_model_quantize_params(ctypes.Structure): ("keep_split", ctypes.c_bool), ("imatrix", ctypes.c_void_p), ("kv_overrides", ctypes.c_void_p), + ("tensor_types", ctypes.c_void_p), ] @@ -3812,6 +3820,7 @@ def llama_sampler_init_softmax() -> llama_sampler_p: # /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 +# /// Setting k <= 0 makes this a noop # LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); @ctypes_function("llama_sampler_init_top_k", [ctypes.c_int32], llama_sampler_p_ctypes) def llama_sampler_init_top_k(k: int) -> llama_sampler_p: diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6bf28f0111..8733e0cf6e 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6bf28f0111ff9f21b3c1b1eace20c590281e7ba6 +Subproject commit 8733e0cf6eefc7c7752297cc22d0836706f4222c From cb2edb9c9fef90fe680e528fadc86806c6f385e5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 May 2025 06:59:55 -0400 Subject: [PATCH 403/561] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 605370e7de..affbd5db78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.9] + +- feat: Update llama.cpp to ggerganov/llama.cpp@8733e0cf6eefc7c7752297cc22d0836706f4222c + ## [0.3.8] - feat: Update llama.cpp to ggerganov/llama.cpp@7841fc723e059d1fd9640e5c0ef19050fcc7c698 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index b1a8b9baa5..2c9c527cd4 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.8" +__version__ = "0.3.9" From b1d23df0bbd327b774083b5cf88e67ca0dd52b92 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 8 May 2025 07:02:20 -0400 Subject: [PATCH 404/561] hotfix: Disable curl support --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 443d1ee532..b9178e8564 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,9 @@ if (LLAMA_BUILD) # Enable building of the common library set(LLAMA_BUILD_COMMON ON CACHE BOOL "Build llama.cpp common library" FORCE) + # Disable building curl support + set(LLAMA_CURL OFF CACHE BOOL "llama.cpp: enable curl" FORCE) + # Architecture detection and settings for Apple platforms if (APPLE) # Get the target architecture From 0d475d7bdabd4d57e1be12081c327b21bf5fa1f1 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 1 Jul 2025 00:51:44 -0400 Subject: [PATCH 405/561] feat: Update llama.cpp --- CMakeLists.txt | 84 +-- llama_cpp/_internals.py | 449 ++++++++-------- llama_cpp/llama.py | 10 +- llama_cpp/llama_cpp.py | 1082 ++++++++++++++++++++------------------- vendor/llama.cpp | 2 +- 5 files changed, 811 insertions(+), 816 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b9178e8564..260222bde2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,46 +135,46 @@ if (LLAMA_BUILD) ) endif() - if (LLAVA_BUILD) - if (LLAMA_CUBLAS OR LLAMA_CUDA) - add_compile_definitions(GGML_USE_CUBLAS) - add_compile_definitions(GGML_USE_CUDA) - endif() - - if (LLAMA_METAL) - add_compile_definitions(GGML_USE_METAL) - endif() - - # Building llava - add_subdirectory(vendor/llama.cpp/tools/mtmd) - set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") - - if (WIN32) - set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) - endif() - llama_cpp_python_install_target(llava_shared) - if (WIN32) - install( - FILES $ - DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib - ) - install( - FILES $ - DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib - ) - endif() - - # Fix for llava build: Add include directory for llama.h - # Move these commands after the add_subdirectory call - target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) - target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) - - if (BUILD_SHARED_LIBS) - target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) - target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) - endif() - - target_include_directories(llama-llava-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) - target_include_directories(llama-minicpmv-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) - endif() + # if (LLAVA_BUILD) + # if (LLAMA_CUBLAS OR LLAMA_CUDA) + # add_compile_definitions(GGML_USE_CUBLAS) + # add_compile_definitions(GGML_USE_CUDA) + # endif() + # + # if (LLAMA_METAL) + # add_compile_definitions(GGML_USE_METAL) + # endif() + # + # # Building llava + # add_subdirectory(vendor/llama.cpp/tools/mtmd) + # set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") + # + # if (WIN32) + # set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) + # endif() + # llama_cpp_python_install_target(llava_shared) + # if (WIN32) + # install( + # FILES $ + # DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + # ) + # install( + # FILES $ + # DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib + # ) + # endif() + # + # # Fix for llava build: Add include directory for llama.h + # # Move these commands after the add_subdirectory call + # target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + # target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) + # + # if (BUILD_SHARED_LIBS) + # target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + # target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) + # endif() + # + # target_include_directories(llama-llava-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + # target_include_directories(llama-minicpmv-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + # endif() endif() diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 343581dce0..18d733481f 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -9,6 +9,8 @@ Tuple, Optional, Sequence, + Callable, + Union, ) from dataclasses import dataclass, field from contextlib import ExitStack @@ -48,7 +50,7 @@ def __init__( raise ValueError(f"Model path does not exist: {path_model}") with suppress_stdout_stderr(disable=verbose): - model = llama_cpp.llama_load_model_from_file( + model = llama_cpp.llama_model_load_from_file( self.path_model.encode("utf-8"), self.params ) @@ -62,32 +64,38 @@ def __init__( self.model = model self.vocab = vocab + self.sampler = None # LlamaModel doesn't use samplers, but some cleanup code expects this attribute def free_model(): if self.model is None: return - llama_cpp.llama_free_model(self.model) + llama_cpp.llama_model_free(self.model) self.model = None self._exit_stack.callback(free_model) def close(self): + if self.sampler is not None: + # NOTE: Must remove custom samplers before free or llama.cpp will try to free them + for i, _ in reversed(self.custom_samplers): + llama_cpp.llama_sampler_chain_remove(self.sampler, i) + self.custom_samplers.clear() self._exit_stack.close() def __del__(self): self.close() def vocab_type(self) -> int: - return llama_cpp.llama_vocab_type(self.model) + return llama_cpp.llama_vocab_type(self.vocab) def n_vocab(self) -> int: - return llama_cpp.llama_n_vocab(self.vocab) + return llama_cpp.llama_vocab_n_tokens(self.vocab) def n_ctx_train(self) -> int: - return llama_cpp.llama_n_ctx_train(self.model) + return llama_cpp.llama_model_n_ctx_train(self.model) def n_embd(self) -> int: - return llama_cpp.llama_n_embd(self.model) + return llama_cpp.llama_model_n_embd(self.model) def rope_freq_scale_train(self) -> float: return llama_cpp.llama_model_rope_freq_scale_train(self.model) @@ -109,48 +117,48 @@ def get_tensor(self, name: str) -> ctypes.c_void_p: # Vocab def token_get_text(self, token: int) -> str: - return llama_cpp.llama_token_get_text(self.vocab, token).decode("utf-8") + return llama_cpp.llama_vocab_get_text(self.vocab, token).decode("utf-8") def token_get_score(self, token: int) -> float: - return llama_cpp.llama_token_get_score(self.vocab, token) + return llama_cpp.llama_vocab_get_score(self.vocab, token) def token_get_attr(self, token: int) -> int: - return llama_cpp.llama_token_get_attr(self.vocab, token) + return llama_cpp.llama_vocab_get_attr(self.vocab, token) # Special tokens def token_bos(self) -> int: - return llama_cpp.llama_token_bos(self.vocab) + return llama_cpp.llama_vocab_bos(self.vocab) def token_eos(self) -> int: - return llama_cpp.llama_token_eos(self.vocab) + return llama_cpp.llama_vocab_eos(self.vocab) def token_cls(self) -> int: - return llama_cpp.llama_token_cls(self.vocab) + return llama_cpp.llama_vocab_cls(self.vocab) def token_sep(self) -> int: - return llama_cpp.llama_token_sep(self.vocab) + return llama_cpp.llama_vocab_sep(self.vocab) def token_nl(self) -> int: - return llama_cpp.llama_token_nl(self.vocab) + return llama_cpp.llama_vocab_nl(self.vocab) def token_prefix(self) -> int: - raise NotImplementedError("token_prefix is not implemented in llama.cpp") + return llama_cpp.llama_vocab_fim_pre(self.vocab) def token_middle(self) -> int: - raise NotImplementedError("token_middle is not implemented in llama.cpp") + return llama_cpp.llama_vocab_fim_mid(self.vocab) def token_suffix(self) -> int: - raise NotImplementedError("token_suffix is not implemented in llama.cpp") + return llama_cpp.llama_vocab_fim_suf(self.vocab) def token_eot(self) -> int: - return llama_cpp.llama_token_eot(self.vocab) + return llama_cpp.llama_vocab_eot(self.vocab) def add_bos_token(self) -> bool: - return llama_cpp.llama_add_bos_token(self.vocab) + return llama_cpp.llama_vocab_get_add_bos(self.vocab) def add_eos_token(self) -> bool: - return llama_cpp.llama_add_eos_token(self.vocab) + return llama_cpp.llama_vocab_get_add_eos(self.vocab) # Tokenization @@ -249,12 +257,14 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - ctx = llama_cpp.llama_new_context_with_model(self.model.model, self.params) + ctx = llama_cpp.llama_init_from_model(self.model.model, self.params) if ctx is None: raise ValueError("Failed to create llama_context") self.ctx = ctx + self.memory = llama_cpp.llama_get_memory(self.ctx) + self.sampler = None # LlamaContext doesn't manage samplers directly, but some cleanup code expects this attribute def free_ctx(): if self.ctx is None: @@ -277,22 +287,22 @@ def pooling_type(self) -> int: return llama_cpp.llama_pooling_type(self.ctx) def kv_cache_clear(self): - llama_cpp.llama_kv_cache_clear(self.ctx) + llama_cpp.llama_memory_clear(self.memory, True) def kv_cache_seq_rm(self, seq_id: int, p0: int, p1: int): - llama_cpp.llama_kv_cache_seq_rm(self.ctx, seq_id, p0, p1) + llama_cpp.llama_memory_seq_rm(self.memory, seq_id, p0, p1) def kv_cache_seq_cp(self, seq_id_src: int, seq_id_dst: int, p0: int, p1: int): - llama_cpp.llama_kv_cache_seq_cp(self.ctx, seq_id_src, seq_id_dst, p0, p1) + llama_cpp.llama_memory_seq_cp(self.memory, seq_id_src, seq_id_dst, p0, p1) def kv_cache_seq_keep(self, seq_id: int): - llama_cpp.llama_kv_cache_seq_keep(self.ctx, seq_id) + llama_cpp.llama_memory_seq_keep(self.memory, seq_id) def kv_cache_seq_shift(self, seq_id: int, p0: int, p1: int, shift: int): - llama_cpp.llama_kv_cache_seq_add(self.ctx, seq_id, p0, p1, shift) + llama_cpp.llama_memory_seq_add(self.memory, seq_id, p0, p1, shift) def get_state_size(self) -> int: - return llama_cpp.llama_get_state_size(self.ctx) + return llama_cpp.llama_state_get_size(self.ctx) # TODO: copy_state_data @@ -310,6 +320,14 @@ def decode(self, batch: LlamaBatch): if return_code != 0: raise RuntimeError(f"llama_decode returned {return_code}") + def encode(self, batch: LlamaBatch): + return_code = llama_cpp.llama_encode( + self.ctx, + batch.batch, + ) + if return_code != 0: + raise RuntimeError(f"llama_encode returned {return_code}") + def set_n_threads(self, n_threads: int, n_threads_batch: int): llama_cpp.llama_set_n_threads(self.ctx, n_threads, n_threads_batch) @@ -322,12 +340,16 @@ def get_logits_ith(self, i: int): def get_embeddings(self): return llama_cpp.llama_get_embeddings(self.ctx) - # Sampling functions + def get_embeddings_ith(self, i: int): + return llama_cpp.llama_get_embeddings_ith(self.ctx, i) + + def get_embeddings_seq(self, seq_id: int): + return llama_cpp.llama_get_embeddings_seq(self.ctx, seq_id) + + # Sampling functions - deprecated, use LlamaSampler instead def set_rng_seed(self, seed: int): - # TODO: Fix - # llama_cpp.llama_set_rng_seed(self.ctx, seed) - raise NotImplementedError("set_rng_seed is not implemented in llama.cpp") + raise NotImplementedError("set_rng_seed is deprecated, use LlamaSampler instead") def sample_repetition_penalties( self, @@ -338,63 +360,30 @@ def sample_repetition_penalties( penalty_freq: float, penalty_present: float, ): - # llama_cpp.llama_sample_repetition_penalties( - # self.ctx, - # llama_cpp.byref(candidates.candidates), - # last_tokens_data, - # penalty_last_n, - # penalty_repeat, - # penalty_freq, - # penalty_present, - # ) - raise NotImplementedError("sample_repetition_penalties is not implemented in llama.cpp") + raise NotImplementedError("sample_repetition_penalties is deprecated, use LlamaSampler instead") def sample_softmax(self, candidates: "_LlamaTokenDataArray"): - # llama_cpp.llama_sample_softmax( - # self.ctx, - # llama_cpp.byref(candidates.candidates), - # ) - raise NotImplementedError("sample_softmax is not implemented in llama.cpp") + raise NotImplementedError("sample_softmax is deprecated, use LlamaSampler instead") def sample_top_k(self, candidates: "_LlamaTokenDataArray", k: int, min_keep: int): - # llama_cpp.llama_sample_top_k( - # self.ctx, llama_cpp.byref(candidates.candidates), k, min_keep - # ) - raise NotImplementedError("sample_top_k is not implemented in llama.cpp") + raise NotImplementedError("sample_top_k is deprecated, use LlamaSampler instead") def sample_top_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - # llama_cpp.llama_sample_top_p( - # self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep - # ) - raise NotImplementedError("sample_top_p is not implemented in llama.cpp") + raise NotImplementedError("sample_top_p is deprecated, use LlamaSampler instead") def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - # llama_cpp.llama_sample_min_p( - # self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep - # ) - raise NotImplementedError("sample_min_p is not implemented in llama.cpp") + raise NotImplementedError("sample_min_p is deprecated, use LlamaSampler instead") def sample_typical( self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int ): - # llama_cpp.llama_sample_typical( - # self.ctx, llama_cpp.byref(candidates.candidates), p, min_keep - # ) - raise NotImplementedError("sample_typical is not implemented in llama.cpp") + raise NotImplementedError("sample_typical is deprecated, use LlamaSampler instead") def sample_temp(self, candidates: "_LlamaTokenDataArray", temp: float): - # llama_cpp.llama_sample_temp( - # self.ctx, llama_cpp.byref(candidates.candidates), temp - # ) - raise NotImplementedError("sample_temp is not implemented in llama.cpp") + raise NotImplementedError("sample_temp is deprecated, use LlamaSampler instead") def sample_grammar(self, candidates: "_LlamaTokenDataArray", grammar: LlamaGrammar): - # llama_cpp.llama_sample_grammar( - # self.ctx, - # llama_cpp.byref(candidates.candidates), - # grammar.grammar, - # ) - raise NotImplementedError("sample_grammar is not implemented in llama.cpp") + raise NotImplementedError("sample_grammar is deprecated, use LlamaSampler instead") def sample_token_mirostat( self, @@ -404,15 +393,7 @@ def sample_token_mirostat( m: int, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - raise NotImplementedError("sample_token_mirostat is not implemented in llama.cpp") - # return llama_cpp.llama_sample_token_mirostat( - # self.ctx, - # llama_cpp.byref(candidates.candidates), - # tau, - # eta, - # m, - # mu, - # ) + raise NotImplementedError("sample_token_mirostat is deprecated, use LlamaSampler instead") def sample_token_mirostat_v2( self, @@ -421,33 +402,17 @@ def sample_token_mirostat_v2( eta: float, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - raise NotImplementedError("sample_token_mirostat_v2 is not implemented in llama.cpp") - # return llama_cpp.llama_sample_token_mirostat_v2( - # self.ctx, - # llama_cpp.byref(candidates.candidates), - # tau, - # eta, - # mu, - # ) + raise NotImplementedError("sample_token_mirostat_v2 is deprecated, use LlamaSampler instead") def sample_token_greedy(self, candidates: "_LlamaTokenDataArray") -> int: - raise NotImplementedError("sample_token_greedy is not implemented in llama.cpp") - # return llama_cpp.llama_sample_token_greedy( - # self.ctx, - # llama_cpp.byref(candidates.candidates), - # ) + raise NotImplementedError("sample_token_greedy is deprecated, use LlamaSampler instead") def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: - raise NotImplementedError("sample_token is not implemented in llama.cpp") - # return llama_cpp.llama_sample_token( - # self.ctx, - # llama_cpp.byref(candidates.candidates), - # ) + raise NotImplementedError("sample_token is deprecated, use LlamaSampler instead") # Grammar def grammar_accept_token(self, grammar: LlamaGrammar, token: int): - raise NotImplementedError("grammar_accept_token is not implemented in llama.cpp") - # llama_cpp.llama_grammar_accept_token(grammar.grammar, self.ctx, token) + raise NotImplementedError("grammar_accept_token is deprecated, use LlamaSampler instead") def reset_timings(self): llama_cpp.llama_perf_context_reset(self.ctx) @@ -478,6 +443,7 @@ def __init__( raise ValueError("Failed to create llama_batch") self.batch = batch + self.sampler = None # LlamaBatch doesn't use samplers, but some cleanup code expects this attribute def free_batch(): if self.batch is None: @@ -540,6 +506,7 @@ def __init__(self, *, n_vocab: int): ) self.default_candidates_data_id = np.arange(self.n_vocab, dtype=np.intc) # type: ignore self.default_candidates_data_p = np.zeros(self.n_vocab, dtype=np.single) + self.sampler = None # LlamaTokenDataArray doesn't use samplers, but some cleanup code expects this attribute def copy_logits(self, logits: npt.NDArray[np.single]): self.candidates_data.id[:] = self.default_candidates_data_id @@ -628,103 +595,16 @@ def sample( idx: int = 0, logits_array: Optional[npt.NDArray[np.single]] = None, ): - n_vocab = ctx_main.model.n_vocab() - id: int = 0 - - if logits_array is None: - logits = ctx_main.get_logits_ith(idx) - logits_array = np.array( - ctypes.cast(logits, ctypes.POINTER(ctypes.c_float * n_vocab)).contents, - dtype=np.single, - ) - - # apply logit_bias - for token, logit_bias in self.params.logit_bias.items(): - logits_array[token] += logit_bias - - token_data_array = LlamaTokenDataArray( - n_vocab=n_vocab - ) # TODO: Only create this once - token_data_array.copy_logits(logits_array) - - # apply penalties - if len(self.prev) > 0: - nl_token = ctx_main.model.token_nl() - nl_logit = logits_array[nl_token] - last_tokens = self.prev[-self.params.penalty_last_n :] - last_tokens_size = min(len(last_tokens), self.params.penalty_last_n) - if last_tokens_size > 0: - last_tokens_p = (llama_cpp.llama_token * len(last_tokens))(*last_tokens) - ctx_main.sample_repetition_penalties( - token_data_array, - last_tokens_p, - last_tokens_size, - self.params.penalty_repeat, - self.params.penalty_freq, - self.params.penalty_present, - ) - if not self.params.penalize_nl: - token_data_array.candidates_data.logit[nl_token] = nl_logit - - if self.grammar is not None: - ctx_main.sample_grammar(token_data_array, self.grammar) - - if self.params.temp < 0: - ctx_main.sample_softmax(token_data_array) - id = token_data_array.candidates_data.id[0] - elif self.params.temp == 0: - id = ctx_main.sample_token_greedy(token_data_array) - else: - if self.params.mirostat == 1: - mirostat_m = 100 - ctx_main.sample_temp(token_data_array, self.params.temp) - id = ctx_main.sample_token_mirostat( - token_data_array, - self.params.mirostat_tau, - self.params.mirostat_eta, - mirostat_m, - ctypes.pointer(self.mirostat_mu), - ) - elif self.params.mirostat == 2: - ctx_main.sample_temp(token_data_array, self.params.temp) - id = ctx_main.sample_token_mirostat_v2( - token_data_array, - self.params.mirostat_tau, - self.params.mirostat_eta, - ctypes.pointer(self.mirostat_mu), - ) - else: - min_keep = max(1, self.params.n_probs) - ctx_main.sample_top_k( - token_data_array, self.params.top_k, min_keep=min_keep - ) - ctx_main.sample_typical( - token_data_array, self.params.typical_p, min_keep=min_keep - ) - ctx_main.sample_top_p( - token_data_array, self.params.top_p, min_keep=min_keep - ) - ctx_main.sample_min_p( - token_data_array, self.params.min_p, min_keep=min_keep - ) - ctx_main.sample_temp(token_data_array, self.params.temp) - id = ctx_main.sample_token(token_data_array) - return id + # This method is deprecated in favor of using LlamaSampler directly + raise NotImplementedError("LlamaSamplingContext.sample is deprecated, use LlamaSampler instead") def accept(self, ctx_main: LlamaContext, id: int, apply_grammar: bool): - if apply_grammar and self.grammar is not None: - ctx_main.grammar_accept_token(self.grammar, id) self.prev.append(id) -from typing import List, Callable, Optional, Union -import ctypes -import llama_cpp - - class CustomSampler: def __init__( - self, apply_func: typing.Callable[[llama_cpp.llama_token_data_array], None] + self, apply_func: Callable[[llama_cpp.llama_token_data_array], None] ): self.apply_func = apply_func @@ -757,72 +637,117 @@ def get_sampler(self) -> llama_cpp.llama_sampler_p: class LlamaSampler: def __init__(self): - params = llama_cpp.llama_sampler_chain_params() + params = llama_cpp.llama_sampler_chain_default_params() self.sampler = llama_cpp.llama_sampler_chain_init(params) - self.samplers: List[llama_cpp.llama_sampler_p] = [] self.custom_samplers: List[Tuple[int, CustomSampler]] = [] + self._exit_stack = ExitStack() + + def free_sampler(): + if self.sampler is not None: + # NOTE: Must remove custom samplers before free or llama.cpp will try to free them + for i, _ in reversed(self.custom_samplers): + llama_cpp.llama_sampler_chain_remove(self.sampler, i) + llama_cpp.llama_sampler_free(self.sampler) + self.sampler = None + + self._exit_stack.callback(free_sampler) + + def close(self): + self._exit_stack.close() + + def __del__(self): + self.close() def add_greedy(self): sampler = llama_cpp.llama_sampler_init_greedy() - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_dist(self, seed: int): sampler = llama_cpp.llama_sampler_init_dist(seed) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_softmax(self): sampler = llama_cpp.llama_sampler_init_softmax() - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_top_k(self, k: int): sampler = llama_cpp.llama_sampler_init_top_k(k) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) - def add_top_p(self, p: float, min_keep: int): + def add_top_p(self, p: float, min_keep: int = 1): sampler = llama_cpp.llama_sampler_init_top_p(p, min_keep) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) - def add_min_p(self, p: float, min_keep: int): + def add_min_p(self, p: float, min_keep: int = 1): sampler = llama_cpp.llama_sampler_init_min_p(p, min_keep) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) - def add_typical(self, p: float, min_keep: int): + def add_typical(self, p: float, min_keep: int = 1): sampler = llama_cpp.llama_sampler_init_typical(p, min_keep) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_temp(self, temp: float): sampler = llama_cpp.llama_sampler_init_temp(temp) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_temp_ext(self, t: float, delta: float, exponent: float): sampler = llama_cpp.llama_sampler_init_temp_ext(t, delta, exponent) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) + + def add_xtc(self, p: float, t: float, min_keep: int, seed: int): + sampler = llama_cpp.llama_sampler_init_xtc(p, t, min_keep, seed) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) + + def add_top_n_sigma(self, n: float): + sampler = llama_cpp.llama_sampler_init_top_n_sigma(n) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_mirostat(self, n_vocab: int, seed: int, tau: float, eta: float, m: int): sampler = llama_cpp.llama_sampler_init_mirostat(n_vocab, seed, tau, eta, m) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_mirostat_v2(self, seed: int, tau: float, eta: float): sampler = llama_cpp.llama_sampler_init_mirostat_v2(seed, tau, eta) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_grammar(self, model: LlamaModel, grammar: LlamaGrammar): sampler = llama_cpp.llama_sampler_init_grammar( model.vocab, grammar._grammar.encode("utf-8"), grammar._root.encode("utf-8") ) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) + + def add_grammar_lazy_patterns( + self, + model: LlamaModel, + grammar: LlamaGrammar, + trigger_patterns: List[str], + trigger_tokens: List[int] + ): + # Convert patterns to C array + pattern_ptrs = (ctypes.c_char_p * len(trigger_patterns))() + for i, pattern in enumerate(trigger_patterns): + pattern_ptrs[i] = pattern.encode("utf-8") + + # Convert tokens to C array + token_array = (llama_cpp.llama_token * len(trigger_tokens))(*trigger_tokens) + + sampler = llama_cpp.llama_sampler_init_grammar_lazy_patterns( + model.vocab, + grammar._grammar.encode("utf-8"), + grammar._root.encode("utf-8"), + pattern_ptrs, + len(trigger_patterns), + token_array, + len(trigger_tokens) + ) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_penalties( self, - n_vocab: int, - special_eos_id: int, - linefeed_id: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, - penalize_nl: bool, - ignore_eos: bool, ): sampler = llama_cpp.llama_sampler_init_penalties( penalty_last_n, @@ -830,50 +755,96 @@ def add_penalties( penalty_freq, penalty_present, ) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) - def init_logit_bias( - self, n_vocab: int, n_logit_bias, logit_bias: llama_cpp.llama_logit_bias_p + def add_dry( + self, + model: LlamaModel, + n_ctx_train: int, + dry_multiplier: float, + dry_base: float, + dry_allowed_length: int, + dry_penalty_last_n: int, + seq_breakers: List[str] + ): + # Convert seq_breakers to C array + breaker_ptrs = (ctypes.c_char_p * len(seq_breakers))() + for i, breaker in enumerate(seq_breakers): + breaker_ptrs[i] = breaker.encode("utf-8") + + sampler = llama_cpp.llama_sampler_init_dry( + model.vocab, + n_ctx_train, + dry_multiplier, + dry_base, + dry_allowed_length, + dry_penalty_last_n, + breaker_ptrs, + len(seq_breakers) + ) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) + + def add_logit_bias( + self, + n_vocab: int, + logit_bias: Dict[int, float] ): + # Convert logit_bias dict to C array + bias_array = (llama_cpp.llama_logit_bias * len(logit_bias))() + for i, (token, bias) in enumerate(logit_bias.items()): + bias_array[i].token = token + bias_array[i].bias = bias + sampler = llama_cpp.llama_sampler_init_logit_bias( - n_vocab, n_logit_bias, logit_bias + n_vocab, + len(logit_bias), + bias_array ) - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) + + def add_infill(self, model: LlamaModel): + sampler = llama_cpp.llama_sampler_init_infill(model.vocab) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_custom( self, apply_func: Callable[[llama_cpp.llama_token_data_array], None] ): custom_sampler = CustomSampler(apply_func) sampler = custom_sampler.get_sampler() - self._add_sampler(sampler) + llama_cpp.llama_sampler_chain_add(self.sampler, sampler) # NOTE: Must remove custom samplers before free or llama.cpp will try to free them self.custom_samplers.append( (llama_cpp.llama_sampler_chain_n(self.sampler) - 1, custom_sampler) ) - def _add_sampler(self, sampler: llama_cpp.llama_sampler_p): - assert self.sampler is not None - llama_cpp.llama_sampler_chain_add(self.sampler, sampler) - self.samplers.append(sampler) - def get_seed(self) -> int: - assert self.sampler is not None return llama_cpp.llama_sampler_get_seed(self.sampler) - def sample(self, ctx: LlamaContext, idx: int) -> int: - assert self.sampler is not None - assert ctx.ctx is not None + def sample(self, ctx: LlamaContext, idx: int = -1) -> int: return llama_cpp.llama_sampler_sample(self.sampler, ctx.ctx, idx) - def close(self): - if self.sampler: - # NOTE: Must remove custom samplers before free or llama.cpp will try to free them - for i, _ in reversed(self.custom_samplers): - llama_cpp.llama_sampler_chain_remove(self.sampler, i) - llama_cpp.llama_sampler_free(self.sampler) - self.sampler = None - self.samplers.clear() - self.custom_samplers.clear() + def accept(self, token: int): + llama_cpp.llama_sampler_accept(self.sampler, token) - def __del__(self): - self.close() + def reset(self): + llama_cpp.llama_sampler_reset(self.sampler) + + def clone(self): + # NOTE: Custom samplers cannot be cloned due to Python callback limitations + if self.custom_samplers: + raise NotImplementedError("Cannot clone LlamaSampler that contains custom samplers") + + cloned_sampler = llama_cpp.llama_sampler_clone(self.sampler) + # Create a new wrapper around the cloned sampler + new_sampler = LlamaSampler.__new__(LlamaSampler) + new_sampler.sampler = cloned_sampler + new_sampler.custom_samplers = [] + new_sampler._exit_stack = ExitStack() + + def free_sampler(): + if new_sampler.sampler is not None: + llama_cpp.llama_sampler_free(new_sampler.sampler) + new_sampler.sampler = None + + new_sampler._exit_stack.callback(free_sampler) + return new_sampler diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 7e9a6af23d..72834a0e8e 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -709,15 +709,15 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): sampler.add_custom(apply_func) sampler.add_penalties( - n_vocab=self._n_vocab, - special_eos_id=self._token_eos, - linefeed_id=self._token_nl, + # n_vocab=self._n_vocab, + # special_eos_id=self._token_eos, + # linefeed_id=self._token_nl, penalty_last_n=self.last_n_tokens_size, penalty_repeat=repeat_penalty, penalty_freq=frequency_penalty, penalty_present=presence_penalty, - penalize_nl=penalize_nl, - ignore_eos=False, + # penalize_nl=penalize_nl, + # ignore_eos=False, ) if grammar is not None: diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 63de3a93a4..d13d60458a 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -161,11 +161,11 @@ llama_context_p = NewType("llama_context_p", int) llama_context_p_ctypes = ctypes.c_void_p -# # struct llama_sampler; -# llama_sampler_p = NewType("llama_sampler_p", int) -# llama_sampler_p_ctypes = ctypes.c_void_p +# typedef struct llama_memory_i * llama_memory_t; +llama_memory_t = NewType("llama_memory_t", int) +llama_memory_t_ctypes = ctypes.c_void_p -# struct llama_kv_cache; +# struct llama_kv_cache; (DEPRECATED) llama_kv_cache_p = NewType("llama_kv_cache_p", int) llama_kv_cache_p_ctypes = ctypes.c_void_p @@ -237,6 +237,7 @@ # LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, # LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, # LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, +# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35, # }; LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 @@ -273,6 +274,7 @@ LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32 LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33 LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34 +LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35 # // note: these values should be synchronized with ggml_rope @@ -421,14 +423,14 @@ # LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, # LLAMA_ROPE_SCALING_TYPE_YARN = 2, # LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, -# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN, +# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE, # }; LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1 LLAMA_ROPE_SCALING_TYPE_NONE = 0 LLAMA_ROPE_SCALING_TYPE_LINEAR = 1 LLAMA_ROPE_SCALING_TYPE_YARN = 2 LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3 -LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN +LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE # enum llama_pooling_type { # LLAMA_POOLING_TYPE_UNSPECIFIED = -1, @@ -458,7 +460,7 @@ # enum llama_split_mode { # LLAMA_SPLIT_MODE_NONE = 0, // single GPU # LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs -# LLAMA_SPLIT_MODE_ROW = 2, // split rows across GPUs +# LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported # }; LLAMA_SPLIT_MODE_NONE = 0 LLAMA_SPLIT_MODE_LAYER = 1 @@ -532,18 +534,21 @@ class llama_token_data_array(ctypes.Structure): ) -# // Input data for llama_decode +# // Input data for llama_encode/llama_decode # // A llama_batch object can contain input about one or many sequences # // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens # // # // - token : the token ids of the input (used when embd is NULL) # // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) # // - pos : the positions of the respective token in the sequence -# // (if set to NULL, the token position will be tracked automatically by llama_decode) +# // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode) # // - seq_id : the sequence to which the respective token belongs # // (if set to NULL, the sequence ID will be assumed to be 0) # // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output -# // (if set to NULL, only the logits for last token will be returned) +# // (if set to NULL: +# // - if embeddings: all tokens are output +# // - if not: only the last token is output +# // ) # // # typedef struct llama_batch { # int32_t n_tokens; @@ -553,10 +558,10 @@ class llama_token_data_array(ctypes.Structure): # llama_pos * pos; # int32_t * n_seq_id; # llama_seq_id ** seq_id; -# int8_t * logits; // TODO: rename this to "output" +# int8_t * logits; // TODO: rename this to "output" # } llama_batch; class llama_batch(ctypes.Structure): - """Input data for llama_decode + """Input data for llama_encode/llama_decode A llama_batch object can contain input about one or many sequences @@ -660,10 +665,7 @@ class llama_model_kv_override(ctypes.Structure): # int32_t n_gpu_layers; // number of layers to store in VRAM # enum llama_split_mode split_mode; // how to split the model across multiple GPUs -# // main_gpu interpretation depends on split_mode: -# // LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model -# // LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results -# // LLAMA_SPLIT_MODE_LAYER: ignored +# // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() @@ -680,7 +682,6 @@ class llama_model_kv_override(ctypes.Structure): # // override key-value pairs of the model meta data # const struct llama_model_kv_override * kv_overrides; - # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights # bool use_mmap; // use mmap if possible @@ -695,7 +696,7 @@ class llama_model_params(ctypes.Structure): tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern n_gpu_layers (int): number of layers to store in VRAM split_mode (int): how to split the model across multiple GPUs - main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored + main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback @@ -738,7 +739,7 @@ class llama_model_params(ctypes.Structure): # // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations -# // https://github.com/ggerganov/llama.cpp/pull/7544 +# // https://github.com/ggml-org/llama.cpp/pull/7544 # struct llama_context_params { # uint32_t n_ctx; // text context, 0 = from model # uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode @@ -751,7 +752,7 @@ class llama_model_params(ctypes.Structure): # enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id # enum llama_attention_type attention_type; // attention type to use for embeddings -# // ref: https://github.com/ggerganov/llama.cpp/pull/2054 +# // ref: https://github.com/ggml-org/llama.cpp/pull/2054 # float rope_freq_base; // RoPE base frequency, 0 = from model # float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model # float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model @@ -759,7 +760,7 @@ class llama_model_params(ctypes.Structure): # float yarn_beta_fast; // YaRN low correction dim # float yarn_beta_slow; // YaRN high correction dim # uint32_t yarn_orig_ctx; // YaRN original context size -# float defrag_thold; // defragment the KV cache if holes/size > thold, < 0 disabled (default) +# float defrag_thold; // defragment the KV cache if holes/size > thold, <= 0 disabled (default) # ggml_backend_sched_eval_callback cb_eval; # void * cb_eval_user_data; @@ -767,19 +768,21 @@ class llama_model_params(ctypes.Structure): # enum ggml_type type_k; // data type for K cache [EXPERIMENTAL] # enum ggml_type type_v; // data type for V cache [EXPERIMENTAL] -# // Keep the booleans together to avoid misalignment during copy-by-value. -# bool logits_all; // the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) -# bool embeddings; // if true, extract embeddings (together with logits) -# bool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU -# bool flash_attn; // whether to use flash attention [EXPERIMENTAL] -# bool no_perf; // whether to measure performance timings - - # // Abort callback # // if it returns true, execution of llama_decode() will be aborted # // currently works only with CPU execution # ggml_abort_callback abort_callback; # void * abort_callback_data; + +# // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value. +# bool embeddings; // if true, extract embeddings (together with logits) +# bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU +# bool flash_attn; // use flash attention [EXPERIMENTAL] +# bool no_perf; // measure performance timings +# bool op_offload; // offload host tensor operations to device +# bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +# // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases +# // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 # }; class llama_context_params(ctypes.Structure): """Parameters for llama_context @@ -801,18 +804,19 @@ class llama_context_params(ctypes.Structure): yarn_beta_fast (float): YaRN low correction dim yarn_beta_slow (float): YaRN high correction dim yarn_orig_ctx (int): YaRN original context size - defrag_thold (float): defragment the KV cache if holes/size > thold, < 0 disabled (default) + defrag_thold (float): defragment the KV cache if holes/size > thold, <= 0 disabled (default) cb_eval (ggml_backend_sched_eval_callback): callback for scheduling eval cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval type_k (int): data type for K cache type_v (int): data type for V cache - logits_all (bool): the llama_decode() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead) + abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted + abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU flash_attn (bool): whether to use flash attention no_perf (bool): whether to measure performance timings - abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted - abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback + op_offload (bool): offload host tensor operations to device + swa_full (bool): use full-size SWA cache """ if TYPE_CHECKING: @@ -837,13 +841,14 @@ class llama_context_params(ctypes.Structure): cb_eval_user_data: ctypes.c_void_p type_k: int type_v: int - logits_all: bool + abort_callback: Callable[[ctypes.c_void_p], bool] + abort_callback_data: ctypes.c_void_p embeddings: bool offload_kqv: bool flash_attn: bool no_perf: bool - abort_callback: Callable[[ctypes.c_void_p], bool] - abort_callback_data: ctypes.c_void_p + op_offload: bool + swa_full: bool _fields_ = [ ("n_ctx", ctypes.c_uint32), @@ -867,13 +872,14 @@ class llama_context_params(ctypes.Structure): ("cb_eval_user_data", ctypes.c_void_p), ("type_k", ctypes.c_int), ("type_v", ctypes.c_int), - ("logits_all", ctypes.c_bool), + ("abort_callback", ggml_abort_callback), + ("abort_callback_data", ctypes.c_void_p), ("embeddings", ctypes.c_bool), ("offload_kqv", ctypes.c_bool), ("flash_attn", ctypes.c_bool), ("no_perf", ctypes.c_bool), - ("abort_callback", ggml_abort_callback), - ("abort_callback_data", ctypes.c_void_p), + ("op_offload", ctypes.c_bool), + ("swa_full", ctypes.c_bool), ] @@ -907,6 +913,7 @@ class llama_context_params(ctypes.Structure): # void * imatrix; // pointer to importance matrix data # void * kv_overrides; // pointer to vector containing overrides # void * tensor_types; // pointer to vector containing tensor types +# void * prune_layers; // pointer to vector containing layer indices to prune # } llama_model_quantize_params; class llama_model_quantize_params(ctypes.Structure): """Parameters for llama_model_quantize @@ -924,6 +931,7 @@ class llama_model_quantize_params(ctypes.Structure): imatrix (ctypes.c_void_p): pointer to importance matrix data kv_overrides (ctypes.c_void_p): pointer to vector containing overrides tensor_types (ctypes.c_void_p): pointer to vector containing tensor types + prune_layers (ctypes.c_void_p): pointer to vector containing layer indices to prune """ if TYPE_CHECKING: @@ -939,6 +947,7 @@ class llama_model_quantize_params(ctypes.Structure): imatrix: ctypes.c_void_p kv_overrides: ctypes.c_void_p tensor_types: ctypes.c_void_p + prune_layers: ctypes.c_void_p _fields_ = [ ("nthread", ctypes.c_int32), @@ -953,6 +962,7 @@ class llama_model_quantize_params(ctypes.Structure): ("imatrix", ctypes.c_void_p), ("kv_overrides", ctypes.c_void_p), ("tensor_types", ctypes.c_void_p), + ("prune_layers", ctypes.c_void_p), ] @@ -1063,7 +1073,6 @@ def llama_model_quantize_default_params() -> llama_model_quantize_params: # // Initialize the llama + ggml backend # // If numa is true, use NUMA optimizations # // Call once at the start of the program -# LLAMA_API void llama_backend_init(bool numa); # LLAMA_API void llama_backend_init(void); @ctypes_function( "llama_backend_init", @@ -1072,7 +1081,6 @@ def llama_model_quantize_default_params() -> llama_model_quantize_params: ) def llama_backend_init(): """Initialize the llama + ggml backend - If numa is true, use NUMA optimizations Call once at the start of the program""" ... @@ -1186,7 +1194,21 @@ def llama_model_load_from_splits( ... -# LLAMA_API void llama_free_model(struct llama_model * model); +# LLAMA_API void llama_model_save_to_file( +# const struct llama_model * model, +# const char * path_model); +@ctypes_function( + "llama_model_save_to_file", + [llama_model_p_ctypes, ctypes.c_char_p], + None, +) +def llama_model_save_to_file(model: llama_model_p, path_model: bytes, /): + """Save the model to a file""" + ... + + +# DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model), +# "use llama_model_free instead"); @ctypes_function( "llama_free_model", [llama_model_p_ctypes], @@ -1263,6 +1285,12 @@ def llama_max_devices() -> int: ... +# LLAMA_API size_t llama_max_parallel_sequences(void); +@ctypes_function("llama_max_parallel_sequences", [], ctypes.c_size_t) +def llama_max_parallel_sequences() -> int: + ... + + # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) def llama_supports_mmap() -> bool: @@ -1311,8 +1339,6 @@ def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... - - # DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_ctx_train(model: llama_model_p, /) -> int: @@ -1349,14 +1375,10 @@ def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... -# LLAMA_API struct llama_kv_cache * llama_get_kv_self ( struct llama_context * ctx); -@ctypes_function( - "llama_get_kv_self", - [llama_context_p_ctypes], - llama_kv_cache_p_ctypes, -) -def llama_get_kv_self(ctx: llama_context_p, /) -> Optional[llama_kv_cache_p]: - """Get the KV cache for self-attention""" +# LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx); +@ctypes_function("llama_get_memory", [llama_context_p_ctypes], llama_memory_t_ctypes) +def llama_get_memory(ctx: llama_context_p, /) -> Optional[llama_memory_t]: + """Get the memory for the context""" ... @@ -1366,6 +1388,17 @@ def llama_pooling_type(ctx: llama_context_p, /) -> int: ... +# DEPRECATED(LLAMA_API struct llama_kv_cache * llama_get_kv_self(struct llama_context * ctx), "use llama_get_memory instead"); +@ctypes_function( + "llama_get_kv_self", + [llama_context_p_ctypes], + llama_kv_cache_p_ctypes, +) +def llama_get_kv_self(ctx: llama_context_p, /) -> Optional[llama_kv_cache_p]: + """Get the KV cache for self-attention (DEPRECATED)""" + ... + + # LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); @ctypes_function("llama_model_get_vocab", [llama_model_p_ctypes], llama_vocab_p_ctypes) def llama_model_get_vocab(model: llama_model_p, /) -> Optional[llama_vocab_p]: @@ -1408,6 +1441,12 @@ def llama_model_n_head_kv(model: llama_model_p, /) -> int: ... +# LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model); +@ctypes_function("llama_model_n_swa", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_swa(model: llama_model_p, /) -> int: + ... + + # // Get the model's RoPE frequency scaling factor # LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); @ctypes_function("llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float) @@ -1415,9 +1454,26 @@ def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: ... +# // Returns the number of classifier outputs (only valid for classifier models) +# // Undefined behavior for non-classifier models +# LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model); +@ctypes_function("llama_model_n_cls_out", [llama_model_p_ctypes], ctypes.c_uint32) +def llama_model_n_cls_out(model: llama_model_p, /) -> int: + """Returns the number of classifier outputs (only valid for classifier models)""" + ... + + +# // Returns label of classifier output by index ( Optional[bytes]: + """Returns label of classifier output by index. Returns None if no label provided""" + ... + + # LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); -@ctypes_function("llama_vocab_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_vocab_type(model: llama_model_p, /) -> int: +@ctypes_function("llama_vocab_type", [llama_vocab_p_ctypes], ctypes.c_int) +def llama_vocab_type(vocab: llama_vocab_p, /) -> int: ... @@ -1615,6 +1671,10 @@ def llama_model_quantize( ... +# // +# // Adapters +# // + # // Load a LoRA adapter from file # LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init( # struct llama_model * model, @@ -1738,178 +1798,256 @@ def llama_apply_adapter_cvec( # // -# // KV cache +# // Memory # // +# // Clear the memory contents +# // If data == true, the data buffers will also be cleared together with the metadata +# LLAMA_API void llama_memory_clear( +# llama_memory_t mem, +# bool data); +@ctypes_function( + "llama_memory_clear", + [llama_memory_t_ctypes, ctypes.c_bool], + None, +) +def llama_memory_clear(mem: llama_memory_t, data: bool, /): + """Clear the memory contents + If data == true, the data buffers will also be cleared together with the metadata""" + ... -# // Information associated with an individual cell in the KV cache view. -# struct llama_kv_cache_view_cell { -# // The position for this cell. Takes KV cache shifts into account. -# // May be negative if the cell is not populated. -# llama_pos pos; -# }; -class llama_kv_cache_view_cell(ctypes.Structure): - """Information associated with an individual cell in the KV cache view. - - Attributes: - pos (llama_pos): The position for this cell. Takes KV cache shifts into account. - May be negative if the cell is not populated.""" - - if TYPE_CHECKING: - pos: llama_pos - - _fields_ = [("pos", llama_pos)] - - -# // An updateable view of the KV cache. -# struct llama_kv_cache_view { -# // Number of KV cache cells. This will be the same as the context size. -# int32_t n_cells; -# // Maximum number of sequences that can exist in a cell. It's not an error -# // if there are more sequences in a cell than this value, however they will -# // not be visible in the view cells_sequences. -# int32_t n_seq_max; +# // Removes all tokens that belong to the specified sequence and have positions in [p0, p1) +# // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails +# // seq_id < 0 : match any sequence +# // p0 < 0 : [0, p1] +# // p1 < 0 : [p0, inf) +# LLAMA_API bool llama_memory_seq_rm( +# llama_memory_t mem, +# llama_seq_id seq_id, +# llama_pos p0, +# llama_pos p1); +@ctypes_function( + "llama_memory_seq_rm", + [ + llama_memory_t_ctypes, + llama_seq_id, + llama_pos, + llama_pos, + ], + ctypes.c_bool, +) +def llama_memory_seq_rm( + mem: llama_memory_t, + seq_id: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + /, +) -> bool: + """Removes all tokens that belong to the specified sequence and have positions in [p0, p1) -# // Number of tokens in the cache. For example, if there are two populated -# // cells, the first with 1 sequence id in it and the second with 2 sequence -# // ids then you'll have 3 tokens. -# int32_t token_count; + Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails -# // Number of populated cache cells. -# int32_t used_cells; + seq_id < 0 : match any sequence + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... -# // Maximum contiguous empty slots in the cache. -# int32_t max_contiguous; -# // Index to the start of the max_contiguous slot range. Can be negative -# // when cache is full. -# int32_t max_contiguous_idx; +# // Copy all tokens that belong to the specified sequence to another sequence +# // p0 < 0 : [0, p1] +# // p1 < 0 : [p0, inf) +# LLAMA_API void llama_memory_seq_cp( +# llama_memory_t mem, +# llama_seq_id seq_id_src, +# llama_seq_id seq_id_dst, +# llama_pos p0, +# llama_pos p1); +@ctypes_function( + "llama_memory_seq_cp", + [ + llama_memory_t_ctypes, + llama_seq_id, + llama_seq_id, + llama_pos, + llama_pos, + ], + None, +) +def llama_memory_seq_cp( + mem: llama_memory_t, + seq_id_src: Union[llama_seq_id, int], + seq_id_dst: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + /, +): + """Copy all tokens that belong to the specified sequence to another sequence + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... -# // Information for an individual cell. -# struct llama_kv_cache_view_cell * cells; +# // Removes all tokens that do not belong to the specified sequence +# LLAMA_API void llama_memory_seq_keep( +# llama_memory_t mem, +# llama_seq_id seq_id); +@ctypes_function( + "llama_memory_seq_keep", [llama_memory_t_ctypes, llama_seq_id], None +) +def llama_memory_seq_keep(mem: llama_memory_t, seq_id: Union[llama_seq_id, int], /): + """Removes all tokens that do not belong to the specified sequence""" + ... -# // The sequences for each cell. There will be n_seq_max items per cell. -# llama_seq_id * cells_sequences; -# }; -class llama_kv_cache_view(ctypes.Structure): - if TYPE_CHECKING: - n_cells: int - n_max_seq: int - token_count: int - used_cells: int - max_contiguous: int - max_contiguous_idx: int - cells: CtypesArray[llama_kv_cache_view_cell] - cells_sequences: CtypesArray[llama_seq_id] - _fields_ = [ - ("n_cells", ctypes.c_int32), - ("n_max_seq", ctypes.c_int32), - ("token_count", ctypes.c_int32), - ("used_cells", ctypes.c_int32), - ("max_contiguous", ctypes.c_int32), - ("max_contiguous_idx", ctypes.c_int32), - ("cells", ctypes.POINTER(llama_kv_cache_view_cell)), - ("cells_sequences", ctypes.POINTER(llama_seq_id)), - ] +# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) +# // p0 < 0 : [0, p1] +# // p1 < 0 : [p0, inf) +# LLAMA_API void llama_memory_seq_add( +# llama_memory_t mem, +# llama_seq_id seq_id, +# llama_pos p0, +# llama_pos p1, +# llama_pos delta); +@ctypes_function( + "llama_memory_seq_add", + [ + llama_memory_t_ctypes, + llama_seq_id, + llama_pos, + llama_pos, + llama_pos, + ], + None, +) +def llama_memory_seq_add( + mem: llama_memory_t, + seq_id: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + delta: Union[llama_pos, int], + /, +): + """Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... -llama_kv_cache_view_p = ctypes.POINTER(llama_kv_cache_view) +# // Integer division of the positions by factor of `d > 1` +# // p0 < 0 : [0, p1] +# // p1 < 0 : [p0, inf) +# LLAMA_API void llama_memory_seq_div( +# llama_memory_t mem, +# llama_seq_id seq_id, +# llama_pos p0, +# llama_pos p1, +# int d); +@ctypes_function( + "llama_memory_seq_div", + [ + llama_memory_t_ctypes, + llama_seq_id, + llama_pos, + llama_pos, + ctypes.c_int, + ], + None, +) +def llama_memory_seq_div( + mem: llama_memory_t, + seq_id: Union[llama_seq_id, int], + p0: Union[llama_pos, int], + p1: Union[llama_pos, int], + d: Union[ctypes.c_int, int], + /, +): + """Integer division of the positions by factor of `d > 1` + p0 < 0 : [0, p1] + p1 < 0 : [p0, inf)""" + ... -# // Create an empty KV cache view. (use only for debugging purposes) -# LLAMA_API struct llama_kv_cache_view llama_kv_cache_view_init(const struct llama_context * ctx, int32_t n_seq_max); +# // Returns the smallest position present in the memory for the specified sequence +# // This is typically non-zero only for SWA caches +# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory +# // Return -1 if the sequence is empty +# LLAMA_API llama_pos llama_memory_seq_pos_min( +# llama_memory_t mem, +# llama_seq_id seq_id); @ctypes_function( - "llama_kv_cache_view_init", - [llama_context_p_ctypes, ctypes.c_int32], - llama_kv_cache_view, + "llama_memory_seq_pos_min", [llama_memory_t_ctypes, llama_seq_id], llama_pos ) -def llama_kv_cache_view_init( - ctx: llama_context_p, n_seq_max: Union[ctypes.c_int32, int], / -) -> llama_kv_cache_view: - """Create an empty KV cache view. (use only for debugging purposes)""" +def llama_memory_seq_pos_min( + mem: llama_memory_t, seq_id: Union[llama_seq_id, int], / +) -> int: + """Returns the smallest position present in the memory for the specified sequence + This is typically non-zero only for SWA caches + Return -1 if the sequence is empty""" ... -# // Free a KV cache view. (use only for debugging purposes) -# LLAMA_API void llama_kv_cache_view_free(struct llama_kv_cache_view * view); -@ctypes_function("llama_kv_cache_view_free", [llama_kv_cache_view_p], None) -def llama_kv_cache_view_free(view: "ctypes.pointer[llama_kv_cache_view]", /): # type: ignore - """Free a KV cache view. (use only for debugging purposes)""" +# // Returns the largest position present in the memory for the specified sequence +# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory +# // Return -1 if the sequence is empty +# LLAMA_API llama_pos llama_memory_seq_pos_max( +# llama_memory_t mem, +# llama_seq_id seq_id); +@ctypes_function( + "llama_memory_seq_pos_max", [llama_memory_t_ctypes, llama_seq_id], llama_pos +) +def llama_memory_seq_pos_max( + mem: llama_memory_t, seq_id: Union[llama_seq_id, int], / +) -> int: + """Returns the largest position present in the memory for the specified sequence + Return -1 if the sequence is empty""" ... -# // Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes) -# LLAMA_API void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_kv_cache_view * view); -@ctypes_function( - "llama_kv_cache_view_update", [llama_context_p_ctypes, llama_kv_cache_view_p], None -) -def llama_kv_cache_view_update(ctx: llama_context_p, view: CtypesPointerOrRef[llama_kv_cache_view], /): # type: ignore - """Update the KV cache view structure with the current state of the KV cache. (use only for debugging purposes)""" +# // Check if the memory supports shifting +# LLAMA_API bool llama_memory_can_shift(llama_memory_t mem); +@ctypes_function("llama_memory_can_shift", [llama_memory_t_ctypes], ctypes.c_bool) +def llama_memory_can_shift(mem: llama_memory_t, /) -> bool: + """Check if the memory supports shifting""" ... +# // +# // KV cache for self-attention (TODO: deprecate in favor of llama_memory) +# // + # // Returns the number of tokens in the KV cache (slow, use only for debug) # // If a KV cell has multiple sequences assigned to it, it will be counted multiple times -# LLAMA_API int32_t llama_kv_self_n_tokens(const struct llama_context * ctx); +# DEPRECATED(LLAMA_API int32_t llama_kv_self_n_tokens(const struct llama_context * ctx), +# "Use llama_kv_self_seq_pos_max() and llama_kv_self_seq_pos_min() instead (https://github.com/ggml-org/llama.cpp/issues/13793)"); @ctypes_function( "llama_kv_self_n_tokens", [llama_context_p_ctypes], ctypes.c_int32 ) def llama_kv_self_n_tokens(ctx: llama_context_p, /) -> int: - """Returns the number of tokens in the KV cache (slow, use only for debug) - If a KV cell has multiple sequences assigned to it, it will be counted multiple times - """ - ... - - -# DEPRECATED(LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx), -# "use llama_kv_self_n_tokens instead"); -@ctypes_function( - "llama_get_kv_cache_token_count", [llama_context_p_ctypes], ctypes.c_int32 -) -def llama_get_kv_cache_token_count(ctx: llama_context_p, /) -> int: - """Returns the number of tokens in the KV cache (slow, use only for debug) - If a KV cell has multiple sequences assigned to it, it will be counted multiple times - """ + """Returns the number of tokens in the KV cache (slow, use only for debug) (DEPRECATED)""" ... # // Returns the number of used KV cells (i.e. have at least one sequence assigned to them) -# LLAMA_API int32_t llama_kv_self_used_cells(const struct llama_context * ctx); +# DEPRECATED(LLAMA_API int32_t llama_kv_self_used_cells(const struct llama_context * ctx), +# "Use llama_kv_self_seq_pos_max() and llama_kv_self_seq_pos_min() instead (https://github.com/ggml-org/llama.cpp/issues/13793)"); @ctypes_function( "llama_kv_self_used_cells", [llama_context_p_ctypes], ctypes.c_int32 ) def llama_kv_self_used_cells(ctx: llama_context_p, /) -> int: - """Returns the number of used KV cells (i.e. have at least one sequence assigned to them)""" - ... - - -# DEPRECATED(LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx), -# "use llama_kv_self_used_cells instead"); -@ctypes_function( - "llama_get_kv_cache_used_cells", [llama_context_p_ctypes], ctypes.c_int32 -) -def llama_get_kv_cache_used_cells(ctx: llama_context_p, /) -> int: - """Returns the number of used KV cells (i.e. have at least one sequence assigned to them)""" + """Returns the number of used KV cells (DEPRECATED)""" ... # // Clear the KV cache - both cell info is erased and KV data is zeroed -# LLAMA_API void llama_kv_self_clear( -# struct llama_context * ctx); +# DEPRECATED(LLAMA_API void llama_kv_self_clear( +# struct llama_context * ctx), +# "Use llama_memory_clear() instead"); @ctypes_function( "llama_kv_self_clear", [llama_context_p_ctypes], None ) def llama_kv_self_clear(ctx: llama_context_p, /): - """Clear the KV cache - both cell info is erased and KV data is zeroed""" - ... - -# NOTE: Deprecated -@ctypes_function("llama_kv_self_clear", [llama_context_p_ctypes], None) -def llama_kv_cache_clear(ctx: llama_context_p, /): - """Clear the KV cache""" + """Clear the KV cache (DEPRECATED)""" ... @@ -1918,13 +2056,14 @@ def llama_kv_cache_clear(ctx: llama_context_p, /): # // seq_id < 0 : match any sequence # // p0 < 0 : [0, p1] # // p1 < 0 : [p0, inf) -# LLAMA_API bool llama_kv_cache_seq_rm( +# DEPRECATED(LLAMA_API bool llama_kv_self_seq_rm( # struct llama_context * ctx, # llama_seq_id seq_id, # llama_pos p0, -# llama_pos p1); +# llama_pos p1), +# "Use llama_memory_seq_rm() instead"); @ctypes_function( - "llama_kv_cache_seq_rm", + "llama_kv_self_seq_rm", [ llama_context_p_ctypes, llama_seq_id, @@ -1933,20 +2072,14 @@ def llama_kv_cache_clear(ctx: llama_context_p, /): ], ctypes.c_bool, ) -def llama_kv_cache_seq_rm( +def llama_kv_self_seq_rm( ctx: llama_context_p, seq_id: Union[llama_seq_id, int], p0: Union[llama_pos, int], p1: Union[llama_pos, int], /, ) -> bool: - """Removes all tokens that belong to the specified sequence and have positions in [p0, p1) - - Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails - - seq_id < 0 : match any sequence - p0 < 0 : [0, p1] - p1 < 0 : [p0, inf)""" + """Remove tokens from KV cache (DEPRECATED)""" ... @@ -1954,12 +2087,13 @@ def llama_kv_cache_seq_rm( # // Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence # // p0 < 0 : [0, p1] # // p1 < 0 : [p0, inf) -# LLAMA_API void llama_kv_self_seq_cp( +# DEPRECATED(LLAMA_API void llama_kv_self_seq_cp( # struct llama_context * ctx, # llama_seq_id seq_id_src, # llama_seq_id seq_id_dst, # llama_pos p0, -# llama_pos p1); +# llama_pos p1), +# "Use llama_memory_seq_cp() instead"); @ctypes_function( "llama_kv_self_seq_cp", [ @@ -1979,74 +2113,35 @@ def llama_kv_self_seq_cp( p1: Union[llama_pos, int], /, ): - """Copy all tokens that belong to the specified sequence to another sequence - Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence - p0 < 0 : [0, p1] - p1 < 0 : [p0, inf)""" - ... - - -# NOTE: Deprecated -@ctypes_function( - "llama_kv_self_seq_cp", - [ - llama_context_p_ctypes, - llama_seq_id, - llama_seq_id, - llama_pos, - llama_pos, - ], - None, -) -def llama_kv_cache_seq_cp( - ctx: llama_context_p, - seq_id_src: Union[llama_seq_id, int], - seq_id_dst: Union[llama_seq_id, int], - p0: Union[llama_pos, int], - p1: Union[llama_pos, int], - /, -): - """Copy all tokens that belong to the specified sequence to another sequence - Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence - p0 < 0 : [0, p1] - p1 < 0 : [p0, inf)""" + """Copy tokens in KV cache (DEPRECATED)""" ... # // Removes all tokens that do not belong to the specified sequence -# LLAMA_API void llama_kv_self_seq_keep( +# DEPRECATED(LLAMA_API void llama_kv_self_seq_keep( # struct llama_context * ctx, -# llama_seq_id seq_id); +# llama_seq_id seq_id), +# "Use llama_memory_seq_keep() instead"); @ctypes_function( "llama_kv_self_seq_keep", [llama_context_p_ctypes, llama_seq_id], None ) def llama_kv_self_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /): - """Removes all tokens that do not belong to the specified sequence""" - ... - - -# NOTE: Deprecated -@ctypes_function( - "llama_kv_self_seq_keep", [llama_context_p_ctypes, llama_seq_id], None -) -def llama_kv_cache_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /): - """Removes all tokens that do not belong to the specified sequence""" + """Keep only specified sequence in KV cache (DEPRECATED)""" ... - # // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) # // If the KV cache is RoPEd, the KV data is updated accordingly: # // - lazily on next llama_decode() -# // - explicitly with llama_kv_cache_update() # // p0 < 0 : [0, p1] # // p1 < 0 : [p0, inf) -# LLAMA_API void llama_kv_cache_seq_add( +# DEPRECATED(LLAMA_API void llama_kv_self_seq_add( # struct llama_context * ctx, # llama_seq_id seq_id, # llama_pos p0, # llama_pos p1, -# llama_pos delta); +# llama_pos delta), +# "Use llama_memory_seq_add() instead"); @ctypes_function( "llama_kv_self_seq_add", [ @@ -2066,66 +2161,22 @@ def llama_kv_self_seq_add( delta: Union[llama_pos, int], /, ): - """Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) - If the KV cache is RoPEd, the KV data is updated accordingly: - - lazily on next llama_decode() - - explicitly with llama_kv_cache_update() - p0 < 0 : [0, p1] - p1 < 0 : [p0, inf)""" + """Add delta to sequence positions in KV cache (DEPRECATED)""" ... -# // NOTE: Deprecated -# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) +# // Integer division of the positions by factor of `d > 1` # // If the KV cache is RoPEd, the KV data is updated accordingly: # // - lazily on next llama_decode() -# // - explicitly with llama_kv_cache_update() -# // p0 < 0 : [0, p1] -# // p1 < 0 : [p0, inf) -# LLAMA_API void llama_kv_cache_seq_add( -# struct llama_context * ctx, -# llama_seq_id seq_id, -# llama_pos p0, -# llama_pos p1, -# llama_pos delta); -@ctypes_function( - "llama_kv_self_seq_add", - [ - llama_context_p_ctypes, - llama_seq_id, - llama_pos, - llama_pos, - llama_pos, - ], - None, -) -def llama_kv_cache_seq_add( - ctx: llama_context_p, - seq_id: Union[llama_seq_id, int], - p0: Union[llama_pos, int], - p1: Union[llama_pos, int], - delta: Union[llama_pos, int], - /, -): - """Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) - If the KV cache is RoPEd, the KV data is updated accordingly: - - lazily on next llama_decode() - - explicitly with llama_kv_cache_update() - p0 < 0 : [0, p1] - p1 < 0 : [p0, inf)""" - ... - - -# // Integer division of the positions by factor of `d > 1` -# // If the KV cache is RoPEd, the KV data is updated accordingly # // p0 < 0 : [0, p1] # // p1 < 0 : [p0, inf) -# LLAMA_API void llama_kv_cache_seq_div( +# DEPRECATED(void llama_kv_self_seq_div( # struct llama_context * ctx, # llama_seq_id seq_id, # llama_pos p0, # llama_pos p1, -# int d); +# int d), +# "Use llama_memory_seq_div() instead"); @ctypes_function( "llama_kv_self_seq_div", [ @@ -2145,123 +2196,71 @@ def llama_kv_self_seq_div( d: Union[ctypes.c_int, int], /, ): - """Integer division of the positions by factor of `d > 1` - If the KV cache is RoPEd, the KV data is updated accordingly - p0 < 0 : [0, p1] - p1 < 0 : [p0, inf)""" + """Divide sequence positions in KV cache (DEPRECATED)""" ... -# // NOTE: Deprecated -# // Integer division of the positions by factor of `d > 1` -# // If the KV cache is RoPEd, the KV data is updated accordingly -# // p0 < 0 : [0, p1] -# // p1 < 0 : [p0, inf) -# LLAMA_API void llama_kv_cache_seq_div( +# // Returns the smallest position present in the KV cache for the specified sequence +# // This is typically non-zero only for SWA caches +# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the KV cache +# // Return -1 if the sequence is empty +# DEPRECATED(LLAMA_API llama_pos llama_kv_self_seq_pos_min( # struct llama_context * ctx, -# llama_seq_id seq_id, -# llama_pos p0, -# llama_pos p1, -# int d); +# llama_seq_id seq_id), +# "Use llama_memory_seq_pos_min() instead"); @ctypes_function( - "llama_kv_self_seq_div", - [ - llama_context_p_ctypes, - llama_seq_id, - llama_pos, - llama_pos, - ctypes.c_int, - ], - None, + "llama_kv_self_seq_pos_min", [llama_context_p_ctypes, llama_seq_id], llama_pos ) -def llama_kv_cache_seq_div( - ctx: llama_context_p, - seq_id: Union[llama_seq_id, int], - p0: Union[llama_pos, int], - p1: Union[llama_pos, int], - d: Union[ctypes.c_int, int], - /, -): - """Integer division of the positions by factor of `d > 1` - If the KV cache is RoPEd, the KV data is updated accordingly - p0 < 0 : [0, p1] - p1 < 0 : [p0, inf)""" +def llama_kv_self_seq_pos_min( + ctx: llama_context_p, seq_id: Union[llama_seq_id, int], / +) -> int: + """Returns the smallest position in KV cache for sequence (DEPRECATED)""" ... # // Returns the largest position present in the KV cache for the specified sequence -# LLAMA_API llama_pos llama_kv_self_seq_pos_max( +# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the KV cache +# // Return -1 if the sequence is empty +# DEPRECATED(LLAMA_API llama_pos llama_kv_self_seq_pos_max( # struct llama_context * ctx, -# llama_seq_id seq_id); +# llama_seq_id seq_id), +# "Use llama_memory_seq_pos_max() instead"); @ctypes_function( "llama_kv_self_seq_pos_max", [llama_context_p_ctypes, llama_seq_id], llama_pos ) def llama_kv_self_seq_pos_max( ctx: llama_context_p, seq_id: Union[llama_seq_id, int], / ) -> int: - """Returns the largest position present in the KV cache for the specified sequence""" + """Returns the largest position in KV cache for sequence (DEPRECATED)""" ... # // Defragment the KV cache # // This will be applied: # // - lazily on next llama_decode() -# // - explicitly with llama_kv_self_update() -# LLAMA_API void llama_kv_self_defrag(struct llama_context * ctx); +# DEPRECATED(LLAMA_API void llama_kv_self_defrag(struct llama_context * ctx), +# "simply remove this call, the context will automatically decide when to do a defragmentation based on 'defrag_thold'"); @ctypes_function("llama_kv_self_defrag", [llama_context_p_ctypes], None) def llama_kv_self_defrag(ctx: llama_context_p, /): - """Defragment the KV cache - This will be applied: - - lazily on next llama_decode() - - explicitly with llama_kv_cache_update()""" - ... - - -# NOTE: Deprecated -# // Defragment the KV cache -# // This will be applied: -# // - lazily on next llama_decode() -# // - explicitly with llama_kv_self_update() -# LLAMA_API void llama_kv_cache_defrag(struct llama_context * ctx); -@ctypes_function("llama_kv_cache_defrag", [llama_context_p_ctypes], None) -def llama_kv_cache_defrag(ctx: llama_context_p, /): - """Defragment the KV cache - This will be applied: - - lazily on next llama_decode() - - explicitly with llama_kv_cache_update()""" - ... - - -# // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) -# LLAMA_API void llama_kv_cache_update(struct llama_context * ctx); -@ctypes_function("llama_kv_self_update", [llama_context_p_ctypes], None) -def llama_kv_self_update(ctx: llama_context_p, /): - """Apply the KV cache updates (such as K-shifts, defragmentation, etc.)""" - ... - -# // NOTE: Deprecated -# // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) -# LLAMA_API void llama_kv_cache_update(struct llama_context * ctx); -@ctypes_function("llama_kv_self_update", [llama_context_p_ctypes], None) -def llama_kv_cache_update(ctx: llama_context_p, /): - """Apply the KV cache updates (such as K-shifts, defragmentation, etc.)""" + """Defragment the KV cache (DEPRECATED)""" ... # // Check if the context supports KV cache shifting -# LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx); +# DEPRECATED(LLAMA_API bool llama_kv_self_can_shift(const struct llama_context * ctx), +# "use llama_memory_can_shift() instead"); @ctypes_function("llama_kv_self_can_shift", [llama_context_p_ctypes], ctypes.c_bool) def llama_kv_self_can_shift(ctx: llama_context_p, /) -> bool: - """Check if the context supports KV cache shifting""" + """Check if the context supports KV cache shifting (DEPRECATED)""" ... -# // NOTE: Deprecated -# // Check if the context supports KV cache shifting -# LLAMA_API bool llama_kv_cache_can_shift(struct llama_context * ctx); -@ctypes_function("llama_kv_self_can_shift", [llama_context_p_ctypes], ctypes.c_bool) -def llama_kv_cache_can_shift(ctx: llama_context_p, /) -> bool: - """Check if the context supports KV cache shifting""" +# // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) +# DEPRECATED(LLAMA_API void llama_kv_self_update(struct llama_context * ctx), +# "simply remove this call, updates are applied lazily on the next llama_decode()"); +@ctypes_function("llama_kv_self_update", [llama_context_p_ctypes], None) +def llama_kv_self_update(ctx: llama_context_p, /): + """Apply the KV cache updates (DEPRECATED)""" ... @@ -2269,14 +2268,13 @@ def llama_kv_cache_can_shift(ctx: llama_context_p, /) -> bool: # // State / sessions # // - # // Returns the *actual* size in bytes of the state -# // (logits, embedding and kv_cache) +# // (logits, embedding and memory) # // Only use when saving the state, not when restoring it, otherwise the size may be too small. # LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); @ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_state_get_size(ctx: llama_context_p, /) -> int: - """Returns the *actual* size in bytes of the state (rng, logits, embedding and kv_cache) - will often be smaller after compacting tokens""" + """Returns the *actual* size in bytes of the state (logits, embedding and memory)""" ... @@ -2284,8 +2282,7 @@ def llama_state_get_size(ctx: llama_context_p, /) -> int: # "use llama_state_get_size instead"); @ctypes_function("llama_get_state_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_get_state_size(ctx: llama_context_p, /) -> int: - """Returns the maximum size in bytes of the state (rng, logits, embedding - and kv_cache) - will often be smaller after compacting tokens""" + """Returns the size in bytes of the state (DEPRECATED)""" ... @@ -2332,9 +2329,7 @@ def llama_state_get_data( def llama_copy_state_data( ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], / ) -> int: - """Copies the state to the specified destination address. - Destination needs to have allocated enough memory. - Returns the number of bytes copied""" + """Copies the state to the specified destination address (DEPRECATED)""" ... @@ -2372,7 +2367,7 @@ def llama_state_set_data( def llama_set_state_data( ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], / ) -> int: - """Set the state reading from the specified address""" + """Set the state reading from the specified address (DEPRECATED)""" ... @@ -2421,7 +2416,7 @@ def llama_state_load_file( ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t), ], - ctypes.c_size_t, + ctypes.c_bool, ) def llama_load_session_file( ctx: llama_context_p, @@ -2430,7 +2425,7 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: +) -> bool: ... @@ -2473,7 +2468,7 @@ def llama_state_save_file( llama_token_p, ctypes.c_size_t, ], - ctypes.c_size_t, + ctypes.c_bool, ) def llama_save_session_file( ctx: llama_context_p, @@ -2481,11 +2476,11 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: +) -> bool: ... -# // Get the exact size needed to copy the KV cache of a single sequence +# // Get the exact size needed to copy the state of a single sequence # LLAMA_API size_t llama_state_seq_get_size( # struct llama_context * ctx, # llama_seq_id seq_id); @@ -2495,11 +2490,11 @@ def llama_save_session_file( ctypes.c_size_t, ) def llama_state_seq_get_size(ctx: llama_context_p, seq_id: llama_seq_id, /) -> int: - """Get the exact size needed to copy the KV cache of a single sequence""" + """Get the exact size needed to copy the state of a single sequence""" ... -# // Copy the KV cache of a single sequence into the specified buffer +# // Copy the state of a single sequence into the specified buffer # LLAMA_API size_t llama_state_seq_get_data( # struct llama_context * ctx, # uint8_t * dst, @@ -2522,7 +2517,7 @@ def llama_state_seq_get_data( seq_id: llama_seq_id, /, ) -> int: - """Copy the KV cache of a single sequence into the specified buffer""" + """Copy the state of a single sequence into the specified buffer""" ... @@ -2552,7 +2547,7 @@ def llama_state_seq_set_data( dest_seq_id: llama_seq_id, /, ) -> int: - """Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence""" + """Copy the sequence data into the specified sequence""" ... @@ -2619,7 +2614,6 @@ def llama_state_seq_load_file( # // Decoding # // - # // Return batch for single sequence of tokens # // The sequence ID will be fixed to 0 # // The position of the tokens will be tracked automatically by llama_decode @@ -2642,7 +2636,7 @@ def llama_batch_get_one( n_tokens: Union[ctypes.c_int, int], /, ) -> llama_batch: - """Return batch for single sequence of tokens starting at pos_0 + """Return batch for single sequence of tokens NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it """ @@ -2687,35 +2681,46 @@ def llama_batch_free(batch: llama_batch, /): ... -# // Processes a batch of tokens with the ecoder part of the encoder-decoder model. -# // Stores the encoder output internally for later use by the decoder cross-attention layers. +# // Process a batch of tokens. +# // In contrast to llama_decode() - this call does not use KV cache. +# // For encode-decoder contexts, processes the batch using the encoder. +# // Can store the encoder output internally for later use by the decoder's cross-attention layers. # // 0 - success -# // < 0 - error +# // < 0 - error. the memory state is restored to the state before this call # LLAMA_API int32_t llama_encode( # struct llama_context * ctx, # struct llama_batch batch); @ctypes_function("llama_encode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: - """Processes a batch of tokens with the ecoder part of the encoder-decoder model. - Stores the encoder output internally for later use by the decoder cross-attention layers. + """Process a batch of tokens using the encoder. 0 - success < 0 - error""" ... +# // Process a batch of tokens. +# // Requires the context to have a memory. +# // For encode-decoder contexts, processes the batch using the decoder. # // Positive return values does not mean a fatal error, but rather a warning. -# // 0 - success -# // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) -# // < 0 - error +# // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context +# // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() +# // Upon other return values, the memory state is restored to the state before this call +# // 0 - success +# // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) +# // 2 - aborted (processed ubatches will remain in the context's memory) +# // -1 - invalid input batch +# // < -1 - fatal error (processed ubatches will remain in the context's memory) # LLAMA_API int32_t llama_decode( # struct llama_context * ctx, # struct llama_batch batch); @ctypes_function("llama_decode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int: - """Positive return values does not mean a fatal error, but rather a warning. + """Process a batch of tokens. 0 - success 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) - < 0 - error""" + 2 - aborted (processed ubatches will remain in the context's memory) + -1 - invalid input batch + < -1 - fatal error (processed ubatches will remain in the context's memory)""" ... @@ -2761,13 +2766,12 @@ def llama_n_threads_batch(ctx: llama_context_p, /) -> int: ... -# // Set whether the model is in embeddings mode or not -# // If true, embeddings will be returned but logits will not +# // Set whether the context outputs embeddings or not +# // TODO: rename to avoid confusion with llama_get_embeddings() # LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); @ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None) def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /): - """Set whether the model is in embeddings model or not - If true, embeddings will be returned but logits will not""" + """Set whether the context outputs embeddings or not""" ... @@ -2896,7 +2900,7 @@ def llama_get_embeddings_ith( # // Get the embeddings for a sequence id # // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE -# // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[1] with the rank of the sequence +# // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence # // otherwise: float[n_embd] (1-dimensional) # LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); @ctypes_function( @@ -2917,7 +2921,6 @@ def llama_get_embeddings_seq( # // Vocab # // - # LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token); @ctypes_function( "llama_vocab_get_text", [llama_vocab_p_ctypes, llama_token], ctypes.c_char_p @@ -2971,8 +2974,6 @@ def llama_vocab_is_control( # // Special tokens - - # LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence @ctypes_function("llama_vocab_bos", [llama_vocab_p_ctypes], llama_token) def llama_vocab_bos(vocab: llama_vocab_p, /) -> llama_token: @@ -3014,6 +3015,7 @@ def llama_vocab_pad(vocab: llama_vocab_p, /) -> llama_token: """padding""" ... + # LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_get_add_bos", @@ -3034,6 +3036,16 @@ def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: ... +# LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab); +@ctypes_function( + "llama_vocab_get_add_sep", + [llama_vocab_p_ctypes], + ctypes.c_bool, +) +def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: + ... + + # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_fim_pre", @@ -3094,7 +3106,7 @@ def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ... - +# DEPRECATED functions # DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead"); @ctypes_function( "llama_token_get_text", @@ -3308,11 +3320,11 @@ def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: # // The API is thread-safe. # // - # /// @details Convert the provided text into tokens. # /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. # /// @return Returns the number of tokens on success, no more than n_tokens_max # /// @return Returns a negative number on failure - the number of tokens that would have been returned +# /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit) # /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so. # /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated # /// as plaintext. Does not insert a leading space. @@ -3355,7 +3367,7 @@ def llama_tokenize( text_len: The length of the text. tokens: The tokens pointer must be large enough to hold the resulting tokens. n_max_tokens: The maximum number of tokens to return. - add_special: Allow adding special tokenns if the model is configured to do so. + add_special: Allow adding special tokens if the model is configured to do so. parse_special: Allow parsing special tokens. Returns: @@ -3413,23 +3425,6 @@ def llama_token_to_piece( ... -# # // check if token0 is contained as a prefix in token1 -# # LLAMA_API bool llama_token_is_prefix( -# # const struct llama_model * model, -# # llama_token token0, -# # llama_token token1); -# @ctypes_function( -# "llama_token_is_prefix", -# [llama_model_p_ctypes, llama_token, llama_token], -# ctypes.c_bool, -# ) -# def llama_token_is_prefix( -# model: llama_model_p, token0: Union[llama_token, int], token1: Union[llama_token, int], / -# ) -> bool: -# """Check if token0 is contained as a prefix in token1""" -# ... - - # /// @details Convert the provided tokens into text (inverse of llama_tokenize()). # /// @param text The char pointer must be large enough to hold the resulting text. # /// @return Returns the number of chars/bytes on success, no more than text_len_max. @@ -3437,7 +3432,7 @@ def llama_token_to_piece( # /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. # /// @param unparse_special If true, special tokens are rendered in the output. # LLAMA_API int32_t llama_detokenize( -# const struct llama_model * model, +# const struct llama_vocab * vocab, # const llama_token * tokens, # int32_t n_tokens, # char * text, @@ -3447,7 +3442,7 @@ def llama_token_to_piece( @ctypes_function( "llama_detokenize", [ - llama_model_p_ctypes, + llama_vocab_p_ctypes, ctypes.POINTER(llama_token), ctypes.c_int32, ctypes.c_char_p, @@ -3458,7 +3453,7 @@ def llama_token_to_piece( ctypes.c_int32, ) def llama_detokenize( - model: llama_model_p, + vocab: llama_vocab_p, tokens: CtypesArray[llama_token], n_tokens: Union[ctypes.c_int, int], text: bytes, @@ -3470,7 +3465,7 @@ def llama_detokenize( """Convert the provided tokens into text (inverse of llama_tokenize()). Args: - model: The model to use for tokenization. + vocab: The vocabulary to use for tokenization. tokens: The tokens to convert. n_tokens: The number of tokens. text: The buffer to write the text to. @@ -3484,11 +3479,10 @@ def llama_detokenize( # // Chat templates # // - # /// Apply chat template. Inspired by hf apply_chat_template() on python. # /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model" -# /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggerganov/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template -# /// @param tmpl A Jinja template to use for this chat. If this is nullptr, the model’s default chat template will be used instead. +# /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template +# /// @param tmpl A Jinja template to use for this chat. If this is nullptr, the model's default chat template will be used instead. # /// @param chat Pointer to a list of multiple llama_chat_message # /// @param n_msg Number of llama_chat_message in this chat # /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message. @@ -3570,41 +3564,6 @@ def llama_chat_builtin_templates( # // # // Sampling API # // -# // Sample usage: -# // -# // // prepare the sampling chain at the start -# // auto sparams = llama_sampler_chain_default_params(); -# // -# // llama_sampler * smpl = llama_sampler_chain_init(sparams); -# // -# // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50)); -# // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1)); -# // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8)); -# // -# // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat" -# // // this sampler will be responsible to select the actual token -# // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); -# // -# // ... -# // -# // // decoding loop: -# // while (...) { -# // ... -# // -# // llama_decode(ctx, batch); -# // -# // // sample from the logits of the last token in the batch -# // const llama_token id = llama_sampler_sample(smpl, ctx, -1); -# // -# // // accepting the token updates the internal state of certain samplers (e.g. grammar, repetition, etc.) -# // llama_sampler_accept(smpl, id); -# // ... -# // } -# // -# // llama_sampler_free(smpl); -# // -# // TODO: In the future, llama_sampler will be utilized to offload the sampling to the backends (e.g. GPU). -# // # typedef void * llama_sampler_context_t; llama_sampler_context_t = ctypes.c_void_p @@ -3618,7 +3577,7 @@ def llama_chat_builtin_templates( # void (*reset) ( struct llama_sampler * smpl); // can be NULL # struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL # void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL -# + # // TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph # //void (*apply_ggml) (struct llama_sampler * smpl, ...); # }; @@ -3627,8 +3586,8 @@ class llama_sampler_i(ctypes.Structure): # struct llama_sampler { -# const struct llama_sampler_i * iface; -# llama_sampler_context_t ctx; +# const struct llama_sampler_i * iface; +# llama_sampler_context_t ctx; # }; class llama_sampler(ctypes.Structure): _fields_ = [ @@ -3739,7 +3698,7 @@ def llama_sampler_free(smpl: llama_sampler_p, /): # // llama_sampler_chain # // a type of llama_sampler that can chain multiple samplers one after another -# + # LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params); @ctypes_function( "llama_sampler_chain_init", @@ -3797,7 +3756,7 @@ def llama_sampler_chain_remove( # // available samplers: -# + # LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void); @ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes) def llama_sampler_init_greedy() -> llama_sampler_p: @@ -3813,7 +3772,7 @@ def llama_sampler_init_dist(seed: int) -> llama_sampler_p: # /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. # /// NOTE: Avoid using on the full vocabulary as the sorting can become slow. For example, apply top-k or top-p sampling first. # DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void), -# "will be removed in the future (see https://github.com/ggerganov/llama.cpp/pull/9896#discussion_r1800920915)"); +# "will be removed in the future (see https://github.com/ggml-org/llama.cpp/pull/9896#discussion_r1800920915)"); @ctypes_function("llama_sampler_init_softmax", [], llama_sampler_p_ctypes) def llama_sampler_init_softmax() -> llama_sampler_p: ... @@ -3838,7 +3797,7 @@ def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: ... -# /// @details Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841 +# /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841 # LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep); @ctypes_function( "llama_sampler_init_min_p", @@ -3860,6 +3819,7 @@ def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: ... +# /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf # LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); @ctypes_function("llama_sampler_init_temp", [ctypes.c_float], llama_sampler_p_ctypes) def llama_sampler_init_temp(t: float) -> llama_sampler_p: @@ -3904,11 +3864,6 @@ def llama_sampler_init_top_n_sigma(n: float, /) -> llama_sampler_p: # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. -# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. -# /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. -# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. -# /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. -# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. # LLAMA_API struct llama_sampler * llama_sampler_init_mirostat( # int32_t n_vocab, # uint32_t seed, @@ -3927,10 +3882,6 @@ def llama_sampler_init_mirostat( # /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. -# /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. -# /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. -# /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. -# /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. # LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2( # uint32_t seed, # float tau, @@ -3947,9 +3898,6 @@ def llama_sampler_init_mirostat_v2( # /// @details Intializes a GBNF grammar, see grammars/README.md for details. -# /// @param vocab The vocabulary that this grammar will be used with. -# /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. -# /// @param grammar_root The name of the start symbol for the grammar. # LLAMA_API struct llama_sampler * llama_sampler_init_grammar( # const struct llama_vocab * vocab, # const char * grammar_str, @@ -3965,9 +3913,42 @@ def llama_sampler_init_grammar( ... +# DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy( +# const struct llama_vocab * vocab, +# const char * grammar_str, +# const char * grammar_root, +# const char ** trigger_words, +# size_t num_trigger_words, +# const llama_token * trigger_tokens, +# size_t num_trigger_tokens), +# "use llama_sampler_init_grammar_lazy_patterns instead"); +@ctypes_function( + "llama_sampler_init_grammar_lazy", + [ + llama_vocab_p_ctypes, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.POINTER(ctypes.c_char_p), + ctypes.c_size_t, + ctypes.POINTER(llama_token), + ctypes.c_size_t, + ], + llama_sampler_p_ctypes, +) +def llama_sampler_init_grammar_lazy( + vocab: llama_vocab_p, + grammar_str: bytes, + grammar_root: bytes, + trigger_words: CtypesArray[bytes], + num_trigger_words: int, + trigger_tokens: CtypesArray[llama_token], + num_trigger_tokens: int, + /, +) -> llama_sampler_p: + ... + + # /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 -# /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. -# /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. # LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( # const struct llama_vocab * vocab, # const char * grammar_str, @@ -4077,26 +4058,6 @@ def llama_sampler_init_logit_bias( # // this sampler is meant to be used for fill-in-the-middle infilling -# // it's supposed to be used after top_k + top_p sampling -# // -# // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG -# // 2. combine probs of tokens that have the same prefix -# // -# // example: -# // -# // - before: -# // "hel": 0.5 -# // "hell": 0.2 -# // "hello": 0.1 -# // "dummy": 0.1 -# // -# // - after: -# // "hel": 0.8 -# // "dummy": 0.1 -# // -# // 3. discard non-EOG tokens with low prob -# // 4. if no tokens are left -> pick EOT -# // # LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab); @ctypes_function( "llama_sampler_init_infill", @@ -4119,15 +4080,6 @@ def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: # /// @details Sample and accept a token from the idx-th output of the last evaluation -# // -# // Shorthand for: -# // const auto * logits = llama_get_logits_ith(ctx, idx); -# // llama_token_data_array cur_p = { ... init from logits ... }; -# // llama_sampler_apply(smpl, &cur_p); -# // auto token = cur_p.data[cur_p.selected].id; -# // llama_sampler_accept(smpl, token); -# // return token; -# // Returns the sampled token # LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx); @ctypes_function( "llama_sampler_sample", @@ -4144,10 +4096,7 @@ def llama_sampler_sample( # // Model split # // - # /// @details Build a split GGUF final path for this chunk. -# /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" -# // Returns the split_path length. # LLAMA_API int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count); @ctypes_function( "llama_split_path", @@ -4167,8 +4116,6 @@ def llama_split_path( # /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. -# /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" -# // Returns the split_prefix length. # LLAMA_API int llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int split_no, int split_count); @ctypes_function( "llama_split_prefix", @@ -4216,16 +4163,13 @@ def llama_log_set( # // # // Performance utils # // -# // NOTE: Used by llama.cpp examples, avoid using in third-party apps. Instead, do your own performance measurements. -# // - # struct llama_perf_context_data { # double t_start_ms; # double t_load_ms; # double t_p_eval_ms; # double t_eval_ms; -# + # int32_t n_p_eval; # int32_t n_eval; # }; @@ -4242,7 +4186,7 @@ class llama_perf_context_data(ctypes.Structure): # struct llama_perf_sampler_data { # double t_sample_ms; -# + # int32_t n_sample; # }; class llama_perf_sampler_data(ctypes.Structure): @@ -4313,3 +4257,83 @@ def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... +# // +# // training +# // + +# // function that returns whether or not a given tensor contains trainable parameters +# typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata); +llama_opt_param_filter = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p) + +# // always returns true +# LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata); +@ctypes_function( + "llama_opt_param_filter_all", + [ctypes.c_void_p, ctypes.c_void_p], + ctypes.c_bool, +) +def llama_opt_param_filter_all(tensor: ctypes.c_void_p, userdata: ctypes.c_void_p, /) -> bool: + ... + + +# struct llama_opt_params { +# uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0 + +# llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters +# void * param_filter_ud; // userdata for determining which tensors contain trainable parameters + +# ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters +# void * get_opt_pars_ud; // userdata for calculating optimizer parameters +# }; +class llama_opt_params(ctypes.Structure): + _fields_ = [ + ("n_ctx_train", ctypes.c_uint32), + ("param_filter", llama_opt_param_filter), + ("param_filter_ud", ctypes.c_void_p), + ("get_opt_pars", ctypes.c_void_p), # ggml_opt_get_optimizer_params - not implemented here + ("get_opt_pars_ud", ctypes.c_void_p), + ] + + +# LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params); +@ctypes_function( + "llama_opt_init", + [llama_context_p_ctypes, llama_model_p_ctypes, llama_opt_params], + None, +) +def llama_opt_init(lctx: llama_context_p, model: llama_model_p, lopt_params: llama_opt_params, /): + ... + + +# LLAMA_API void llama_opt_epoch( +# struct llama_context * lctx, +# ggml_opt_dataset_t dataset, +# ggml_opt_result_t result_train, +# ggml_opt_result_t result_eval, +# int64_t idata_split, +# ggml_opt_epoch_callback callback_train, +# ggml_opt_epoch_callback callback_eval); +@ctypes_function( + "llama_opt_epoch", + [ + llama_context_p_ctypes, + ctypes.c_void_p, # ggml_opt_dataset_t + ctypes.c_void_p, # ggml_opt_result_t + ctypes.c_void_p, # ggml_opt_result_t + ctypes.c_int64, + ctypes.c_void_p, # ggml_opt_epoch_callback + ctypes.c_void_p, # ggml_opt_epoch_callback + ], + None, +) +def llama_opt_epoch( + lctx: llama_context_p, + dataset: ctypes.c_void_p, + result_train: ctypes.c_void_p, + result_eval: ctypes.c_void_p, + idata_split: int, + callback_train: ctypes.c_void_p, + callback_eval: ctypes.c_void_p, + /, +): + ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8733e0cf6e..8846aace49 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8733e0cf6eefc7c7752297cc22d0836706f4222c +Subproject commit 8846aace4934ad29651ea61b8c7e3f6b0556e3d2 From 51dce7417caac2f7e5ef56ba6329a8de760ef106 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 1 Jul 2025 01:58:11 -0400 Subject: [PATCH 406/561] misc: Fix support for new parameters, deprecate rpc_servers parameter --- llama_cpp/llama.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 72834a0e8e..428f049b9f 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -66,7 +66,6 @@ def __init__( split_mode: int = llama_cpp.LLAMA_SPLIT_MODE_LAYER, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, - rpc_servers: Optional[str] = None, vocab_only: bool = False, use_mmap: bool = True, use_mlock: bool = False, @@ -93,6 +92,8 @@ def __init__( embedding: bool = False, offload_kqv: bool = True, flash_attn: bool = False, + op_offloat: bool | None = None, + swa_full: bool | None = None, # Sampling Params no_perf: bool = False, last_n_tokens_size: int = 64, @@ -150,7 +151,6 @@ def __init__( split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options. main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. - rpc_servers: Comma separated list of RPC servers to use for offloading vocab_only: Only load the vocabulary no weights. use_mmap: Use mmap if possible. use_mlock: Force the system to keep the model in RAM. @@ -174,6 +174,8 @@ def __init__( embedding: Embedding mode only. offload_kqv: Offload K, Q, V to GPU. flash_attn: Use flash attention. + op_offloat: offload host tensor operations to device + swa_full: use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) no_perf: Measure performance timings. last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque. lora_base: Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model. @@ -226,11 +228,6 @@ def __init__( ) # 0x7FFFFFFF is INT32 max, will be auto set to all layers self.model_params.split_mode = split_mode self.model_params.main_gpu = main_gpu - if rpc_servers is not None: - self.model_params.rpc_servers = rpc_servers.encode("utf-8") - self._rpc_servers = rpc_servers - else: - self._rpc_servers = None self.tensor_split = tensor_split self._c_tensor_split = None if self.tensor_split is not None: @@ -341,12 +338,17 @@ def __init__( yarn_beta_slow if yarn_beta_slow != 0.0 else 0 ) self.context_params.yarn_orig_ctx = yarn_orig_ctx if yarn_orig_ctx != 0 else 0 - self.context_params.logits_all = ( - logits_all if draft_model is None else True - ) # Must be set to True for speculative decoding + self._logits_all = logits_all if draft_model is None else True self.context_params.embeddings = embedding # TODO: Rename to embeddings self.context_params.offload_kqv = offload_kqv self.context_params.flash_attn = flash_attn + + if op_offloat is not None: + self.context_params.op_offloat = op_offloat + + if swa_full is not None: + self.context_params.swa_full = swa_full + # KV cache quantization if type_k is not None: self.context_params.type_k = type_k @@ -568,7 +570,7 @@ def eval_tokens(self) -> Deque[int]: def eval_logits(self) -> Deque[List[float]]: return deque( self.scores[: self.n_tokens, :].tolist(), - maxlen=self._n_ctx if self.context_params.logits_all else 1, + maxlen=self._n_ctx if self._logits_all else 1, ) def tokenize( @@ -641,13 +643,13 @@ def eval(self, tokens: Sequence[int]): n_past = self.n_tokens n_tokens = len(batch) self._batch.set_batch( - batch=batch, n_past=n_past, logits_all=self.context_params.logits_all + batch=batch, n_past=n_past, logits_all=self._logits_all ) self._ctx.decode(self._batch) # Save tokens self.input_ids[n_past : n_past + n_tokens] = batch # Save logits - if self.context_params.logits_all: + if self._logits_all: rows = n_tokens cols = self._n_vocab logits = np.ctypeslib.as_array( @@ -1288,7 +1290,7 @@ def logit_bias_processor( else: stop_sequences = [] - if logprobs is not None and self.context_params.logits_all is False: + if logprobs is not None and self._logits_all is False: raise ValueError( "logprobs is not supported for models created with logits_all=False" ) @@ -2091,10 +2093,12 @@ def __getstate__(self): yarn_beta_fast=self.context_params.yarn_beta_fast, yarn_beta_slow=self.context_params.yarn_beta_slow, yarn_orig_ctx=self.context_params.yarn_orig_ctx, - logits_all=self.context_params.logits_all, + logits_all=self._logits_all, embedding=self.context_params.embeddings, offload_kqv=self.context_params.offload_kqv, flash_attn=self.context_params.flash_attn, + op_offloat=self.context_params.op_offloat, + swa_full=self.context_params.swa_full, # Sampling Params no_perf=self.context_params.no_perf, last_n_tokens_size=self.last_n_tokens_size, From 5a635f49d0acc61ca2d5a0c2d9a762bd093c4361 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 1 Jul 2025 02:01:09 -0400 Subject: [PATCH 407/561] fix(minor): Fix type hint for older versions of python --- llama_cpp/llama.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 428f049b9f..cdc05c7ad8 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -92,8 +92,8 @@ def __init__( embedding: bool = False, offload_kqv: bool = True, flash_attn: bool = False, - op_offloat: bool | None = None, - swa_full: bool | None = None, + op_offloat: Optional[bool] = None, + swa_full: Optional[bool] = None, # Sampling Params no_perf: bool = False, last_n_tokens_size: int = 64, From 0dec788309dd9dcf70cecf3564d85c736d2ff61e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 1 Jul 2025 02:30:16 -0400 Subject: [PATCH 408/561] fix: Fix missing deprecated symbols on windows with missing LLAMA_API prefix in header file --- CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 260222bde2..505c024b26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,7 +96,15 @@ if (LLAMA_BUILD) set(GGML_METAL_EMBED_LIBRARY "ON" CACHE BOOL "ggml: embed metal library" FORCE) endif() + add_subdirectory(vendor/llama.cpp) + + if (WIN32) + if (TARGET llama) + set_target_properties(llama PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) + endif() + endif() + llama_cpp_python_install_target(llama) llama_cpp_python_install_target(ggml) From cd548bd0f14210627798237d5c2ea78acfb88ccb Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 3 Jul 2025 01:57:43 -0400 Subject: [PATCH 409/561] feat: Add support for new mtmd api, add Qwen2.5-VL chat handler --- CMakeLists.txt | 83 +++++---- llama_cpp/llama_chat_format.py | 301 ++++++++++++++++++++++++--------- llama_cpp/mtmd_cpp.py | 280 ++++++++++++++++++++++++++++++ llama_cpp/server/model.py | 14 ++ 4 files changed, 554 insertions(+), 124 deletions(-) create mode 100644 llama_cpp/mtmd_cpp.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 505c024b26..4b06d98b32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,46 +143,45 @@ if (LLAMA_BUILD) ) endif() - # if (LLAVA_BUILD) - # if (LLAMA_CUBLAS OR LLAMA_CUDA) - # add_compile_definitions(GGML_USE_CUBLAS) - # add_compile_definitions(GGML_USE_CUDA) - # endif() - # - # if (LLAMA_METAL) - # add_compile_definitions(GGML_USE_METAL) - # endif() - # - # # Building llava - # add_subdirectory(vendor/llama.cpp/tools/mtmd) - # set_target_properties(llava_shared PROPERTIES OUTPUT_NAME "llava") - # - # if (WIN32) - # set_target_properties(llava_shared PROPERTIES CUDA_ARCHITECTURES OFF) - # endif() - # llama_cpp_python_install_target(llava_shared) - # if (WIN32) - # install( - # FILES $ - # DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib - # ) - # install( - # FILES $ - # DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib - # ) - # endif() - # - # # Fix for llava build: Add include directory for llama.h - # # Move these commands after the add_subdirectory call - # target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) - # target_include_directories(llava PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) - # - # if (BUILD_SHARED_LIBS) - # target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) - # target_include_directories(llava_shared PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) - # endif() - # - # target_include_directories(llama-llava-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) - # target_include_directories(llama-minicpmv-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) - # endif() + if (LLAVA_BUILD) + if (LLAMA_CUBLAS OR LLAMA_CUDA) + add_compile_definitions(GGML_USE_CUBLAS) + add_compile_definitions(GGML_USE_CUDA) + endif() + + if (LLAMA_METAL) + add_compile_definitions(GGML_USE_METAL) + endif() + + # Building llava + add_subdirectory(vendor/llama.cpp/tools/mtmd) + + if (WIN32) + set_target_properties(mtmd PROPERTIES CUDA_ARCHITECTURES OFF) + endif() + llama_cpp_python_install_target(mtmd) + if (WIN32) + install( + FILES $ + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib + ) + install( + FILES $ + DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp/lib + ) + endif() + + # Fix for mtmd build: Add include directory for llama.h + # Move these commands after the add_subdirectory call + target_include_directories(mtmd PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + target_include_directories(mtmd PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) + + if (BUILD_SHARED_LIBS) + target_include_directories(mtmd PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + target_include_directories(mtmd PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/ggml/include) + endif() + + # target_include_directories(llama-llava-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + # target_include_directories(llama-minicpmv-cli PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp/include) + endif() endif() diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 17575c7008..11208b09ef 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -28,6 +28,7 @@ import numpy as np import numpy.typing as npt +import llama_cpp.llama_cpp as llama_cpp import llama_cpp.llama as llama import llama_cpp.llama_types as llama_types import llama_cpp.llama_grammar as llama_grammar @@ -2651,7 +2652,7 @@ def generate_streaming(tools, functions, function_call, prompt): class Llava15ChatHandler: DEFAULT_SYSTEM_MESSAGE: Optional[str] = ( - "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." + "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions." ) CHAT_FORMAT = ( @@ -2690,70 +2691,72 @@ class Llava15ChatHandler: ) def __init__(self, clip_model_path: str, verbose: bool = True): - import llama_cpp.llava_cpp as llava_cpp + import llama_cpp.mtmd_cpp as mtmd_cpp self.clip_model_path = clip_model_path self.verbose = verbose - - self._llava_cpp = llava_cpp # TODO: Fix + self._mtmd_cpp = mtmd_cpp self._exit_stack = ExitStack() - self._last_image_embed: Optional[ - llava_cpp.CtypesPointer[llava_cpp.llava_image_embed] - ] = None - self._last_image_hash: Optional[int] = None + self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None if not os.path.exists(clip_model_path): raise ValueError(f"Clip model path does not exist: {clip_model_path}") + def _init_mtmd_context(self, llama_model: llama.Llama): + """Initialize mtmd context with the llama model.""" + if self.mtmd_ctx is not None: + return # Already initialized + with suppress_stdout_stderr(disable=self.verbose): - clip_ctx = self._llava_cpp.clip_model_load(self.clip_model_path.encode(), 0) + # Get default parameters + ctx_params = self._mtmd_cpp.mtmd_context_params_default() + # ctx_params.use_gpu = True + ctx_params.print_timings = self.verbose + ctx_params.n_threads = 16 + ctx_params.verbosity = 2 if self.verbose else 0 # GGML_LOG_LEVEL_INFO = 2 + + # Initialize mtmd context + self.mtmd_ctx = self._mtmd_cpp.mtmd_init_from_file( + self.clip_model_path.encode(), + llama_model.model, + ctx_params + ) - if clip_ctx is None: - raise ValueError(f"Failed to load clip model: {clip_model_path}") + if self.mtmd_ctx is None: + raise ValueError(f"Failed to load mtmd context from: {self.clip_model_path}") - self.clip_ctx = clip_ctx + # Check if vision is supported + if not self._mtmd_cpp.mtmd_support_vision(self.mtmd_ctx): + raise ValueError("Vision is not supported by this model") - def clip_free(): + def mtmd_free(): with suppress_stdout_stderr(disable=self.verbose): - self._llava_cpp.clip_free(self.clip_ctx) - - self._exit_stack.callback(clip_free) + if self.mtmd_ctx is not None: + self._mtmd_cpp.mtmd_free(self.mtmd_ctx) + self.mtmd_ctx = None - def last_image_embed_free(): - with suppress_stdout_stderr(disable=self.verbose): - if self._last_image_embed is not None: - self._llava_cpp.llava_image_embed_free(self._last_image_embed) - self._last_image_embed = None - - self._exit_stack.callback(last_image_embed_free) + self._exit_stack.callback(mtmd_free) def load_image(self, image_url: str) -> bytes: return self._load_image(image_url) - def _embed_image_bytes(self, image_bytes: bytes, n_threads_batch: int = 1): - if ( - self._last_image_embed is not None - and self._last_image_hash is not None - and hash(image_bytes) == self._last_image_hash - ): - return self._last_image_embed + def _create_bitmap_from_bytes(self, image_bytes: bytes): + """Create mtmd_bitmap from image bytes.""" + if self.mtmd_ctx is None: + raise ValueError("mtmd context not initialized") + with suppress_stdout_stderr(disable=self.verbose): - # Free the previous image embed - if self._last_image_embed is not None: - self._llava_cpp.llava_image_embed_free(self._last_image_embed) - self._last_image_embed = None - self._last_image_hash = None - embed = self._llava_cpp.llava_image_embed_make_with_bytes( - self.clip_ctx, - n_threads_batch, - (ctypes.c_uint8 * len(image_bytes)).from_buffer( - bytearray(image_bytes) - ), - len(image_bytes), + # Create bitmap from buffer using helper function + bitmap = self._mtmd_cpp.mtmd_helper_bitmap_init_from_buf( + self.mtmd_ctx, + (ctypes.c_uint8 * len(image_bytes)).from_buffer(bytearray(image_bytes)), + len(image_bytes) ) - self._last_image_embed = embed - self._last_image_hash = hash(image_bytes) - return embed + + if bitmap is None: + raise ValueError("Failed to create bitmap from image bytes") + + return bitmap def __call__( self, @@ -2794,7 +2797,9 @@ def __call__( llama_types.CreateChatCompletionResponse, Iterator[llama_types.CreateChatCompletionStreamResponse], ]: - assert self.clip_ctx is not None + # Initialize mtmd context + self._init_mtmd_context(llama) + assert self.mtmd_ctx is not None system_prompt = _get_system_message(messages) if system_prompt == "" and self.DEFAULT_SYSTEM_MESSAGE is not None: @@ -2809,54 +2814,131 @@ def __call__( trim_blocks=True, lstrip_blocks=True, ).from_string(self.CHAT_FORMAT) + + # Get the default media marker + media_marker = self._mtmd_cpp.mtmd_default_marker().decode('utf-8') + + # Replace image URLs with media markers in the template text = template.render( messages=messages, add_generation_prompt=True, eos_token=llama.detokenize([llama.token_eos()]), bos_token=llama.detokenize([llama.token_bos()]), ) - split_text = self.split_text_on_image_urls(text, image_urls) + + # Replace image URLs in text with media markers + for image_url in image_urls: + text = text.replace(image_url, media_marker) if self.verbose: print(text, file=sys.stderr) + # Create bitmaps from images + bitmaps = [] + bitmap_cleanup = [] + try: + for image_url in image_urls: + image_bytes = self.load_image(image_url) + bitmap = self._create_bitmap_from_bytes(image_bytes) + bitmaps.append(bitmap) + bitmap_cleanup.append(bitmap) + + # Create input text structure + input_text = self._mtmd_cpp.mtmd_input_text() + input_text.text = text.encode('utf-8') + input_text.add_special = True + input_text.parse_special = True + + # Create input chunks + chunks = self._mtmd_cpp.mtmd_input_chunks_init() + if chunks is None: + raise ValueError("Failed to create input chunks") - # Evaluate prompt - llama.reset() - llama._ctx.kv_cache_clear() - for type_, value in split_text: - if type_ == "text": - tokens = llama.tokenize( - value.encode("utf8"), add_bos=False, special=True + try: + # Tokenize text and images together + bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps) + result = self._mtmd_cpp.mtmd_tokenize( + self.mtmd_ctx, + chunks, + ctypes.byref(input_text), + bitmap_array, + len(bitmaps) ) - if llama.n_tokens + len(tokens) > llama.n_ctx(): - raise ValueError( - f"Prompt exceeds n_ctx: {llama.n_tokens + len(tokens)} > {llama.n_ctx()}" - ) - llama.eval(tokens) - else: - image_bytes = self.load_image(value) - embed = self._embed_image_bytes(image_bytes, llama.context_params.n_threads_batch) - if llama.n_tokens + embed.contents.n_image_pos > llama.n_ctx(): - raise ValueError( - f"Prompt exceeds n_ctx: {llama.n_tokens + embed.contents.n_image_pos} > {llama.n_ctx()}" - ) - n_past = ctypes.c_int(llama.n_tokens) - n_past_p = ctypes.pointer(n_past) - with suppress_stdout_stderr(disable=self.verbose): - self._llava_cpp.llava_eval_image_embed( - llama.ctx, - embed, - llama.n_batch, - n_past_p, - ) - # Required to avoid issues with hf tokenizer - llama.input_ids[llama.n_tokens : n_past.value] = -1 - llama.n_tokens = n_past.value - # Get prompt tokens to avoid a cache miss - prompt = llama.input_ids[: llama.n_tokens].tolist() + if result != 0: + raise ValueError(f"Failed to tokenize input: error code {result}") + + # Reset llama context + llama.reset() + llama._ctx.kv_cache_clear() + + # Process each chunk + n_past = llama_cpp.llama_pos(0) + n_chunks = self._mtmd_cpp.mtmd_input_chunks_size(chunks) + + for i in range(n_chunks): + chunk = self._mtmd_cpp.mtmd_input_chunks_get(chunks, i) + if chunk is None: + continue + + chunk_type = self._mtmd_cpp.mtmd_input_chunk_get_type(chunk) + + if chunk_type == self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_TEXT: + # Handle text chunk + n_tokens_out = ctypes.c_size_t() + tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text( + chunk, ctypes.byref(n_tokens_out) + ) + + if tokens_ptr and n_tokens_out.value > 0: + # Convert ctypes array to Python list + tokens = [tokens_ptr[j] for j in range(n_tokens_out.value)] + + if llama.n_tokens + len(tokens) > llama.n_ctx(): + raise ValueError( + f"Prompt exceeds n_ctx: {llama.n_tokens + len(tokens)} > {llama.n_ctx()}" + ) + llama.eval(tokens) + + elif chunk_type in [self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_IMAGE, self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_AUDIO]: + # Handle image/audio chunk using helper + chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk) + + if llama.n_tokens + chunk_n_tokens > llama.n_ctx(): + raise ValueError( + f"Prompt exceeds n_ctx: {llama.n_tokens + chunk_n_tokens} > {llama.n_ctx()}" + ) + + new_n_past = llama_cpp.llama_pos(0) + result = self._mtmd_cpp.mtmd_helper_eval_chunk_single( + self.mtmd_ctx, + llama._ctx.ctx, + chunk, + llama_cpp.llama_pos(llama.n_tokens), + llama_cpp.llama_seq_id(0), + llama.n_batch, + False, # logits_last + ctypes.byref(new_n_past) + ) + + if result != 0: + raise ValueError(f"Failed to evaluate chunk: error code {result}") + + # Update llama's token count + llama.n_tokens = new_n_past.value + + # Get prompt tokens to avoid a cache miss + prompt = llama.input_ids[: llama.n_tokens].tolist() + finally: + self._mtmd_cpp.mtmd_input_chunks_free(chunks) + + finally: + # Cleanup bitmaps + for bitmap in bitmap_cleanup: + self._mtmd_cpp.mtmd_bitmap_free(bitmap) + + # Handle response format and tools (same as before) if response_format is not None and response_format["type"] == "json_object": grammar = _grammar_for_response_format(response_format) @@ -2931,6 +3013,7 @@ def __call__( grammar=grammar, logit_bias=logit_bias, ) + if tool is not None: tool_name = tool["function"]["name"] return _convert_completion_to_chat_function( @@ -2943,12 +3026,10 @@ def _load_image(image_url: str) -> bytes: # TODO: Add Pillow support for other image formats beyond (jpg, png) if image_url.startswith("data:"): import base64 - image_bytes = base64.b64decode(image_url.split(",")[1]) return image_bytes else: import urllib.request - with urllib.request.urlopen(image_url) as f: image_bytes = f.read() return image_bytes @@ -2974,6 +3055,7 @@ def get_image_urls(messages: List[llama_types.ChatCompletionRequestMessage]): @staticmethod def split_text_on_image_urls(text: str, image_urls: List[str]): + """This method is no longer used in the new implementation.""" def find_first(s: str, substrs: List[str]): for i, substr in enumerate(substrs): pos = s.find(substr) @@ -3373,6 +3455,61 @@ class MiniCPMv26ChatHandler(Llava15ChatHandler): ) +class Qwen25VLChatHandler(Llava15ChatHandler): + DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant." + + CHAT_FORMAT = ( + "<|im_start|>system\n" + "You are a helpful assistant.<|im_end|>\n" + "{% for message in messages %}" + "{% if message['role'] == 'user' %}" + "<|im_start|>user\n" + "{% if message['content'] is string %}" + "{{ message['content'] }}" + "{% else %}" + "{% for content in message['content'] %}" + "{% if content['type'] == 'text' %}" + "{{ content['text'] }}" + "{% elif content['type'] == 'image_url' %}" + "{% if content.image_url is string %}" + "{{ content.image_url }}" + "{% else %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "<|im_end|>\n" + "{% endif %}" + "{% endfor %}" + "<|im_start|>assistant\n" + ) + + def __call__(self, **kwargs): + llama = kwargs['llama'] + + # Clear state for multiple runs + llama.reset() + llama._ctx.kv_cache_clear() + llama.n_tokens = 0 + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + # Clear any handler state + if hasattr(self, '_last_image_embed'): + self._last_image_embed = None + self._last_image_hash = None + + if self.verbose: + messages = kwargs.get('messages', []) + image_count = len(self.get_image_urls(messages)) + print(f"Minimal - Cleared state, processing {image_count} images", file=sys.stderr) + + # Use parent implementation + return super().__call__(**kwargs) + + @register_chat_completion_handler("chatml-function-calling") def chatml_function_calling( llama: llama.Llama, diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py new file mode 100644 index 0000000000..a45f8f4064 --- /dev/null +++ b/llama_cpp/mtmd_cpp.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +import os +from ctypes import ( + c_bool, + c_char_p, + c_int, + c_uint8, + c_uint32, + c_float, + c_void_p, + c_size_t, + POINTER, + _Pointer, # type: ignore + Structure, + byref, +) +import pathlib +from typing import ( + Union, + NewType, + Optional, + TYPE_CHECKING, +) + +import llama_cpp.llama_cpp as llama_cpp + +from llama_cpp._ctypes_extensions import ( + load_shared_library, + ctypes_function_for_shared_library, +) + +if TYPE_CHECKING: + from llama_cpp._ctypes_extensions import ( + CtypesArray, + ) + + +# Specify the base name of the shared library to load +_libmtmd_base_name = "mtmd" +_libmtmd_override_path = os.environ.get("MTMD_CPP_LIB") +_libmtmd_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _libmtmd_override_path is None else pathlib.Path() + +# Load the library +_libmtmd = load_shared_library(_libmtmd_base_name, _libmtmd_base_path) + +ctypes_function = ctypes_function_for_shared_library(_libmtmd) + +################################################ +# mtmd.h types +################################################ + +# Opaque types +mtmd_context_p = NewType("mtmd_context_p", int) +mtmd_context_p_ctypes = c_void_p + +mtmd_bitmap_p = NewType("mtmd_bitmap_p", int) +mtmd_bitmap_p_ctypes = c_void_p + +mtmd_image_tokens_p = NewType("mtmd_image_tokens_p", int) +mtmd_image_tokens_p_ctypes = c_void_p + +mtmd_input_chunk_p = NewType("mtmd_input_chunk_p", int) +mtmd_input_chunk_p_ctypes = c_void_p + +mtmd_input_chunks_p = NewType("mtmd_input_chunks_p", int) +mtmd_input_chunks_p_ctypes = c_void_p + +# Enums +MTMD_INPUT_CHUNK_TYPE_TEXT = 0 +MTMD_INPUT_CHUNK_TYPE_IMAGE = 1 +MTMD_INPUT_CHUNK_TYPE_AUDIO = 2 + +# Structures +class mtmd_context_params(Structure): + _fields_ = [ + ("use_gpu", c_bool), + ("print_timings", c_bool), + ("n_threads", c_int), + ("verbosity", c_int), # ggml_log_level + ("image_marker", c_char_p), + ("media_marker", c_char_p), + ] + +class mtmd_input_text(Structure): + _fields_ = [ + ("text", c_char_p), + ("add_special", c_bool), + ("parse_special", c_bool), + ] + +################################################ +# mtmd.h functions +################################################ + +# MTMD_API const char * mtmd_default_marker(void); +@ctypes_function("mtmd_default_marker", [], c_char_p) +def mtmd_default_marker() -> bytes: + ... + +# MTMD_API struct mtmd_context_params mtmd_context_params_default(void); +@ctypes_function("mtmd_context_params_default", [], mtmd_context_params) +def mtmd_context_params_default() -> mtmd_context_params: + ... + +# MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname, +# const struct llama_model * text_model, +# const struct mtmd_context_params ctx_params); +@ctypes_function( + "mtmd_init_from_file", + [c_char_p, llama_cpp.llama_model_p_ctypes, mtmd_context_params], + mtmd_context_p_ctypes +) +def mtmd_init_from_file( + mmproj_fname: bytes, + text_model: llama_cpp.llama_model_p, + ctx_params: mtmd_context_params, + /, +) -> Optional[mtmd_context_p]: + ... + +# MTMD_API void mtmd_free(mtmd_context * ctx); +@ctypes_function("mtmd_free", [mtmd_context_p_ctypes], None) +def mtmd_free(ctx: mtmd_context_p, /): + ... + +# MTMD_API bool mtmd_support_vision(mtmd_context * ctx); +@ctypes_function("mtmd_support_vision", [mtmd_context_p_ctypes], c_bool) +def mtmd_support_vision(ctx: mtmd_context_p, /) -> bool: + ... + +# MTMD_API mtmd_bitmap * mtmd_bitmap_init(uint32_t nx, uint32_t ny, const unsigned char * data); +@ctypes_function( + "mtmd_bitmap_init", + [c_uint32, c_uint32, POINTER(c_uint8)], + mtmd_bitmap_p_ctypes +) +def mtmd_bitmap_init( + nx: Union[c_uint32, int], + ny: Union[c_uint32, int], + data: CtypesArray[c_uint8], + /, +) -> Optional[mtmd_bitmap_p]: + ... + +# MTMD_API void mtmd_bitmap_free(mtmd_bitmap * bitmap); +@ctypes_function("mtmd_bitmap_free", [mtmd_bitmap_p_ctypes], None) +def mtmd_bitmap_free(bitmap: mtmd_bitmap_p, /): + ... + +# MTMD_API mtmd_input_chunks * mtmd_input_chunks_init(void); +@ctypes_function("mtmd_input_chunks_init", [], mtmd_input_chunks_p_ctypes) +def mtmd_input_chunks_init() -> Optional[mtmd_input_chunks_p]: + ... + +# MTMD_API void mtmd_input_chunks_free(mtmd_input_chunks * chunks); +@ctypes_function("mtmd_input_chunks_free", [mtmd_input_chunks_p_ctypes], None) +def mtmd_input_chunks_free(chunks: mtmd_input_chunks_p, /): + ... + +# MTMD_API size_t mtmd_input_chunks_size(const mtmd_input_chunks * chunks); +@ctypes_function("mtmd_input_chunks_size", [mtmd_input_chunks_p_ctypes], c_size_t) +def mtmd_input_chunks_size(chunks: mtmd_input_chunks_p, /) -> int: + ... + +# MTMD_API const mtmd_input_chunk * mtmd_input_chunks_get(const mtmd_input_chunks * chunks, size_t idx); +@ctypes_function( + "mtmd_input_chunks_get", + [mtmd_input_chunks_p_ctypes, c_size_t], + mtmd_input_chunk_p_ctypes +) +def mtmd_input_chunks_get( + chunks: mtmd_input_chunks_p, idx: Union[c_size_t, int], / +) -> Optional[mtmd_input_chunk_p]: + ... + +# MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, +# mtmd_input_chunks * output, +# const mtmd_input_text * text, +# const mtmd_bitmap ** bitmaps, +# size_t n_bitmaps); +@ctypes_function( + "mtmd_tokenize", + [ + mtmd_context_p_ctypes, + mtmd_input_chunks_p_ctypes, + POINTER(mtmd_input_text), + POINTER(mtmd_bitmap_p_ctypes), + c_size_t, + ], + c_int, +) +def mtmd_tokenize( + ctx: mtmd_context_p, + output: mtmd_input_chunks_p, + text: "_Pointer[mtmd_input_text]", + bitmaps: CtypesArray[mtmd_bitmap_p_ctypes], + n_bitmaps: Union[c_size_t, int], + /, +) -> int: + ... + +# MTMD_API size_t mtmd_input_chunk_get_n_tokens(const mtmd_input_chunk * chunk); +@ctypes_function("mtmd_input_chunk_get_n_tokens", [mtmd_input_chunk_p_ctypes], c_size_t) +def mtmd_input_chunk_get_n_tokens(chunk: mtmd_input_chunk_p, /) -> int: + ... + +# MTMD_API enum mtmd_input_chunk_type mtmd_input_chunk_get_type(const mtmd_input_chunk * chunk); +@ctypes_function("mtmd_input_chunk_get_type", [mtmd_input_chunk_p_ctypes], c_int) +def mtmd_input_chunk_get_type(chunk: mtmd_input_chunk_p, /) -> int: + ... + +# MTMD_API const llama_token * mtmd_input_chunk_get_tokens_text(const mtmd_input_chunk * chunk, size_t * n_tokens_output); +@ctypes_function( + "mtmd_input_chunk_get_tokens_text", + [mtmd_input_chunk_p_ctypes, POINTER(c_size_t)], + POINTER(llama_cpp.llama_token) +) +def mtmd_input_chunk_get_tokens_text( + chunk: mtmd_input_chunk_p, n_tokens_output: "_Pointer[c_size_t]", / +) -> Optional["_Pointer[llama_cpp.llama_token]"]: + ... + +################################################ +# mtmd-helper.h functions +################################################ + +# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len); +@ctypes_function( + "mtmd_helper_bitmap_init_from_buf", + [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t], + mtmd_bitmap_p_ctypes +) +def mtmd_helper_bitmap_init_from_buf( + ctx: mtmd_context_p, + buf: CtypesArray[c_uint8], + length: Union[c_size_t, int], + /, +) -> Optional[mtmd_bitmap_p]: + ... + +# MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); +@ctypes_function("mtmd_helper_get_n_tokens", [mtmd_input_chunks_p_ctypes], c_size_t) +def mtmd_helper_get_n_tokens(chunks: mtmd_input_chunks_p, /) -> int: + ... + +# MTMD_API int32_t mtmd_helper_eval_chunk_single(mtmd_context * ctx, +# struct llama_context * lctx, +# const mtmd_input_chunk * chunk, +# llama_pos n_past, +# llama_seq_id seq_id, +# int32_t n_batch, +# bool logits_last, +# llama_pos * new_n_past); +@ctypes_function( + "mtmd_helper_eval_chunk_single", + [ + mtmd_context_p_ctypes, + llama_cpp.llama_context_p_ctypes, + mtmd_input_chunk_p_ctypes, + llama_cpp.llama_pos, + llama_cpp.llama_seq_id, + c_int, + c_bool, + POINTER(llama_cpp.llama_pos), + ], + c_int, +) +def mtmd_helper_eval_chunk_single( + ctx: mtmd_context_p, + lctx: llama_cpp.llama_context_p, + chunk: mtmd_input_chunk_p, + n_past: llama_cpp.llama_pos, + seq_id: llama_cpp.llama_seq_id, + n_batch: Union[c_int, int], + logits_last: Union[c_bool, bool], + new_n_past: "_Pointer[llama_cpp.llama_pos]", + /, +) -> int: + ... diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index c6716f9196..11bd363b56 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -171,6 +171,20 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler = llama_cpp.llama_chat_format.MiniCPMv26ChatHandler( clip_model_path=settings.clip_model_path, verbose=settings.verbose ) + elif settings.chat_format == "qwen2.5-vl": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.Qwen25VLChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.Qwen25VLChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) elif settings.chat_format == "hf-autotokenizer": assert ( settings.hf_pretrained_model_name_or_path is not None From 07a979f9077f28b180f4102b9f089246327b96df Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 3 Jul 2025 02:01:24 -0400 Subject: [PATCH 410/561] fix: Use num_threads from llama model for mtmd --- llama_cpp/llama_chat_format.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 11208b09ef..a288db7b0e 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2710,9 +2710,9 @@ def _init_mtmd_context(self, llama_model: llama.Llama): with suppress_stdout_stderr(disable=self.verbose): # Get default parameters ctx_params = self._mtmd_cpp.mtmd_context_params_default() - # ctx_params.use_gpu = True + ctx_params.use_gpu = True # TODO: Make this configurable ctx_params.print_timings = self.verbose - ctx_params.n_threads = 16 + ctx_params.n_threads = llama_model.n_threads ctx_params.verbosity = 2 if self.verbose else 0 # GGML_LOG_LEVEL_INFO = 2 # Initialize mtmd context From 6f3f0bf4d1a7db581ea0f87b89212e3a08b5ded7 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 3 Jul 2025 02:04:50 -0400 Subject: [PATCH 411/561] docs: Add Qwen2.5-VL to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e004565809..088a23779e 100644 --- a/README.md +++ b/README.md @@ -505,6 +505,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [nanollava](https://huggingface.co/abetlen/nanollava-gguf) | `NanollavaChatHandler` | `nanollava` | | [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha-gguf) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | | [minicpm-v-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `MiniCPMv26ChatHandler` | `minicpm-v-2.6` | +| [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. From 9770b8405fb05ed2599dcd3941e4db54d861c12e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 3 Jul 2025 11:25:09 -0400 Subject: [PATCH 412/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index affbd5db78..e08e52c10f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.10] + +- feat: Update llama.cpp to ggerganov/llama.cpp@8846aace4934ad29651ea61b8c7e3f6b0556e3d2 +- feat: Add support for llama.cpp multimodal, add Qwen2.5-VL chat handler by @abetlen in cd548bd0f14210627798237d5c2ea78acfb88ccb + ## [0.3.9] - feat: Update llama.cpp to ggerganov/llama.cpp@8733e0cf6eefc7c7752297cc22d0836706f4222c diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 2c9c527cd4..11a5113909 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.9" +__version__ = "0.3.10" From 9e5a4eaa84156084ed7bbb91e6efcc91dc6217bc Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 5 Jul 2025 02:15:10 -0400 Subject: [PATCH 413/561] fix: Update reference to in Llama.embed. Closes #2037 --- llama_cpp/llama.py | 4 ++-- tests/test_llama.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index cdc05c7ad8..2e93670e60 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1041,7 +1041,7 @@ def embed( data: Union[List[List[float]], List[List[List[float]]]] = [] def decode_batch(seq_sizes: List[int]): - llama_cpp.llama_kv_cache_clear(self._ctx.ctx) + llama_cpp.llama_kv_self_clear(self._ctx.ctx) self._ctx.decode(self._batch) self._batch.reset() @@ -1112,7 +1112,7 @@ def decode_batch(seq_sizes: List[int]): output = data[0] if isinstance(input, str) else data - llama_cpp.llama_kv_cache_clear(self._ctx.ctx) + llama_cpp.llama_kv_self_clear(self._ctx.ctx) self.reset() if return_count: diff --git a/tests/test_llama.py b/tests/test_llama.py index fc182ae20f..0a1a9f5ad3 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -216,3 +216,19 @@ def logit_processor_func(input_ids, logits): assert number_1 != number_2 assert number_1 == number_3 + + +def test_real_llama_embeddings(llama_cpp_model_path): + model = llama_cpp.Llama( + llama_cpp_model_path, + n_ctx=32, + n_batch=32, + n_ubatch=32, + n_threads=multiprocessing.cpu_count(), + n_threads_batch=multiprocessing.cpu_count(), + logits_all=False, + flash_attn=True, + embedding=True + ) + # Smoke test for now + model.embed("Hello World") From ae54cde83aafbbef6d203fa18627a7f6b97b705f Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 5 Jul 2025 02:20:25 -0400 Subject: [PATCH 414/561] fix(ci): Update cuda build action to use ubuntu 22.04 --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 745b2e602d..63b4c26eaf 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -20,7 +20,7 @@ jobs: id: set-matrix run: | $matrix = @{ - 'os' = @('ubuntu-latest', 'windows-2019') + 'os' = @('ubuntu-22.04', 'windows-2022') 'pyver' = @("3.9", "3.10", "3.11", "3.12") 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") #, "12.5.1", "12.6.1") 'releasetag' = @("basic") From 083fcf657dc36e0614f77c0ca468429a876a0862 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 5 Jul 2025 02:21:05 -0400 Subject: [PATCH 415/561] fix(ci): Add git to package list --- docker/simple/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/simple/Dockerfile b/docker/simple/Dockerfile index 3594df1a50..06483d44e5 100644 --- a/docker/simple/Dockerfile +++ b/docker/simple/Dockerfile @@ -9,6 +9,7 @@ ARG IMAGE # Update and upgrade the existing packages RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ + git \ python3 \ python3-pip \ ninja-build \ From 11d28df7d86c7ae34816cf68fc0ca1ac5023b59d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 5 Jul 2025 02:23:36 -0400 Subject: [PATCH 416/561] fix(ci): Remove macos-13 builds to fix cross compilation error --- .github/workflows/build-wheels-metal.yaml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 9b97bf2f56..98f511e4a6 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-13, macos-14, macos-15] + os: [macos-14, macos-15] steps: - uses: actions/checkout@v4 @@ -23,32 +23,21 @@ jobs: with: python-version: "3.12" cache: 'pip' - + - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' run: | python -m pip install --upgrade pip python -m pip install uv RUST_LOG=trace python -m uv pip install -e .[all] --verbose shell: bash - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - env: - RUST_LOG: trace - run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - shell: cmd - - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" CIBW_ARCHS: "arm64" - CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DGGML_METAL=on" + CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DGGML_METAL=on -DCMAKE_CROSSCOMPILING=ON" CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" with: package-dir: . @@ -69,7 +58,7 @@ jobs: with: merge-multiple: true path: dist2 - + - uses: softprops/action-gh-release@v2 with: files: dist2/* From 1580839fe8fa01324a0d33bc52d63279df368eec Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 5 Jul 2025 02:25:41 -0400 Subject: [PATCH 417/561] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e08e52c10f..6e336962f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.11] + +- fix: Update reference to `llama_kv_cache_clear` in Llama.embed. Closes #2037 by @abetlen in 9e5a4eaa84156084ed7bbb91e6efcc91dc6217bc + ## [0.3.10] - feat: Update llama.cpp to ggerganov/llama.cpp@8846aace4934ad29651ea61b8c7e3f6b0556e3d2 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 11a5113909..e35c5014e1 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.10" +__version__ = "0.3.11" From 82ad829c4c95af4435c30cd07dfa51025e404712 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sat, 5 Jul 2025 02:35:56 -0400 Subject: [PATCH 418/561] fix(ci): update runners for cpu builds --- .github/workflows/build-and-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 7307c85ab6..29a6913c2b 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, windows-2019, macos-13] + os: [ubuntu-22.04, windows-2022, macos-14, macos-15] steps: - uses: actions/checkout@v4 From 7011bc198ddc6a3fb2756c554915d4137893e21c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 6 Jul 2025 00:45:59 -0400 Subject: [PATCH 419/561] fix(ci): Update docker runner --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index b5c7346db4..b290f6273f 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -9,7 +9,7 @@ permissions: jobs: docker: name: Build and push Docker image - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 From b39e9d4518e1d747dc02fd3bb611ea3d22a9393e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 6 Jul 2025 00:52:13 -0400 Subject: [PATCH 420/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8846aace49..a0374a67e2 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8846aace4934ad29651ea61b8c7e3f6b0556e3d2 +Subproject commit a0374a67e2924f2e845cdc59dd67d9a44065a89c From 98fda8cdf9097a5cd8d59347ea4b8e7012f32a8a Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 6 Jul 2025 00:55:34 -0400 Subject: [PATCH 421/561] fix(ci): Temporarily disable windows cuda wheels --- .github/workflows/build-wheels-cuda.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 63b4c26eaf..07b30cfc06 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -8,7 +8,7 @@ permissions: jobs: define_matrix: name: Define Build Matrix - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} defaults: @@ -20,7 +20,7 @@ jobs: id: set-matrix run: | $matrix = @{ - 'os' = @('ubuntu-22.04', 'windows-2022') + 'os' = @('ubuntu-22.04') #, 'windows-2022') 'pyver' = @("3.9", "3.10", "3.11", "3.12") 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") #, "12.5.1", "12.6.1") 'releasetag' = @("basic") From 8866fbd7159736d776e1269ce778872a20e25cc2 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 6 Jul 2025 00:56:40 -0400 Subject: [PATCH 422/561] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e336962f7..6017812bb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.12] + +- feat: Update llama.cpp to ggerganov/llama.cpp@a0374a67e2924f2e845cdc59dd67d9a44065a89c + ## [0.3.11] - fix: Update reference to `llama_kv_cache_clear` in Llama.embed. Closes #2037 by @abetlen in 9e5a4eaa84156084ed7bbb91e6efcc91dc6217bc diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index e35c5014e1..b16bb7dc98 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.11" +__version__ = "0.3.12" From cce48873166e6b6fbfe0d944a6184ef11858e735 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Sun, 6 Jul 2025 01:32:07 -0400 Subject: [PATCH 423/561] fix(ci): Fix macos cpu builds --- .github/workflows/build-and-release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 29a6913c2b..7eaf017fbc 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -74,6 +74,7 @@ jobs: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" CIBW_ARCHS: "aarch64" + CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DCMAKE_CROSSCOMPILING=ON" CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" with: output-dir: wheelhouse From a99fd214dd0b7b97f2cd5d22ae4c7c91236aec6c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 15 Jul 2025 07:35:40 -0400 Subject: [PATCH 424/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 1 + vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d13d60458a..bd8e36b792 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -200,6 +200,7 @@ """RWKV tokenizer based on greedy tokenization""" +# NOTE: Deprecated and will be removed in the future. (already gone in llama.cpp) # // pre-tokenization types # enum llama_vocab_pre_type { # LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a0374a67e2..bdca38376f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a0374a67e2924f2e845cdc59dd67d9a44065a89c +Subproject commit bdca38376f7e8dd928defe01ce6a16218a64b040 From c8579d767aa53c48221f409c49df03e4019ed771 Mon Sep 17 00:00:00 2001 From: Alcoft <101468238+alcoftTAO@users.noreply.github.com> Date: Tue, 15 Jul 2025 11:37:45 +0000 Subject: [PATCH 425/561] fix: Better chat format for Qwen2.5-VL (#2040) --- llama_cpp/llama_chat_format.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index a288db7b0e..2a7d575ce6 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3459,26 +3459,29 @@ class Qwen25VLChatHandler(Llava15ChatHandler): DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant." CHAT_FORMAT = ( - "<|im_start|>system\n" - "You are a helpful assistant.<|im_end|>\n" + #"{% set image_count = namespace(value=0) %}" + #"{% set video_count = namespace(value=0) %}" "{% for message in messages %}" - "{% if message['role'] == 'user' %}" - "<|im_start|>user\n" + "{% if loop.first and message['role'] != 'system' %}" + "<|im_start|>system\n" + "{{ self.DEFAULT_SYSTEM_MESSAGE }}<|im_end|>\n" + "{% endif %}" + "<|im_start|>{{ message['role'] }}\n" "{% if message['content'] is string %}" - "{{ message['content'] }}" + "{{ message['content'] }}<|im_end|>\n" "{% else %}" "{% for content in message['content'] %}" - "{% if content['type'] == 'text' %}" - "{{ content['text'] }}" - "{% elif content['type'] == 'image_url' %}" + "{% if content['type'] == 'image_url' %}" "{% if content.image_url is string %}" "{{ content.image_url }}" "{% else %}" "{{ content.image_url.url }}" "{% endif %}" + #"{% set image_count.value = image_count.value + 1 %}" + "{% elif content['type'] == 'text' %}" + "{{ content['text'] }}" "{% endif %}" "{% endfor %}" - "{% endif %}" "<|im_end|>\n" "{% endif %}" "{% endfor %}" From d9749cb9678d3ceb93b9ee63f4d723e3f01eb5c6 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Tue, 15 Jul 2025 07:39:46 -0400 Subject: [PATCH 426/561] chore: Bump version --- CHANGELOG.md | 5 +++++ llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6017812bb2..88b9a1b45b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.13] + +- feat: Update llama.cpp to ggerganov/llama.cpp@ +- fix: Better chat format for Qwen2.5-VL by @alcoftTAO in #2040 + ## [0.3.12] - feat: Update llama.cpp to ggerganov/llama.cpp@a0374a67e2924f2e845cdc59dd67d9a44065a89c diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index b16bb7dc98..0c869dcae1 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.12" +__version__ = "0.3.13" From 95292e36c6cef7430cff0bda16578fe9f7303e01 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Wed, 16 Jul 2025 02:47:55 -0400 Subject: [PATCH 427/561] feat: Update llama.cpp --- llama_cpp/llama_cpp.py | 17 ++++++++++------- vendor/llama.cpp | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index bd8e36b792..b9e245e2f5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -179,12 +179,13 @@ # enum llama_vocab_type { -# LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab -# LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback -# LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE -# LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece -# LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram -# LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization +# LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab +# LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback +# LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE +# LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece +# LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram +# LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization +# LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming # }; LLAMA_VOCAB_TYPE_NONE = 0 """For models without vocab""" @@ -198,6 +199,8 @@ """T5 tokenizer based on Unigram""" LLAMA_VOCAB_TYPE_RWKV = 5 """RWKV tokenizer based on greedy tokenization""" +LLAMA_VOCAB_TYPE_PLAMO2 = 6 +"""PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming""" # NOTE: Deprecated and will be removed in the future. (already gone in llama.cpp) @@ -2171,7 +2174,7 @@ def llama_kv_self_seq_add( # // - lazily on next llama_decode() # // p0 < 0 : [0, p1] # // p1 < 0 : [p0, inf) -# DEPRECATED(void llama_kv_self_seq_div( +# DEPRECATED(LLAMA_API void llama_kv_self_seq_div( # struct llama_context * ctx, # llama_seq_id seq_id, # llama_pos p0, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index bdca38376f..79e0b68c17 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit bdca38376f7e8dd928defe01ce6a16218a64b040 +Subproject commit 79e0b68c178656bb0632cb8602d2940b755077f8 From e1af05f43f57d2b660edfb77935dd2d2641ec602 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Fri, 18 Jul 2025 12:45:45 -0400 Subject: [PATCH 428/561] chore: Bump version --- CHANGELOG.md | 6 +++++- llama_cpp/__init__.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b9a1b45b..e743c45843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.14] + +- feat: Update llama.cpp to ggerganov/llama.cpp@79e0b68c178656bb0632cb8602d2940b755077f8 + ## [0.3.13] -- feat: Update llama.cpp to ggerganov/llama.cpp@ +- feat: Update llama.cpp to ggerganov/llama.cpp@bdca38376f7e8dd928defe01ce6a16218a64b040 - fix: Better chat format for Qwen2.5-VL by @alcoftTAO in #2040 ## [0.3.12] diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 0c869dcae1..409c59514f 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.13" +__version__ = "0.3.14" From 4f260288ff0f34801f1c2ede975093201c8dcf4c Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 7 Aug 2025 06:40:18 -0700 Subject: [PATCH 429/561] feat: Update llama.cpp --- llama_cpp/_internals.py | 6 ++++++ llama_cpp/llama_cpp.py | 33 ++++++++++++++++++++++++++++++++- vendor/llama.cpp | 2 +- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 18d733481f..b5175a7f2e 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -287,18 +287,24 @@ def pooling_type(self) -> int: return llama_cpp.llama_pooling_type(self.ctx) def kv_cache_clear(self): + assert self.memory is not None, "Memory is not initialized" llama_cpp.llama_memory_clear(self.memory, True) def kv_cache_seq_rm(self, seq_id: int, p0: int, p1: int): + assert self.memory is not None, "Memory is not initialized" + seq_id = seq_id if seq_id >= 0 else 0 llama_cpp.llama_memory_seq_rm(self.memory, seq_id, p0, p1) def kv_cache_seq_cp(self, seq_id_src: int, seq_id_dst: int, p0: int, p1: int): + assert self.memory is not None, "Memory is not initialized" llama_cpp.llama_memory_seq_cp(self.memory, seq_id_src, seq_id_dst, p0, p1) def kv_cache_seq_keep(self, seq_id: int): + assert self.memory is not None, "Memory is not initialized" llama_cpp.llama_memory_seq_keep(self.memory, seq_id) def kv_cache_seq_shift(self, seq_id: int, p0: int, p1: int, shift: int): + assert self.memory is not None, "Memory is not initialized" llama_cpp.llama_memory_seq_add(self.memory, seq_id, p0, p1, shift) def get_state_size(self) -> int: diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index b9e245e2f5..711d42a6ae 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -381,6 +381,7 @@ # //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack # LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors # LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -419,6 +420,7 @@ # LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 LLAMA_FTYPE_MOSTLY_TQ1_0 = 36 LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 +LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { @@ -691,6 +693,7 @@ class llama_model_kv_override(ctypes.Structure): # bool use_mmap; // use mmap if possible # bool use_mlock; // force system to keep model in RAM # bool check_tensors; // validate model tensor data +# bool use_extra_bufts; // use extra buffer types (used for weight repacking) # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -708,7 +711,8 @@ class llama_model_params(ctypes.Structure): vocab_only (bool): only load the vocabulary, no weights use_mmap (bool): use mmap if possible use_mlock (bool): force system to keep model in RAM - check_tensors (bool): validate model tensor data""" + check_tensors (bool): validate model tensor data + use_extra_bufts (bool): use extra buffer types (used for weight repacking)""" if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused @@ -724,6 +728,7 @@ class llama_model_params(ctypes.Structure): use_mmap: bool use_mlock: bool check_tensors: bool + use_extra_bufts: bool _fields_ = [ ("devices", ctypes.c_void_p), # NOTE: unnused @@ -739,6 +744,7 @@ class llama_model_params(ctypes.Structure): ("use_mmap", ctypes.c_bool), ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), + ("use_extra_bufts", ctypes.c_bool), ] @@ -787,6 +793,9 @@ class llama_model_params(ctypes.Structure): # bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) # // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases # // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 +# bool kv_unified; // use a unified buffer across the input sequences when computing the attention +# // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix +# // ref: https://github.com/ggml-org/llama.cpp/pull/14363 # }; class llama_context_params(ctypes.Structure): """Parameters for llama_context @@ -821,6 +830,7 @@ class llama_context_params(ctypes.Structure): no_perf (bool): whether to measure performance timings op_offload (bool): offload host tensor operations to device swa_full (bool): use full-size SWA cache + kv_unified (bool): use a unified buffer across the input sequences when computing the attention """ if TYPE_CHECKING: @@ -853,6 +863,7 @@ class llama_context_params(ctypes.Structure): no_perf: bool op_offload: bool swa_full: bool + kv_unified: bool _fields_ = [ ("n_ctx", ctypes.c_uint32), @@ -884,6 +895,7 @@ class llama_context_params(ctypes.Structure): ("no_perf", ctypes.c_bool), ("op_offload", ctypes.c_bool), ("swa_full", ctypes.c_bool), + ("kv_unified", ctypes.c_bool), ] @@ -1651,6 +1663,14 @@ def llama_model_is_recurrent(model: llama_model_p, /) -> bool: ... +# // Returns true if the model is diffusion-based (like LLaDA, Dream, etc.) +# LLAMA_API bool llama_model_is_diffusion(const struct llama_model * model); +@ctypes_function("llama_model_is_diffusion", [llama_model_p_ctypes], ctypes.c_bool) +def llama_model_is_diffusion(model: llama_model_p, /) -> bool: + """Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)""" + ... + + # // Returns 0 on success # LLAMA_API uint32_t llama_model_quantize( # const char * fname_inp, @@ -2833,6 +2853,7 @@ def llama_synchronize(ctx: llama_context_p, /): # // in the order they have appeared in the batch. # // Rows: number of tokens for which llama_batch.logits[i] != 0 # // Cols: n_vocab +# // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) # LLAMA_API float * llama_get_logits(struct llama_context * ctx); @ctypes_function( "llama_get_logits", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float) @@ -2873,6 +2894,7 @@ def llama_get_logits_ith( # // in the order they have appeared in the batch. # // shape: [n_outputs*n_embd] # // Otherwise, returns NULL. +# // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) # LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); @ctypes_function( "llama_get_embeddings", [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float) @@ -3020,6 +3042,13 @@ def llama_vocab_pad(vocab: llama_vocab_p, /) -> llama_token: ... +# LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask +@ctypes_function("llama_vocab_mask", [llama_vocab_p_ctypes], llama_token) +def llama_vocab_mask(vocab: llama_vocab_p, /) -> llama_token: + """mask""" + ... + + # LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_get_add_bos", @@ -4176,6 +4205,7 @@ def llama_log_set( # int32_t n_p_eval; # int32_t n_eval; +# int32_t n_reused; // number of times a ggml compute graph had been reused # }; class llama_perf_context_data(ctypes.Structure): _fields_ = [ @@ -4185,6 +4215,7 @@ class llama_perf_context_data(ctypes.Structure): ("t_eval_ms", ctypes.c_double), ("n_p_eval", ctypes.c_int32), ("n_eval", ctypes.c_int32), + ("n_reused", ctypes.c_int32), ] diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 79e0b68c17..9a96389544 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 79e0b68c178656bb0632cb8602d2940b755077f8 +Subproject commit 9a96389544a08fd829fccda28142ce2066017fde From d12ca479885bd530abf4543cd576b7eecb1b20e9 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 7 Aug 2025 06:42:58 -0700 Subject: [PATCH 430/561] misc: Update pypi downloads badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 088a23779e..382f7cbed6 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![PyPI](https://img.shields.io/pypi/v/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/) [![PyPI - License](https://img.shields.io/pypi/l/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/) -[![PyPI - Downloads](https://img.shields.io/pypi/dm/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/) +[![PyPI - Downloads](https://static.pepy.tech/badge/llama-cpp-python/month)](https://pepy.tech/projects/llama-cpp-python) [![Github All Releases](https://img.shields.io/github/downloads/abetlen/llama-cpp-python/total.svg?label=Github%20Downloads)]() Simple Python bindings for **@ggerganov's** [`llama.cpp`](https://github.com/ggerganov/llama.cpp) library. From 68e89e86c8135e865995d088ca7e5f4a38370c20 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 7 Aug 2025 06:43:30 -0700 Subject: [PATCH 431/561] misc: Add Python 3.13 classifier tag --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9983ef777d..f5ae7b59c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] From af637928db7351e030011085f818b034c6efc047 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 7 Aug 2025 06:47:26 -0700 Subject: [PATCH 432/561] feat: Add gpt-oss chat format support through strftime_now in chat format by @iamlemec --- llama_cpp/llama_chat_format.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 2a7d575ce6..f738ab9bb4 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -8,6 +8,7 @@ import random import string +from datetime import datetime from contextlib import ExitStack from typing import ( Any, @@ -214,6 +215,10 @@ def __init__( lstrip_blocks=True, ).from_string(self.template) + @staticmethod + def strftime_now(f: str) -> str: + return datetime.now().strftime(f) + def __call__( self, *, @@ -237,6 +242,7 @@ def raise_exception(message: str): function_call=function_call, tools=tools, tool_choice=tool_choice, + strftime_now=self.strftime_now, ) stopping_criteria = None From 30ddd56e827e7fef6d5020809c574bdc0e166196 Mon Sep 17 00:00:00 2001 From: sergey21000 <67040429+sergey21000@users.noreply.github.com> Date: Thu, 7 Aug 2025 16:49:10 +0300 Subject: [PATCH 433/561] fix: rename op_offloat to op_offload in llama.py (#2046) --- llama_cpp/llama.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2e93670e60..71d94ebd82 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -92,7 +92,7 @@ def __init__( embedding: bool = False, offload_kqv: bool = True, flash_attn: bool = False, - op_offloat: Optional[bool] = None, + op_offload: Optional[bool] = None, swa_full: Optional[bool] = None, # Sampling Params no_perf: bool = False, @@ -174,7 +174,7 @@ def __init__( embedding: Embedding mode only. offload_kqv: Offload K, Q, V to GPU. flash_attn: Use flash attention. - op_offloat: offload host tensor operations to device + op_offload: offload host tensor operations to device swa_full: use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) no_perf: Measure performance timings. last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque. @@ -343,8 +343,8 @@ def __init__( self.context_params.offload_kqv = offload_kqv self.context_params.flash_attn = flash_attn - if op_offloat is not None: - self.context_params.op_offloat = op_offloat + if op_offload is not None: + self.context_params.op_offload = op_offload if swa_full is not None: self.context_params.swa_full = swa_full @@ -2097,7 +2097,7 @@ def __getstate__(self): embedding=self.context_params.embeddings, offload_kqv=self.context_params.offload_kqv, flash_attn=self.context_params.flash_attn, - op_offloat=self.context_params.op_offloat, + op_offload=self.context_params.op_offload, swa_full=self.context_params.swa_full, # Sampling Params no_perf=self.context_params.no_perf, From dfc9bf503bb7d4be166410e525971509373bee0e Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 7 Aug 2025 06:53:07 -0700 Subject: [PATCH 434/561] chore: Bump version --- CHANGELOG.md | 6 ++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e743c45843..929363721f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.15] + +- feat: Update llama.cpp to ggerganov/llama.cpp@9a96389544a08fd829fccda28142ce2066017fde +- feat: Add gpt-oss chat format support through strftime_now in chat format by @iamlemec in af637928db7351e030011085f818b034c6efc047 +- fix: rename op_offloat to op_offload in llama.py by @sergey21000 in #2046 + ## [0.3.14] - feat: Update llama.cpp to ggerganov/llama.cpp@79e0b68c178656bb0632cb8602d2940b755077f8 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 409c59514f..1e256a776c 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.14" +__version__ = "0.3.15" From ce6fd8bbc808196dea90dd259bbcd4301c69b0b5 Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 14 Aug 2025 21:52:49 -0700 Subject: [PATCH 435/561] feat: Update llama.cpp --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 9a96389544..4227c9be42 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 9a96389544a08fd829fccda28142ce2066017fde +Subproject commit 4227c9be4268ac844921b90f31595f81236bd317 From c37132bac860fcc333255c36313f89c4f49d4c8d Mon Sep 17 00:00:00 2001 From: Andrei Betlen Date: Thu, 14 Aug 2025 21:55:21 -0700 Subject: [PATCH 436/561] chore: Bump version --- CHANGELOG.md | 4 ++++ llama_cpp/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 929363721f..16954eb881 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.16] + +- feat: Update llama.cpp to ggerganov/llama.cpp@4227c9be4268ac844921b90f31595f81236bd317 + ## [0.3.15] - feat: Update llama.cpp to ggerganov/llama.cpp@9a96389544a08fd829fccda28142ce2066017fde diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 1e256a776c..c1dde70468 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.15" +__version__ = "0.3.16" From ca3b00a204d5a48b1c54eb609b20203dcdaa87be Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 22 Mar 2026 15:20:48 -0700 Subject: [PATCH 437/561] fix(ci): Rename `huggingface-cli` to `hf` (#2149) * Fix model download in test workflow * Use hf CLI in test workflow * Use hf CLI name in CI and docs * Reference PR in changelog --- .github/workflows/test.yaml | 2 +- CHANGELOG.md | 2 ++ README.md | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 95f6e5a272..1d2b1983cc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,7 +22,7 @@ jobs: - name: Install huggingface-hub run: pip install huggingface-hub - name: Download model - run: huggingface-cli download ${{ env.REPO_ID }} ${{ env.MODEL_FILE }} + run: hf download ${{ env.REPO_ID }} ${{ env.MODEL_FILE }} - name: Cache model uses: actions/cache@v4 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 16954eb881..1f577c1a41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(ci): Use the `hf` CLI instead of the deprecated `huggingface-cli` name in GitHub Actions and docs by @abetlen in #2149 + ## [0.3.16] - feat: Update llama.cpp to ggerganov/llama.cpp@4227c9be4268ac844921b90f31595f81236bd317 diff --git a/README.md b/README.md index 382f7cbed6..d2ba297caf 100644 --- a/README.md +++ b/README.md @@ -328,7 +328,7 @@ llm = Llama.from_pretrained( ) ``` -By default [`from_pretrained`](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.from_pretrained) will download the model to the huggingface cache directory, you can then manage installed model files with the [`huggingface-cli`](https://huggingface.co/docs/huggingface_hub/en/guides/cli) tool. +By default [`from_pretrained`](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama.from_pretrained) will download the model to the huggingface cache directory, you can then manage installed model files with the [`hf`](https://huggingface.co/docs/huggingface_hub/en/guides/cli) tool. ### Chat Completion From 9f661ff2cf63e72aea328daab15e521230dd20b0 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 22 Mar 2026 16:10:47 -0700 Subject: [PATCH 438/561] fix(ci): Fix macos tests, support both Intel and Apple Silicon testing (#2150) * fix(ci): use supported macos runner label * fix(ci): add apple silicon macos test coverage * fix(ci): run standard macos tests on apple silicon * fix(ci): simplify apple silicon macos install * fix(ci): disable ggml native on apple silicon runner * docs: update changelog for macos ci runner fix --- .github/workflows/test.yaml | 11 +++++------ CHANGELOG.md | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1d2b1983cc..af4cacac40 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -96,7 +96,7 @@ jobs: build-macos: needs: download-model - runs-on: macos-13 + runs-on: macos-15 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] @@ -127,17 +127,16 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install uv - python3 -m uv pip install -e .[all] --verbose - CMAKE_ARGS="-DLLAMA_METAL=off" python3 -m uv pip install .[all] --verbose + CMAKE_ARGS="-DGGML_NATIVE=off" python3 -m uv pip install -e .[all] --verbose shell: bash - name: Test with pytest run: | python3 -m pytest - build-macos-metal: + build-macos-intel: needs: download-model - runs-on: macos-13 + runs-on: macos-15-intel steps: - uses: actions/checkout@v4 with: @@ -163,7 +162,7 @@ jobs: - name: Install dependencies run: | python3 -m pip install --upgrade pip - CMAKE_ARGS="-DLLAMA_METAL=on" python3 -m pip install .[all] --verbose + python3 -m pip install .[all] --verbose shell: bash - name: Test with pytest diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f577c1a41..9bc1c9a0a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(ci): Run macOS CI on supported Apple Silicon and Intel runners by @abetlen in #2150 - fix(ci): Use the `hf` CLI instead of the deprecated `huggingface-cli` name in GitHub Actions and docs by @abetlen in #2149 ## [0.3.16] From a9b4a067300c89857334195518e0bb9430d1c059 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 22 Mar 2026 16:12:02 -0700 Subject: [PATCH 439/561] misc: Add Ruff formatting (#2148) * Add Ruff formatting and safe lint baseline * Update changelog for Ruff setup --- .github/workflows/lint.yaml | 29 ++ .gitignore | 1 + CHANGELOG.md | 1 + Makefile | 10 + README.md | 14 + llama_cpp/_ggml.py | 2 +- llama_cpp/_internals.py | 98 ++++--- llama_cpp/_logger.py | 5 +- llama_cpp/llama.py | 22 +- llama_cpp/llama_cache.py | 6 +- llama_cpp/llama_chat_format.py | 192 ++++++++----- llama_cpp/llama_cpp.py | 470 ++++++++++++++------------------ llama_cpp/llama_grammar.py | 60 ++-- llama_cpp/llava_cpp.py | 28 +- llama_cpp/mtmd_cpp.py | 94 ++++--- llama_cpp/server/app.py | 6 +- llama_cpp/server/cli.py | 4 +- llama_cpp/server/model.py | 12 +- pyproject.toml | 13 +- tests/test_llama.py | 22 +- tests/test_llama_chat_format.py | 19 +- tests/test_llama_speculative.py | 9 +- 22 files changed, 607 insertions(+), 510 deletions(-) create mode 100644 .github/workflows/lint.yaml diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000000..8b3e6322dc --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,29 @@ +name: Lint + +on: + pull_request: + branches: + - main + push: + branches: + - main + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Ruff + run: python -m pip install "ruff>=0.15.7" + + - name: Lint with Ruff + run: python -m ruff check llama_cpp tests + + - name: Check formatting with Ruff + run: python -m ruff format --check llama_cpp tests diff --git a/.gitignore b/.gitignore index 9d68dbcd9d..ff773c6684 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.ruff_cache/ cover/ # Translations diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bc1c9a0a1..7044f44d65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- chore(dev): Add Ruff-based formatting and a safe lint baseline, and run it in CI for pull requests and pushes to `main` - fix(ci): Run macOS CI on supported Apple Silicon and Intel runners by @abetlen in #2150 - fix(ci): Use the `hf` CLI instead of the deprecated `huggingface-cli` name in GitHub Actions and docs by @abetlen in #2149 diff --git a/Makefile b/Makefile index 26ddf2c7a1..8e6cae2c1f 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,14 @@ deploy.gh-docs: test: python3 -m pytest --full-trace -v +lint: + python3 -m ruff check llama_cpp tests + python3 -m ruff format --check llama_cpp tests + +format: + python3 -m ruff check --fix llama_cpp tests + python3 -m ruff format llama_cpp tests + docker: docker build -t llama-cpp-python:latest -f docker/simple/Dockerfile . @@ -93,5 +101,7 @@ clean: build.sdist \ deploy.pypi \ deploy.gh-docs \ + lint \ + format \ docker \ clean diff --git a/README.md b/README.md index d2ba297caf..b57c958076 100644 --- a/README.md +++ b/README.md @@ -752,6 +752,9 @@ pip install --upgrade pip # Install with pip pip install -e . +# install development tooling (tests, docs, ruff) +pip install -e '.[dev]' + # if you want to use the fastapi / openapi server pip install -e '.[server]' @@ -768,6 +771,17 @@ Now try running the tests pytest ``` +And check formatting / linting before opening a PR: + +```bash +python -m ruff check llama_cpp tests +python -m ruff format --check llama_cpp tests + +# or use the Makefile targets +make lint +make format +``` + There's a `Makefile` available with useful targets. A typical workflow would look like this: diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 5bee8a93b5..5ece01e032 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -2,6 +2,7 @@ This module provides a minimal interface for working with ggml tensors from llama-cpp-python """ + import os import pathlib @@ -9,4 +10,3 @@ libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" libggml = ctypes_ext.load_shared_library("ggml", libggml_base_path) - diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index b5175a7f2e..b520b7ea5f 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -355,7 +355,9 @@ def get_embeddings_seq(self, seq_id: int): # Sampling functions - deprecated, use LlamaSampler instead def set_rng_seed(self, seed: int): - raise NotImplementedError("set_rng_seed is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "set_rng_seed is deprecated, use LlamaSampler instead" + ) def sample_repetition_penalties( self, @@ -366,30 +368,44 @@ def sample_repetition_penalties( penalty_freq: float, penalty_present: float, ): - raise NotImplementedError("sample_repetition_penalties is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_repetition_penalties is deprecated, use LlamaSampler instead" + ) def sample_softmax(self, candidates: "_LlamaTokenDataArray"): - raise NotImplementedError("sample_softmax is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_softmax is deprecated, use LlamaSampler instead" + ) def sample_top_k(self, candidates: "_LlamaTokenDataArray", k: int, min_keep: int): - raise NotImplementedError("sample_top_k is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_top_k is deprecated, use LlamaSampler instead" + ) def sample_top_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - raise NotImplementedError("sample_top_p is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_top_p is deprecated, use LlamaSampler instead" + ) def sample_min_p(self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int): - raise NotImplementedError("sample_min_p is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_min_p is deprecated, use LlamaSampler instead" + ) def sample_typical( self, candidates: "_LlamaTokenDataArray", p: float, min_keep: int ): - raise NotImplementedError("sample_typical is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_typical is deprecated, use LlamaSampler instead" + ) def sample_temp(self, candidates: "_LlamaTokenDataArray", temp: float): raise NotImplementedError("sample_temp is deprecated, use LlamaSampler instead") def sample_grammar(self, candidates: "_LlamaTokenDataArray", grammar: LlamaGrammar): - raise NotImplementedError("sample_grammar is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_grammar is deprecated, use LlamaSampler instead" + ) def sample_token_mirostat( self, @@ -399,7 +415,9 @@ def sample_token_mirostat( m: int, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - raise NotImplementedError("sample_token_mirostat is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_token_mirostat is deprecated, use LlamaSampler instead" + ) def sample_token_mirostat_v2( self, @@ -408,17 +426,25 @@ def sample_token_mirostat_v2( eta: float, mu: llama_cpp.CtypesPointerOrRef[ctypes.c_float], ) -> int: - raise NotImplementedError("sample_token_mirostat_v2 is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_token_mirostat_v2 is deprecated, use LlamaSampler instead" + ) def sample_token_greedy(self, candidates: "_LlamaTokenDataArray") -> int: - raise NotImplementedError("sample_token_greedy is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_token_greedy is deprecated, use LlamaSampler instead" + ) def sample_token(self, candidates: "_LlamaTokenDataArray") -> int: - raise NotImplementedError("sample_token is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "sample_token is deprecated, use LlamaSampler instead" + ) # Grammar def grammar_accept_token(self, grammar: LlamaGrammar, token: int): - raise NotImplementedError("grammar_accept_token is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "grammar_accept_token is deprecated, use LlamaSampler instead" + ) def reset_timings(self): llama_cpp.llama_perf_context_reset(self.ctx) @@ -602,16 +628,16 @@ def sample( logits_array: Optional[npt.NDArray[np.single]] = None, ): # This method is deprecated in favor of using LlamaSampler directly - raise NotImplementedError("LlamaSamplingContext.sample is deprecated, use LlamaSampler instead") + raise NotImplementedError( + "LlamaSamplingContext.sample is deprecated, use LlamaSampler instead" + ) def accept(self, ctx_main: LlamaContext, id: int, apply_grammar: bool): self.prev.append(id) class CustomSampler: - def __init__( - self, apply_func: Callable[[llama_cpp.llama_token_data_array], None] - ): + def __init__(self, apply_func: Callable[[llama_cpp.llama_token_data_array], None]): self.apply_func = apply_func def apply_wrapper( @@ -723,20 +749,20 @@ def add_grammar(self, model: LlamaModel, grammar: LlamaGrammar): llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_grammar_lazy_patterns( - self, - model: LlamaModel, + self, + model: LlamaModel, grammar: LlamaGrammar, trigger_patterns: List[str], - trigger_tokens: List[int] + trigger_tokens: List[int], ): # Convert patterns to C array pattern_ptrs = (ctypes.c_char_p * len(trigger_patterns))() for i, pattern in enumerate(trigger_patterns): pattern_ptrs[i] = pattern.encode("utf-8") - + # Convert tokens to C array token_array = (llama_cpp.llama_token * len(trigger_tokens))(*trigger_tokens) - + sampler = llama_cpp.llama_sampler_init_grammar_lazy_patterns( model.vocab, grammar._grammar.encode("utf-8"), @@ -744,7 +770,7 @@ def add_grammar_lazy_patterns( pattern_ptrs, len(trigger_patterns), token_array, - len(trigger_tokens) + len(trigger_tokens), ) llama_cpp.llama_sampler_chain_add(self.sampler, sampler) @@ -771,13 +797,13 @@ def add_dry( dry_base: float, dry_allowed_length: int, dry_penalty_last_n: int, - seq_breakers: List[str] + seq_breakers: List[str], ): # Convert seq_breakers to C array breaker_ptrs = (ctypes.c_char_p * len(seq_breakers))() for i, breaker in enumerate(seq_breakers): breaker_ptrs[i] = breaker.encode("utf-8") - + sampler = llama_cpp.llama_sampler_init_dry( model.vocab, n_ctx_train, @@ -786,25 +812,19 @@ def add_dry( dry_allowed_length, dry_penalty_last_n, breaker_ptrs, - len(seq_breakers) + len(seq_breakers), ) llama_cpp.llama_sampler_chain_add(self.sampler, sampler) - def add_logit_bias( - self, - n_vocab: int, - logit_bias: Dict[int, float] - ): + def add_logit_bias(self, n_vocab: int, logit_bias: Dict[int, float]): # Convert logit_bias dict to C array bias_array = (llama_cpp.llama_logit_bias * len(logit_bias))() for i, (token, bias) in enumerate(logit_bias.items()): bias_array[i].token = token bias_array[i].bias = bias - + sampler = llama_cpp.llama_sampler_init_logit_bias( - n_vocab, - len(logit_bias), - bias_array + n_vocab, len(logit_bias), bias_array ) llama_cpp.llama_sampler_chain_add(self.sampler, sampler) @@ -838,15 +858,17 @@ def reset(self): def clone(self): # NOTE: Custom samplers cannot be cloned due to Python callback limitations if self.custom_samplers: - raise NotImplementedError("Cannot clone LlamaSampler that contains custom samplers") - + raise NotImplementedError( + "Cannot clone LlamaSampler that contains custom samplers" + ) + cloned_sampler = llama_cpp.llama_sampler_clone(self.sampler) # Create a new wrapper around the cloned sampler new_sampler = LlamaSampler.__new__(LlamaSampler) new_sampler.sampler = cloned_sampler new_sampler.custom_samplers = [] new_sampler._exit_stack = ExitStack() - + def free_sampler(): if new_sampler.sampler is not None: llama_cpp.llama_sampler_free(new_sampler.sampler) diff --git a/llama_cpp/_logger.py b/llama_cpp/_logger.py index 787b3f108e..31d89d0995 100644 --- a/llama_cpp/_logger.py +++ b/llama_cpp/_logger.py @@ -25,6 +25,7 @@ _last_log_level = GGML_LOG_LEVEL_TO_LOGGING_LEVEL[0] + # typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); @llama_cpp.llama_log_callback def llama_log_callback( @@ -34,7 +35,9 @@ def llama_log_callback( ): # TODO: Correctly implement continue previous log global _last_log_level - log_level = GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level] if level != 5 else _last_log_level + log_level = ( + GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level] if level != 5 else _last_log_level + ) if logger.level <= GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level]: print(text.decode("utf-8"), end="", flush=True, file=sys.stderr) _last_log_level = log_level diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 71d94ebd82..21a7430a01 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -934,7 +934,8 @@ def generate( sample_idx += 1 if stopping_criteria is not None and stopping_criteria( - self._input_ids[: sample_idx], self._scores[sample_idx - self.n_tokens, :] + self._input_ids[:sample_idx], + self._scores[sample_idx - self.n_tokens, :], ): return tokens_or_none = yield token @@ -1157,9 +1158,9 @@ def _create_completion( bos_token_id: int = self.token_bos() cls_token_id: int = self._model.token_cls() sep_token_id: int = self._model.token_sep() - prefix_token_id: int = 0 # self._model.token_prefix() # TODO: Fix - middle_token_id: int = 0 # self._model.token_middle() # TODO: Fix - suffix_token_id: int = 0 # self._model.token_suffix() # TODO: Fix + prefix_token_id: int = 0 # self._model.token_prefix() # TODO: Fix + middle_token_id: int = 0 # self._model.token_middle() # TODO: Fix + suffix_token_id: int = 0 # self._model.token_suffix() # TODO: Fix add_space_prefix: bool = ( self.metadata.get("tokenizer.ggml.add_space_prefix", "true") == "true" ) @@ -1315,7 +1316,7 @@ def logit_bias_processor( if seed is not None: self.set_seed(seed) else: - self.set_seed(random.Random(self._seed).randint(0, 2 ** 32)) + self.set_seed(random.Random(self._seed).randint(0, 2**32)) finish_reason = "length" multibyte_fix = 0 @@ -2056,7 +2057,10 @@ def create_chat_completion_openai_v1( stream = kwargs.get("stream", False) # type: ignore assert isinstance(stream, bool) if stream: - return (ChatCompletionChunk(**chunk) for chunk in self.create_chat_completion(*args, **kwargs)) # type: ignore + return ( + ChatCompletionChunk(**chunk) + for chunk in self.create_chat_completion(*args, **kwargs) + ) # type: ignore else: return ChatCompletion(**self.create_chat_completion(*args, **kwargs)) # type: ignore except ImportError: @@ -2318,7 +2322,11 @@ def from_pretrained( if additional_files: for additonal_file_name in additional_files: # find the additional shard file: - matching_additional_files = [file for file in file_list if fnmatch.fnmatch(file, additonal_file_name)] + matching_additional_files = [ + file + for file in file_list + if fnmatch.fnmatch(file, additonal_file_name) + ] if len(matching_additional_files) == 0: raise ValueError( diff --git a/llama_cpp/llama_cache.py b/llama_cpp/llama_cache.py index e059e98e1b..5220c79337 100644 --- a/llama_cpp/llama_cache.py +++ b/llama_cpp/llama_cache.py @@ -52,9 +52,9 @@ class LlamaRAMCache(BaseLlamaCache): def __init__(self, capacity_bytes: int = (2 << 30)): super().__init__(capacity_bytes) self.capacity_bytes = capacity_bytes - self.cache_state: OrderedDict[ - Tuple[int, ...], "llama_cpp.llama.LlamaState" - ] = OrderedDict() + self.cache_state: OrderedDict[Tuple[int, ...], "llama_cpp.llama.LlamaState"] = ( + OrderedDict() + ) @property def cache_size(self): diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index f738ab9bb4..8e8ac7bb30 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -286,11 +286,15 @@ def _convert_text_completion_logprobs_to_chat( } for top_token, top_logprob in top_logprobs.items() ], - } for (token, logprob, top_logprobs) in zip(logprobs["tokens"], logprobs["token_logprobs"], logprobs["top_logprobs"]) + } + for (token, logprob, top_logprobs) in zip( + logprobs["tokens"], logprobs["token_logprobs"], logprobs["top_logprobs"] + ) ], "refusal": None, } + def _convert_text_completion_to_chat( completion: llama_types.Completion, ) -> llama_types.ChatCompletion: @@ -307,7 +311,9 @@ def _convert_text_completion_to_chat( "role": "assistant", "content": completion["choices"][0]["text"], }, - "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + completion["choices"][0]["logprobs"] + ), "finish_reason": completion["choices"][0]["finish_reason"], } ], @@ -351,7 +357,9 @@ def _convert_text_completion_chunks_to_chat( if chunk["choices"][0]["finish_reason"] is None else {} ), - "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + chunk["choices"][0]["logprobs"] + ), "finish_reason": chunk["choices"][0]["finish_reason"], } ], @@ -368,7 +376,9 @@ def _convert_completion_to_chat( llama_types.CreateChatCompletionResponse, Iterator[llama_types.ChatCompletionChunk] ]: if stream: - chunks: Iterator[llama_types.CreateCompletionStreamResponse] = completion_or_chunks # type: ignore + chunks: Iterator[llama_types.CreateCompletionStreamResponse] = ( + completion_or_chunks # type: ignore + ) return _convert_text_completion_chunks_to_chat(chunks) else: completion: llama_types.Completion = completion_or_chunks # type: ignore @@ -414,7 +424,9 @@ def _convert_completion_to_chat_function( } ], }, - "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + completion["choices"][0]["logprobs"] + ), "finish_reason": "tool_calls", } ], @@ -422,7 +434,9 @@ def _convert_completion_to_chat_function( } return chat_completion else: - chunks: Iterator[llama_types.CreateCompletionStreamResponse] = completion_or_chunks # type: ignore + chunks: Iterator[llama_types.CreateCompletionStreamResponse] = ( + completion_or_chunks # type: ignore + ) def _stream_response_to_function_stream( chunks: Iterator[llama_types.CreateCompletionStreamResponse], @@ -467,7 +481,9 @@ def _stream_response_to_function_stream( { "index": 0, "finish_reason": None, - "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + chunk["choices"][0]["logprobs"] + ), "delta": { "role": None, "content": None, @@ -504,7 +520,9 @@ def _stream_response_to_function_stream( { "index": 0, "finish_reason": None, - "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + chunk["choices"][0]["logprobs"] + ), "delta": { "role": None, "content": None, @@ -702,7 +720,7 @@ def chat_completion_handler( def hf_autotokenizer_to_chat_formatter( - pretrained_model_name_or_path: Union[str, os.PathLike[str]] + pretrained_model_name_or_path: Union[str, os.PathLike[str]], ) -> ChatFormatter: # https://huggingface.co/docs/transformers/main/chat_templating # https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1#instruction-format @@ -727,7 +745,7 @@ def format_autotokenizer( def hf_autotokenizer_to_chat_completion_handler( - pretrained_model_name_or_path: Union[str, os.PathLike[str]] + pretrained_model_name_or_path: Union[str, os.PathLike[str]], ) -> LlamaChatCompletionHandler: chat_formatter = hf_autotokenizer_to_chat_formatter(pretrained_model_name_or_path) return chat_formatter_to_chat_completion_handler(chat_formatter) @@ -1552,9 +1570,9 @@ def prepare_messages_for_inference( message["name"] = f"functions.{message['name']}" # Function call requests by assistant if "function_call" in message: - message["function_call"][ - "name" - ] = f"functions.{message['function_call']['name']}" + message["function_call"]["name"] = ( + f"functions.{message['function_call']['name']}" + ) all_messages.append(message) all_messages.append( @@ -1632,7 +1650,9 @@ def message_to_str(msg: llama_types.ChatCompletionRequestMessage): logits_processor=logits_processor, grammar=grammar, ) - return _convert_completion_to_chat(completion_or_completion_chunks, stream=stream) # type: ignore + return _convert_completion_to_chat( + completion_or_completion_chunks, stream=stream + ) # type: ignore if function_call is None or ( isinstance(function_call, str) and function_call == "auto" @@ -1748,7 +1768,9 @@ def message_to_str(msg: llama_types.ChatCompletionRequestMessage): } ], }, - "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + completion["choices"][0]["logprobs"] + ), "finish_reason": "tool_calls", } ], @@ -1789,9 +1811,9 @@ def functionary_v1_v2_chat_handler( SYSTEM_MESSAGE = """A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. The assistant calls functions with appropriate input when necessary""" tokenizer = llama.tokenizer_ - assert hasattr( - tokenizer, "hf_tokenizer" - ), "Please provide a valid hf_tokenizer_path from https://huggingface.co/meetkai when initializing the Llama class" + assert hasattr(tokenizer, "hf_tokenizer"), ( + "Please provide a valid hf_tokenizer_path from https://huggingface.co/meetkai when initializing the Llama class" + ) from transformers import AutoTokenizer if "<|START_OF_FUNCTION_CALL|>" in tokenizer.hf_tokenizer.additional_special_tokens: @@ -1941,9 +1963,9 @@ def prepare_messages_for_inference( message["name"] = f"functions.{message['name']}" # Function call requests by assistant if "function_call" in message: - message["function_call"][ - "name" - ] = f"functions.{message['function_call']['name']}" + message["function_call"]["name"] = ( + f"functions.{message['function_call']['name']}" + ) all_messages.append(message) if version == "v1": @@ -2005,7 +2027,9 @@ def prepare_messages_for_inference( completion_or_completion_chunks["choices"][0]["text"] = ( completion_or_completion_chunks["choices"][0]["text"].lstrip() ) - return _convert_completion_to_chat(completion_or_completion_chunks, stream=stream) # type: ignore + return _convert_completion_to_chat( + completion_or_completion_chunks, stream=stream + ) # type: ignore def get_grammar(function_call): function_body = None @@ -2160,7 +2184,9 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + chunk["choices"][0]["logprobs"] + ), "delta": { "role": None, "content": None, @@ -2262,7 +2288,9 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + chunk["choices"][0]["logprobs"] + ), "delta": { "role": "assistant", "content": None, @@ -2300,7 +2328,9 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + chunk["choices"][0]["logprobs"] + ), "delta": { "role": "assistant", "content": buffer.pop(0), @@ -2323,7 +2353,9 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + chunk["choices"][0]["logprobs"] + ), "delta": { "role": "assistant", "content": ( @@ -2409,7 +2441,9 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": _convert_text_completion_logprobs_to_chat(chunk["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + chunk["choices"][0]["logprobs"] + ), "delta": { "role": None, "content": None, @@ -2643,7 +2677,9 @@ def generate_streaming(tools, functions, function_call, prompt): choices=[ { "index": 0, - "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + completion["choices"][0]["logprobs"] + ), "message": { "role": "assistant", "content": None if content == "" else content, @@ -2716,20 +2752,20 @@ def _init_mtmd_context(self, llama_model: llama.Llama): with suppress_stdout_stderr(disable=self.verbose): # Get default parameters ctx_params = self._mtmd_cpp.mtmd_context_params_default() - ctx_params.use_gpu = True # TODO: Make this configurable + ctx_params.use_gpu = True # TODO: Make this configurable ctx_params.print_timings = self.verbose ctx_params.n_threads = llama_model.n_threads ctx_params.verbosity = 2 if self.verbose else 0 # GGML_LOG_LEVEL_INFO = 2 # Initialize mtmd context self.mtmd_ctx = self._mtmd_cpp.mtmd_init_from_file( - self.clip_model_path.encode(), - llama_model.model, - ctx_params + self.clip_model_path.encode(), llama_model.model, ctx_params ) if self.mtmd_ctx is None: - raise ValueError(f"Failed to load mtmd context from: {self.clip_model_path}") + raise ValueError( + f"Failed to load mtmd context from: {self.clip_model_path}" + ) # Check if vision is supported if not self._mtmd_cpp.mtmd_support_vision(self.mtmd_ctx): @@ -2756,12 +2792,12 @@ def _create_bitmap_from_bytes(self, image_bytes: bytes): bitmap = self._mtmd_cpp.mtmd_helper_bitmap_init_from_buf( self.mtmd_ctx, (ctypes.c_uint8 * len(image_bytes)).from_buffer(bytearray(image_bytes)), - len(image_bytes) + len(image_bytes), ) - + if bitmap is None: raise ValueError("Failed to create bitmap from image bytes") - + return bitmap def __call__( @@ -2820,10 +2856,10 @@ def __call__( trim_blocks=True, lstrip_blocks=True, ).from_string(self.CHAT_FORMAT) - + # Get the default media marker - media_marker = self._mtmd_cpp.mtmd_default_marker().decode('utf-8') - + media_marker = self._mtmd_cpp.mtmd_default_marker().decode("utf-8") + # Replace image URLs with media markers in the template text = template.render( messages=messages, @@ -2831,7 +2867,7 @@ def __call__( eos_token=llama.detokenize([llama.token_eos()]), bos_token=llama.detokenize([llama.token_bos()]), ) - + # Replace image URLs in text with media markers for image_url in image_urls: text = text.replace(image_url, media_marker) @@ -2851,7 +2887,7 @@ def __call__( # Create input text structure input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode('utf-8') + input_text.text = text.encode("utf-8") input_text.add_special = True input_text.parse_special = True @@ -2862,13 +2898,15 @@ def __call__( try: # Tokenize text and images together - bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps) + bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))( + *bitmaps + ) result = self._mtmd_cpp.mtmd_tokenize( self.mtmd_ctx, chunks, ctypes.byref(input_text), bitmap_array, - len(bitmaps) + len(bitmaps), ) if result != 0: @@ -2881,40 +2919,45 @@ def __call__( # Process each chunk n_past = llama_cpp.llama_pos(0) n_chunks = self._mtmd_cpp.mtmd_input_chunks_size(chunks) - + for i in range(n_chunks): chunk = self._mtmd_cpp.mtmd_input_chunks_get(chunks, i) if chunk is None: continue chunk_type = self._mtmd_cpp.mtmd_input_chunk_get_type(chunk) - + if chunk_type == self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_TEXT: # Handle text chunk n_tokens_out = ctypes.c_size_t() tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text( chunk, ctypes.byref(n_tokens_out) ) - + if tokens_ptr and n_tokens_out.value > 0: # Convert ctypes array to Python list tokens = [tokens_ptr[j] for j in range(n_tokens_out.value)] - + if llama.n_tokens + len(tokens) > llama.n_ctx(): raise ValueError( f"Prompt exceeds n_ctx: {llama.n_tokens + len(tokens)} > {llama.n_ctx()}" ) llama.eval(tokens) - - elif chunk_type in [self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_IMAGE, self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_AUDIO]: + + elif chunk_type in [ + self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_IMAGE, + self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_AUDIO, + ]: # Handle image/audio chunk using helper - chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk) - + chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens( + chunk + ) + if llama.n_tokens + chunk_n_tokens > llama.n_ctx(): raise ValueError( f"Prompt exceeds n_ctx: {llama.n_tokens + chunk_n_tokens} > {llama.n_ctx()}" ) - + new_n_past = llama_cpp.llama_pos(0) result = self._mtmd_cpp.mtmd_helper_eval_chunk_single( self.mtmd_ctx, @@ -2924,12 +2967,14 @@ def __call__( llama_cpp.llama_seq_id(0), llama.n_batch, False, # logits_last - ctypes.byref(new_n_past) + ctypes.byref(new_n_past), ) - + if result != 0: - raise ValueError(f"Failed to evaluate chunk: error code {result}") - + raise ValueError( + f"Failed to evaluate chunk: error code {result}" + ) + # Update llama's token count llama.n_tokens = new_n_past.value @@ -3019,7 +3064,7 @@ def __call__( grammar=grammar, logit_bias=logit_bias, ) - + if tool is not None: tool_name = tool["function"]["name"] return _convert_completion_to_chat_function( @@ -3032,10 +3077,12 @@ def _load_image(image_url: str) -> bytes: # TODO: Add Pillow support for other image formats beyond (jpg, png) if image_url.startswith("data:"): import base64 + image_bytes = base64.b64decode(image_url.split(",")[1]) return image_bytes else: import urllib.request + with urllib.request.urlopen(image_url) as f: image_bytes = f.read() return image_bytes @@ -3062,6 +3109,7 @@ def get_image_urls(messages: List[llama_types.ChatCompletionRequestMessage]): @staticmethod def split_text_on_image_urls(text: str, image_urls: List[str]): """This method is no longer used in the new implementation.""" + def find_first(s: str, substrs: List[str]): for i, substr in enumerate(substrs): pos = s.find(substr) @@ -3443,7 +3491,6 @@ class MiniCPMv26ChatHandler(Llava15ChatHandler): "{% endif %}" "{% endif %}" "{% endfor %}" - "{% for content in message['content'] %}" "{% if content.type == 'text' %}" "{{ content.text }}" @@ -3465,8 +3512,8 @@ class Qwen25VLChatHandler(Llava15ChatHandler): DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant." CHAT_FORMAT = ( - #"{% set image_count = namespace(value=0) %}" - #"{% set video_count = namespace(value=0) %}" + # "{% set image_count = namespace(value=0) %}" + # "{% set video_count = namespace(value=0) %}" "{% for message in messages %}" "{% if loop.first and message['role'] != 'system' %}" "<|im_start|>system\n" @@ -3483,7 +3530,7 @@ class Qwen25VLChatHandler(Llava15ChatHandler): "{% else %}" "{{ content.image_url.url }}" "{% endif %}" - #"{% set image_count.value = image_count.value + 1 %}" + # "{% set image_count.value = image_count.value + 1 %}" "{% elif content['type'] == 'text' %}" "{{ content['text'] }}" "{% endif %}" @@ -3495,25 +3542,28 @@ class Qwen25VLChatHandler(Llava15ChatHandler): ) def __call__(self, **kwargs): - llama = kwargs['llama'] + llama = kwargs["llama"] # Clear state for multiple runs llama.reset() llama._ctx.kv_cache_clear() llama.n_tokens = 0 - if hasattr(llama, 'input_ids'): + if hasattr(llama, "input_ids"): llama.input_ids.fill(0) # Clear any handler state - if hasattr(self, '_last_image_embed'): + if hasattr(self, "_last_image_embed"): self._last_image_embed = None self._last_image_hash = None if self.verbose: - messages = kwargs.get('messages', []) + messages = kwargs.get("messages", []) image_count = len(self.get_image_urls(messages)) - print(f"Minimal - Cleared state, processing {image_count} images", file=sys.stderr) + print( + f"Minimal - Cleared state, processing {image_count} images", + file=sys.stderr, + ) # Use parent implementation return super().__call__(**kwargs) @@ -3636,7 +3686,9 @@ def chatml_function_calling( stop = ( [stop, "<|im_end|>"] if isinstance(stop, str) - else stop + ["<|im_end|>"] if stop else ["<|im_end|>"] + else stop + ["<|im_end|>"] + if stop + else ["<|im_end|>"] ) # Case 1: No tool choice by user @@ -3738,7 +3790,7 @@ def chatml_function_calling( # Case 3: Automatic tool choice assert isinstance(tool_choice, str) and tool_choice == "auto" function_names = " | ".join( - [f'''"functions.{tool['function']['name']}:"''' for tool in tools] + [f'''"functions.{tool["function"]["name"]}:"''' for tool in tools] ) initial_gbnf_tool_grammar = ( """root ::= functions | "message:"\n""" @@ -3914,7 +3966,9 @@ def chatml_function_calling( { "finish_reason": "tool_calls", "index": 0, - "logprobs": _convert_text_completion_logprobs_to_chat(completion["choices"][0]["logprobs"]), + "logprobs": _convert_text_completion_logprobs_to_chat( + completion["choices"][0]["logprobs"] + ), "message": { "role": "assistant", "content": None, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 711d42a6ae..f13af67f3b 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -33,7 +33,11 @@ # Specify the base name of the shared library to load _lib_base_name = "llama" _override_base_path = os.environ.get("LLAMA_CPP_LIB_PATH") -_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _override_base_path is None else pathlib.Path(_override_base_path) +_base_path = ( + pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" + if _override_base_path is None + else pathlib.Path(_override_base_path) +) # Load the library _lib = load_shared_library(_lib_base_name, _base_path) @@ -559,6 +563,7 @@ class llama_token_data_array(ctypes.Structure): # typedef struct llama_batch { # int32_t n_tokens; + # llama_token * token; # float * embd; # llama_pos * pos; @@ -688,6 +693,7 @@ class llama_model_kv_override(ctypes.Structure): # // override key-value pairs of the model meta data # const struct llama_model_kv_override * kv_overrides; + # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights # bool use_mmap; // use mmap if possible @@ -716,7 +722,9 @@ class llama_model_params(ctypes.Structure): if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused - tensor_buft_overrides: CtypesArray[llama_model_tensor_buft_override] # NOTE: unused + tensor_buft_overrides: CtypesArray[ + llama_model_tensor_buft_override + ] # NOTE: unused n_gpu_layers: int split_mode: int main_gpu: int @@ -731,8 +739,8 @@ class llama_model_params(ctypes.Structure): use_extra_bufts: bool _fields_ = [ - ("devices", ctypes.c_void_p), # NOTE: unnused - ("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused + ("devices", ctypes.c_void_p), # NOTE: unnused + ("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), @@ -784,6 +792,7 @@ class llama_model_params(ctypes.Structure): # ggml_abort_callback abort_callback; # void * abort_callback_data; + # // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value. # bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU @@ -1137,8 +1146,7 @@ def llama_backend_free(): [ctypes.c_int], None, ) -def llama_numa_init(numa: int, /): - ... +def llama_numa_init(numa: int, /): ... # // Optional: an auto threadpool gets created in ggml if not passed explicitly @@ -1164,8 +1172,7 @@ def llama_numa_init(numa: int, /): ) def llama_load_model_from_file( path_model: bytes, params: llama_model_params, / -) -> Optional[llama_model_p]: - ... +) -> Optional[llama_model_p]: ... # // Load the model from a file @@ -1230,8 +1237,7 @@ def llama_model_save_to_file(model: llama_model_p, path_model: bytes, /): [llama_model_p_ctypes], None, ) -def llama_free_model(model: llama_model_p, /): - ... +def llama_free_model(model: llama_model_p, /): ... # LLAMA_API void llama_model_free(struct llama_model * model); @@ -1240,8 +1246,7 @@ def llama_free_model(model: llama_model_p, /): [llama_model_p_ctypes], None, ) -def llama_model_free(model: llama_model_p, /): - ... +def llama_model_free(model: llama_model_p, /): ... # LLAMA_API struct llama_context * llama_init_from_model( @@ -1254,8 +1259,7 @@ def llama_model_free(model: llama_model_p, /): ) def llama_init_from_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: - ... +) -> Optional[llama_context_p]: ... # DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model( @@ -1269,8 +1273,7 @@ def llama_init_from_model( ) def llama_new_context_with_model( model: llama_model_p, params: llama_context_params, / -) -> Optional[llama_context_p]: - ... +) -> Optional[llama_context_p]: ... # // Frees all allocated memory @@ -1291,104 +1294,87 @@ def llama_free(ctx: llama_context_p, /): [], ctypes.c_int64, ) -def llama_time_us() -> int: - ... +def llama_time_us() -> int: ... # LLAMA_API size_t llama_max_devices(void); @ctypes_function("llama_max_devices", [], ctypes.c_size_t) -def llama_max_devices() -> int: - ... +def llama_max_devices() -> int: ... # LLAMA_API size_t llama_max_parallel_sequences(void); @ctypes_function("llama_max_parallel_sequences", [], ctypes.c_size_t) -def llama_max_parallel_sequences() -> int: - ... +def llama_max_parallel_sequences() -> int: ... # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) -def llama_supports_mmap() -> bool: - ... +def llama_supports_mmap() -> bool: ... # LLAMA_API bool llama_supports_mlock (void); @ctypes_function("llama_supports_mlock", [], ctypes.c_bool) -def llama_supports_mlock() -> bool: - ... +def llama_supports_mlock() -> bool: ... # LLAMA_API bool llama_supports_gpu_offload(void); @ctypes_function("llama_supports_gpu_offload", [], ctypes.c_bool) -def llama_supports_gpu_offload() -> bool: - ... +def llama_supports_gpu_offload() -> bool: ... # LLAMA_API bool llama_supports_rpc (void); @ctypes_function("llama_supports_rpc", [], ctypes.c_bool) -def llama_supports_rpc() -> bool: - ... +def llama_supports_rpc() -> bool: ... # LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); @ctypes_function("llama_n_ctx", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ctx(ctx: llama_context_p, /) -> int: - ... +def llama_n_ctx(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_batch(ctx: llama_context_p, /) -> int: - ... +def llama_n_batch(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); @ctypes_function("llama_n_ubatch", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_ubatch(ctx: llama_context_p, /) -> int: - ... +def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... # LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); @ctypes_function("llama_n_seq_max", [llama_context_p_ctypes], ctypes.c_uint32) -def llama_n_seq_max(ctx: llama_context_p, /) -> int: - ... +def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... # DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_ctx_train(model: llama_model_p, /) -> int: - ... +def llama_n_ctx_train(model: llama_model_p, /) -> int: ... # DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_embd(model: llama_model_p, /) -> int: - ... +def llama_n_embd(model: llama_model_p, /) -> int: ... # DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_layer(model: llama_model_p, /) -> int: - ... +def llama_n_layer(model: llama_model_p, /) -> int: ... # DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); @ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32) -def llama_n_head(model: llama_model_p, /) -> int: - ... +def llama_n_head(model: llama_model_p, /) -> int: ... # DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); @ctypes_function("llama_n_vocab", [llama_vocab_p_ctypes], ctypes.c_int32) -def llama_n_vocab(model: llama_vocab_p, /) -> int: - ... +def llama_n_vocab(model: llama_vocab_p, /) -> int: ... # LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) -def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: - ... +def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... # LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx); @@ -1400,8 +1386,7 @@ def llama_get_memory(ctx: llama_context_p, /) -> Optional[llama_memory_t]: # LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); @ctypes_function("llama_pooling_type", [llama_context_p_ctypes], ctypes.c_int) -def llama_pooling_type(ctx: llama_context_p, /) -> int: - ... +def llama_pooling_type(ctx: llama_context_p, /) -> int: ... # DEPRECATED(LLAMA_API struct llama_kv_cache * llama_get_kv_self(struct llama_context * ctx), "use llama_get_memory instead"); @@ -1417,57 +1402,50 @@ def llama_get_kv_self(ctx: llama_context_p, /) -> Optional[llama_kv_cache_p]: # LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); @ctypes_function("llama_model_get_vocab", [llama_model_p_ctypes], llama_vocab_p_ctypes) -def llama_model_get_vocab(model: llama_model_p, /) -> Optional[llama_vocab_p]: - ... +def llama_model_get_vocab(model: llama_model_p, /) -> Optional[llama_vocab_p]: ... # LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model); @ctypes_function("llama_model_rope_type", [llama_model_p_ctypes], ctypes.c_int) -def llama_model_rope_type(model: llama_model_p, /) -> int: - ... +def llama_model_rope_type(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model); @ctypes_function("llama_model_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) -def llama_model_n_ctx_train(model: llama_model_p, /) -> int: - ... +def llama_model_n_ctx_train(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model); @ctypes_function("llama_model_n_embd", [llama_model_p_ctypes], ctypes.c_int32) -def llama_model_n_embd(model: llama_model_p, /) -> int: - ... +def llama_model_n_embd(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model); @ctypes_function("llama_model_n_layer", [llama_model_p_ctypes], ctypes.c_int32) -def llama_model_n_layer(model: llama_model_p, /) -> int: - ... +def llama_model_n_layer(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); @ctypes_function("llama_model_n_head", [llama_model_p_ctypes], ctypes.c_int32) -def llama_model_n_head(model: llama_model_p, /) -> int: - ... +def llama_model_n_head(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model); @ctypes_function("llama_model_n_head_kv", [llama_model_p_ctypes], ctypes.c_int32) -def llama_model_n_head_kv(model: llama_model_p, /) -> int: - ... +def llama_model_n_head_kv(model: llama_model_p, /) -> int: ... # LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model); @ctypes_function("llama_model_n_swa", [llama_model_p_ctypes], ctypes.c_int32) -def llama_model_n_swa(model: llama_model_p, /) -> int: - ... +def llama_model_n_swa(model: llama_model_p, /) -> int: ... # // Get the model's RoPE frequency scaling factor # LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); -@ctypes_function("llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float) -def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: - ... +@ctypes_function( + "llama_model_rope_freq_scale_train", [llama_model_p_ctypes], ctypes.c_float +) +def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: ... # // Returns the number of classifier outputs (only valid for classifier models) @@ -1481,7 +1459,9 @@ def llama_model_n_cls_out(model: llama_model_p, /) -> int: # // Returns label of classifier output by index ( Optional[bytes]: """Returns label of classifier output by index. Returns None if no label provided""" ... @@ -1489,14 +1469,12 @@ def llama_model_cls_label(model: llama_model_p, i: int, /) -> Optional[bytes]: # LLAMA_API enum llama_vocab_type llama_vocab_type (const struct llama_model * model); @ctypes_function("llama_vocab_type", [llama_vocab_p_ctypes], ctypes.c_int) -def llama_vocab_type(vocab: llama_vocab_p, /) -> int: - ... +def llama_vocab_type(vocab: llama_vocab_p, /) -> int: ... # LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab); @ctypes_function("llama_vocab_n_tokens", [llama_vocab_p_ctypes], ctypes.c_int32) -def llama_vocab_n_tokens(vocab: llama_vocab_p, /) -> int: - ... +def llama_vocab_n_tokens(vocab: llama_vocab_p, /) -> int: ... # // Functions to access the model's GGUF metadata scalar values @@ -1611,8 +1589,14 @@ def llama_model_size(model: llama_model_p, /) -> int: # // Get the default chat template. Returns nullptr if not available # // If name is NULL, returns the default chat template # LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name); -@ctypes_function("llama_model_chat_template", [llama_model_p_ctypes, ctypes.c_char_p], ctypes.c_char_p) -def llama_model_chat_template(model: llama_model_p, name: Optional[bytes], /) -> Optional[bytes]: +@ctypes_function( + "llama_model_chat_template", + [llama_model_p_ctypes, ctypes.c_char_p], + ctypes.c_char_p, +) +def llama_model_chat_template( + model: llama_model_p, name: Optional[bytes], / +) -> Optional[bytes]: """Get the default chat template. Returns None if not available If name is None, returns the default chat template""" ... @@ -1699,6 +1683,7 @@ def llama_model_quantize( # // Adapters # // + # // Load a LoRA adapter from file # LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init( # struct llama_model * model, @@ -1710,8 +1695,7 @@ def llama_model_quantize( ) def llama_adapter_lora_init( model: llama_model_p, path_lora: bytes, / -) -> Optional[llama_adapter_lora_p]: - ... +) -> Optional[llama_adapter_lora_p]: ... # // Manually free a LoRA adapter @@ -1722,8 +1706,7 @@ def llama_adapter_lora_init( [llama_adapter_lora_p_ctypes], None, ) -def llama_adapter_lora_free(adapter: llama_adapter_lora_p, /): - ... +def llama_adapter_lora_free(adapter: llama_adapter_lora_p, /): ... # // The following functions operate on a llama_context, hence the naming: llama_verb_... @@ -1825,6 +1808,7 @@ def llama_apply_adapter_cvec( # // Memory # // + # // Clear the memory contents # // If data == true, the data buffers will also be cleared together with the metadata # LLAMA_API void llama_memory_clear( @@ -1916,9 +1900,7 @@ def llama_memory_seq_cp( # LLAMA_API void llama_memory_seq_keep( # llama_memory_t mem, # llama_seq_id seq_id); -@ctypes_function( - "llama_memory_seq_keep", [llama_memory_t_ctypes, llama_seq_id], None -) +@ctypes_function("llama_memory_seq_keep", [llama_memory_t_ctypes, llama_seq_id], None) def llama_memory_seq_keep(mem: llama_memory_t, seq_id: Union[llama_seq_id, int], /): """Removes all tokens that do not belong to the specified sequence""" ... @@ -2040,13 +2022,12 @@ def llama_memory_can_shift(mem: llama_memory_t, /) -> bool: # // KV cache for self-attention (TODO: deprecate in favor of llama_memory) # // + # // Returns the number of tokens in the KV cache (slow, use only for debug) # // If a KV cell has multiple sequences assigned to it, it will be counted multiple times # DEPRECATED(LLAMA_API int32_t llama_kv_self_n_tokens(const struct llama_context * ctx), # "Use llama_kv_self_seq_pos_max() and llama_kv_self_seq_pos_min() instead (https://github.com/ggml-org/llama.cpp/issues/13793)"); -@ctypes_function( - "llama_kv_self_n_tokens", [llama_context_p_ctypes], ctypes.c_int32 -) +@ctypes_function("llama_kv_self_n_tokens", [llama_context_p_ctypes], ctypes.c_int32) def llama_kv_self_n_tokens(ctx: llama_context_p, /) -> int: """Returns the number of tokens in the KV cache (slow, use only for debug) (DEPRECATED)""" ... @@ -2055,9 +2036,7 @@ def llama_kv_self_n_tokens(ctx: llama_context_p, /) -> int: # // Returns the number of used KV cells (i.e. have at least one sequence assigned to them) # DEPRECATED(LLAMA_API int32_t llama_kv_self_used_cells(const struct llama_context * ctx), # "Use llama_kv_self_seq_pos_max() and llama_kv_self_seq_pos_min() instead (https://github.com/ggml-org/llama.cpp/issues/13793)"); -@ctypes_function( - "llama_kv_self_used_cells", [llama_context_p_ctypes], ctypes.c_int32 -) +@ctypes_function("llama_kv_self_used_cells", [llama_context_p_ctypes], ctypes.c_int32) def llama_kv_self_used_cells(ctx: llama_context_p, /) -> int: """Returns the number of used KV cells (DEPRECATED)""" ... @@ -2067,9 +2046,7 @@ def llama_kv_self_used_cells(ctx: llama_context_p, /) -> int: # DEPRECATED(LLAMA_API void llama_kv_self_clear( # struct llama_context * ctx), # "Use llama_memory_clear() instead"); -@ctypes_function( - "llama_kv_self_clear", [llama_context_p_ctypes], None -) +@ctypes_function("llama_kv_self_clear", [llama_context_p_ctypes], None) def llama_kv_self_clear(ctx: llama_context_p, /): """Clear the KV cache (DEPRECATED)""" ... @@ -2146,9 +2123,7 @@ def llama_kv_self_seq_cp( # struct llama_context * ctx, # llama_seq_id seq_id), # "Use llama_memory_seq_keep() instead"); -@ctypes_function( - "llama_kv_self_seq_keep", [llama_context_p_ctypes, llama_seq_id], None -) +@ctypes_function("llama_kv_self_seq_keep", [llama_context_p_ctypes, llama_seq_id], None) def llama_kv_self_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /): """Keep only specified sequence in KV cache (DEPRECATED)""" ... @@ -2292,6 +2267,7 @@ def llama_kv_self_update(ctx: llama_context_p, /): # // State / sessions # // + # // Returns the *actual* size in bytes of the state # // (logits, embedding and memory) # // Only use when saving the state, not when restoring it, otherwise the size may be too small. @@ -2420,8 +2396,7 @@ def llama_state_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: - ... +) -> bool: ... # LLAMA_API DEPRECATED(bool llama_load_session_file( @@ -2449,8 +2424,7 @@ def llama_load_session_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> bool: - ... +) -> bool: ... # LLAMA_API bool llama_state_save_file( @@ -2474,8 +2448,7 @@ def llama_state_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: - ... +) -> bool: ... # LLAMA_API DEPRECATED(bool llama_save_session_file( @@ -2500,8 +2473,7 @@ def llama_save_session_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> bool: - ... +) -> bool: ... # // Get the exact size needed to copy the state of a single sequence @@ -2599,8 +2571,7 @@ def llama_state_seq_save_file( tokens: CtypesArray[llama_token], n_token_count: Union[ctypes.c_size_t, int], /, -) -> int: - ... +) -> int: ... # LLAMA_API size_t llama_state_seq_load_file( @@ -2630,14 +2601,14 @@ def llama_state_seq_load_file( n_token_capacity: Union[ctypes.c_size_t, int], n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, -) -> int: - ... +) -> int: ... # // # // Decoding # // + # // Return batch for single sequence of tokens # // The sequence ID will be fixed to 0 # // The position of the tokens will be tracked automatically by llama_decode @@ -2947,14 +2918,14 @@ def llama_get_embeddings_seq( # // Vocab # // + # LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token); @ctypes_function( "llama_vocab_get_text", [llama_vocab_p_ctypes, llama_token], ctypes.c_char_p ) def llama_vocab_get_text( vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> bytes: - ... +) -> bytes: ... # LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token); @@ -2963,8 +2934,7 @@ def llama_vocab_get_text( ) def llama_vocab_get_score( vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> float: - ... +) -> float: ... # LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token); @@ -2973,8 +2943,7 @@ def llama_vocab_get_score( ) def llama_vocab_get_attr( vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> int: - ... +) -> int: ... # // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) @@ -3055,8 +3024,7 @@ def llama_vocab_mask(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], ctypes.c_bool, ) -def llama_vocab_get_add_bos(vocab: llama_vocab_p, /) -> bool: - ... +def llama_vocab_get_add_bos(vocab: llama_vocab_p, /) -> bool: ... # LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab); @@ -3065,8 +3033,7 @@ def llama_vocab_get_add_bos(vocab: llama_vocab_p, /) -> bool: [llama_vocab_p_ctypes], ctypes.c_bool, ) -def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: - ... +def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: ... # LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab); @@ -3075,8 +3042,7 @@ def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: [llama_vocab_p_ctypes], ctypes.c_bool, ) -def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: - ... +def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ... # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); @@ -3085,8 +3051,7 @@ def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: [llama_vocab_p_ctypes], llama_token, ) -def llama_vocab_fim_pre(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_vocab_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ... # LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab); @@ -3095,8 +3060,7 @@ def llama_vocab_fim_pre(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_vocab_fim_suf(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_vocab_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ... # LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab); @@ -3105,8 +3069,7 @@ def llama_vocab_fim_suf(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_vocab_fim_mid(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_vocab_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ... # LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab); @@ -3115,8 +3078,7 @@ def llama_vocab_fim_mid(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_vocab_fim_pad(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_vocab_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ... # LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab); @@ -3125,8 +3087,7 @@ def llama_vocab_fim_pad(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_vocab_fim_rep(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_vocab_fim_rep(vocab: llama_vocab_p, /) -> llama_token: ... # LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab); @@ -3135,8 +3096,7 @@ def llama_vocab_fim_rep(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ... # DEPRECATED functions @@ -3148,8 +3108,7 @@ def llama_vocab_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ) def llama_token_get_text( vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> bytes: - ... +) -> bytes: ... # DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); @@ -3160,8 +3119,8 @@ def llama_token_get_text( ) def llama_token_get_score( vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> float: - ... +) -> float: ... + # DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); @ctypes_function( @@ -3171,8 +3130,8 @@ def llama_token_get_score( ) def llama_token_get_attr( vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> int: - ... +) -> int: ... + # DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); @ctypes_function( @@ -3182,8 +3141,8 @@ def llama_token_get_attr( ) def llama_token_is_eog( vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> bool: - ... +) -> bool: ... + # DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); @ctypes_function( @@ -3193,8 +3152,8 @@ def llama_token_is_eog( ) def llama_token_is_control( vocab: llama_vocab_p, token: Union[llama_token, int], / -) -> bool: - ... +) -> bool: ... + # DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); @ctypes_function( @@ -3202,8 +3161,8 @@ def llama_token_is_control( [llama_vocab_p_ctypes], llama_token, ) -def llama_token_bos(vocab: llama_vocab_p, /) -> int: - ... +def llama_token_bos(vocab: llama_vocab_p, /) -> int: ... + # DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); @ctypes_function( @@ -3211,8 +3170,8 @@ def llama_token_bos(vocab: llama_vocab_p, /) -> int: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_eos(vocab: llama_vocab_p, /) -> int: - ... +def llama_token_eos(vocab: llama_vocab_p, /) -> int: ... + # DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); @ctypes_function( @@ -3220,8 +3179,8 @@ def llama_token_eos(vocab: llama_vocab_p, /) -> int: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_eot(vocab: llama_vocab_p, /) -> int: - ... +def llama_token_eot(vocab: llama_vocab_p, /) -> int: ... + # DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); @ctypes_function( @@ -3229,8 +3188,8 @@ def llama_token_eot(vocab: llama_vocab_p, /) -> int: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_cls(vocab: llama_vocab_p, /) -> int: - ... +def llama_token_cls(vocab: llama_vocab_p, /) -> int: ... + # DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); @ctypes_function( @@ -3238,8 +3197,7 @@ def llama_token_cls(vocab: llama_vocab_p, /) -> int: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_sep(vocab: llama_vocab_p, /) -> int: - ... +def llama_token_sep(vocab: llama_vocab_p, /) -> int: ... # DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); @@ -3248,8 +3206,7 @@ def llama_token_sep(vocab: llama_vocab_p, /) -> int: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_nl(vocab: llama_vocab_p, /) -> int: - ... +def llama_token_nl(vocab: llama_vocab_p, /) -> int: ... # DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); @@ -3258,8 +3215,7 @@ def llama_token_nl(vocab: llama_vocab_p, /) -> int: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_pad(vocab: llama_vocab_p, /) -> int: - ... +def llama_token_pad(vocab: llama_vocab_p, /) -> int: ... # DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); @@ -3268,8 +3224,8 @@ def llama_token_pad(vocab: llama_vocab_p, /) -> int: [llama_vocab_p_ctypes], ctypes.c_bool, ) -def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: - ... +def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: ... + # DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); @ctypes_function( @@ -3277,8 +3233,7 @@ def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: [llama_vocab_p_ctypes], ctypes.c_bool, ) -def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: - ... +def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: ... # DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); @@ -3287,8 +3242,8 @@ def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ... + # DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); @ctypes_function( @@ -3296,8 +3251,8 @@ def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ... + # DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); @ctypes_function( @@ -3305,8 +3260,8 @@ def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ... + # DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); @ctypes_function( @@ -3314,8 +3269,8 @@ def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ... + # DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); @ctypes_function( @@ -3323,8 +3278,8 @@ def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: ... + # DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); @ctypes_function( @@ -3332,8 +3287,8 @@ def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ... + # // CLS is equivalent to BOS # DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification @@ -3343,8 +3298,7 @@ def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: [llama_vocab_p_ctypes], llama_token, ) -def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: - ... +def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: ... # // @@ -3353,6 +3307,7 @@ def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: # // The API is thread-safe. # // + # /// @details Convert the provided text into tokens. # /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. # /// @return Returns the number of tokens on success, no more than n_tokens_max @@ -3512,6 +3467,7 @@ def llama_detokenize( # // Chat templates # // + # /// Apply chat template. Inspired by hf apply_chat_template() on python. # /// Both "model" and "custom_template" are optional, but at least one is required. "custom_template" has higher precedence than "model" # /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template @@ -3535,9 +3491,9 @@ def llama_detokenize( ctypes.c_char_p, # tmpl ctypes.POINTER(llama_chat_message), # chat ctypes.c_size_t, # n_msg - ctypes.c_bool, # add_ass (added) + ctypes.c_bool, # add_ass (added) ctypes.c_char_p, # buf - ctypes.c_int32, # length + ctypes.c_int32, # length ], ctypes.c_int32, ) @@ -3611,11 +3567,11 @@ def llama_chat_builtin_templates( # struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL # void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL + # // TODO: API for internal libllama usage for appending the sampling to an existing ggml_cgraph # //void (*apply_ggml) (struct llama_sampler * smpl, ...); # }; -class llama_sampler_i(ctypes.Structure): - ... +class llama_sampler_i(ctypes.Structure): ... # struct llama_sampler { @@ -3662,8 +3618,7 @@ class llama_sampler(ctypes.Structure): ) def llama_sampler_init( iface: ctypes.POINTER(llama_sampler_i), ctx: llama_sampler_context_t, / -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl); @@ -3672,8 +3627,7 @@ def llama_sampler_init( [llama_sampler_p_ctypes], ctypes.c_char_p, ) -def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: - ... +def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: ... # LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token); @@ -3682,8 +3636,7 @@ def llama_sampler_name(smpl: llama_sampler_p, /) -> bytes: [llama_sampler_p_ctypes, llama_token], None, ) -def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /): - ... +def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], /): ... # LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p); @@ -3694,8 +3647,7 @@ def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], ) def llama_sampler_apply( smpl: llama_sampler_p, cur_p: CtypesArray[llama_token_data_array], / -): - ... +): ... # LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl); @@ -3704,8 +3656,7 @@ def llama_sampler_apply( [llama_sampler_p_ctypes], None, ) -def llama_sampler_reset(smpl: llama_sampler_p, /): - ... +def llama_sampler_reset(smpl: llama_sampler_p, /): ... # LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl); @@ -3714,8 +3665,7 @@ def llama_sampler_reset(smpl: llama_sampler_p, /): [llama_sampler_p_ctypes], llama_sampler_p_ctypes, ) -def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: - ... +def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... # // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) @@ -3725,21 +3675,22 @@ def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: [llama_sampler_p_ctypes], None, ) -def llama_sampler_free(smpl: llama_sampler_p, /): - ... +def llama_sampler_free(smpl: llama_sampler_p, /): ... # // llama_sampler_chain # // a type of llama_sampler that can chain multiple samplers one after another + # LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params); @ctypes_function( "llama_sampler_chain_init", [llama_sampler_chain_params], llama_sampler_p_ctypes, ) -def llama_sampler_chain_init(params: llama_sampler_chain_params, /) -> llama_sampler_p: - ... +def llama_sampler_chain_init( + params: llama_sampler_chain_params, / +) -> llama_sampler_p: ... # // important: takes ownership of the sampler object and will free it when llama_sampler_free is called @@ -3749,8 +3700,7 @@ def llama_sampler_chain_init(params: llama_sampler_chain_params, /) -> llama_sam [llama_sampler_p_ctypes, llama_sampler_p_ctypes], None, ) -def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): - ... +def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): ... # LLAMA_API struct llama_sampler * llama_sampler_chain_get(const struct llama_sampler * chain, int32_t i); @@ -3761,8 +3711,7 @@ def llama_sampler_chain_add(chain: llama_sampler_p, smpl: llama_sampler_p, /): ) def llama_sampler_chain_get( chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); @@ -3771,8 +3720,7 @@ def llama_sampler_chain_get( [llama_sampler_p_ctypes], ctypes.c_int, ) -def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: - ... +def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ... # // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed @@ -3784,22 +3732,20 @@ def llama_sampler_chain_n(chain: llama_sampler_p, /) -> int: ) def llama_sampler_chain_remove( chain: llama_sampler_p, i: Union[ctypes.c_int32, int], / -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # // available samplers: + # LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void); @ctypes_function("llama_sampler_init_greedy", [], llama_sampler_p_ctypes) -def llama_sampler_init_greedy() -> llama_sampler_p: - ... +def llama_sampler_init_greedy() -> llama_sampler_p: ... # LLAMA_API struct llama_sampler * llama_sampler_init_dist (uint32_t seed); @ctypes_function("llama_sampler_init_dist", [ctypes.c_uint32], llama_sampler_p_ctypes) -def llama_sampler_init_dist(seed: int) -> llama_sampler_p: - ... +def llama_sampler_init_dist(seed: int) -> llama_sampler_p: ... # /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. @@ -3807,16 +3753,14 @@ def llama_sampler_init_dist(seed: int) -> llama_sampler_p: # DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void), # "will be removed in the future (see https://github.com/ggml-org/llama.cpp/pull/9896#discussion_r1800920915)"); @ctypes_function("llama_sampler_init_softmax", [], llama_sampler_p_ctypes) -def llama_sampler_init_softmax() -> llama_sampler_p: - ... +def llama_sampler_init_softmax() -> llama_sampler_p: ... # /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 # /// Setting k <= 0 makes this a noop # LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); @ctypes_function("llama_sampler_init_top_k", [ctypes.c_int32], llama_sampler_p_ctypes) -def llama_sampler_init_top_k(k: int) -> llama_sampler_p: - ... +def llama_sampler_init_top_k(k: int) -> llama_sampler_p: ... # /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 @@ -3826,8 +3770,7 @@ def llama_sampler_init_top_k(k: int) -> llama_sampler_p: [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: - ... +def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: ... # /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841 @@ -3837,8 +3780,7 @@ def llama_sampler_init_top_p(p: float, min_keep: int) -> llama_sampler_p: [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: - ... +def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: ... # /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. @@ -3848,15 +3790,13 @@ def llama_sampler_init_min_p(p: float, min_keep: int) -> llama_sampler_p: [ctypes.c_float, ctypes.c_size_t], llama_sampler_p_ctypes, ) -def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: - ... +def llama_sampler_init_typical(p: float, min_keep: int) -> llama_sampler_p: ... # /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf # LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); @ctypes_function("llama_sampler_init_temp", [ctypes.c_float], llama_sampler_p_ctypes) -def llama_sampler_init_temp(t: float) -> llama_sampler_p: - ... +def llama_sampler_init_temp(t: float) -> llama_sampler_p: ... # /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. @@ -3868,8 +3808,7 @@ def llama_sampler_init_temp(t: float) -> llama_sampler_p: ) def llama_sampler_init_temp_ext( t: float, delta: float, exponent: float -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 @@ -3881,8 +3820,7 @@ def llama_sampler_init_temp_ext( ) def llama_sampler_init_xtc( p: float, t: float, min_keep: int, seed: int, / -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 @@ -3892,8 +3830,7 @@ def llama_sampler_init_xtc( [ctypes.c_float], llama_sampler_p_ctypes, ) -def llama_sampler_init_top_n_sigma(n: float, /) -> llama_sampler_p: - ... +def llama_sampler_init_top_n_sigma(n: float, /) -> llama_sampler_p: ... # /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3910,8 +3847,7 @@ def llama_sampler_init_top_n_sigma(n: float, /) -> llama_sampler_p: ) def llama_sampler_init_mirostat( n_vocab: int, seed: int, tau: float, eta: float, m: int, / -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. @@ -3926,8 +3862,7 @@ def llama_sampler_init_mirostat( ) def llama_sampler_init_mirostat_v2( seed: int, tau: float, eta: float, / -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # /// @details Intializes a GBNF grammar, see grammars/README.md for details. @@ -3942,8 +3877,7 @@ def llama_sampler_init_mirostat_v2( ) def llama_sampler_init_grammar( vocab: llama_vocab_p, grammar_str: bytes, grammar_root: bytes, / -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy( @@ -3977,8 +3911,7 @@ def llama_sampler_init_grammar_lazy( trigger_tokens: CtypesArray[llama_token], num_trigger_tokens: int, /, -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 @@ -4012,8 +3945,7 @@ def llama_sampler_init_grammar_lazy_patterns( trigger_tokens: CtypesArray[llama_token], num_trigger_tokens: int, /, -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. @@ -4033,8 +3965,7 @@ def llama_sampler_init_penalties( penalty_freq: float, penalty_present: float, /, -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 @@ -4071,8 +4002,7 @@ def llama_sampler_init_dry( seq_breakers, num_breakers: int, /, -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( @@ -4086,8 +4016,7 @@ def llama_sampler_init_dry( ) def llama_sampler_init_logit_bias( n_vocab: int, n_logit_bias: int, logit_bias: CtypesArray[llama_logit_bias], / -) -> llama_sampler_p: - ... +) -> llama_sampler_p: ... # // this sampler is meant to be used for fill-in-the-middle infilling @@ -4097,8 +4026,7 @@ def llama_sampler_init_logit_bias( [llama_vocab_p_ctypes], llama_sampler_p_ctypes, ) -def llama_sampler_init_infill(vocab: llama_vocab_p, /) -> llama_sampler_p: - ... +def llama_sampler_init_infill(vocab: llama_vocab_p, /) -> llama_sampler_p: ... # // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise @@ -4108,8 +4036,7 @@ def llama_sampler_init_infill(vocab: llama_vocab_p, /) -> llama_sampler_p: [llama_sampler_p_ctypes], ctypes.c_uint32, ) -def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: - ... +def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ... # /// @details Sample and accept a token from the idx-th output of the last evaluation @@ -4121,14 +4048,14 @@ def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ) def llama_sampler_sample( smpl: llama_sampler_p, ctx: llama_context_p, idx: int, / -) -> int: - ... +) -> int: ... # // # // Model split # // + # /// @details Build a split GGUF final path for this chunk. # LLAMA_API int llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int split_no, int split_count); @ctypes_function( @@ -4170,8 +4097,7 @@ def llama_split_prefix( # // Print system information # LLAMA_API const char * llama_print_system_info(void); @ctypes_function("llama_print_system_info", [], ctypes.c_char_p) -def llama_print_system_info() -> bytes: - ... +def llama_print_system_info() -> bytes: ... # // Set callback for all future logging events. @@ -4203,6 +4129,7 @@ def llama_log_set( # double t_p_eval_ms; # double t_eval_ms; + # int32_t n_p_eval; # int32_t n_eval; # int32_t n_reused; // number of times a ggml compute graph had been reused @@ -4222,6 +4149,7 @@ class llama_perf_context_data(ctypes.Structure): # struct llama_perf_sampler_data { # double t_sample_ms; + # int32_t n_sample; # }; class llama_perf_sampler_data(ctypes.Structure): @@ -4237,8 +4165,7 @@ class llama_perf_sampler_data(ctypes.Structure): [llama_context_p_ctypes], llama_perf_context_data, ) -def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: - ... +def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: ... # LLAMA_API void llama_perf_context_print(const struct llama_context * ctx); @@ -4247,8 +4174,7 @@ def llama_perf_context(ctx: llama_context_p, /) -> llama_perf_context_data: [llama_context_p_ctypes], None, ) -def llama_perf_context_print(ctx: llama_context_p, /): - ... +def llama_perf_context_print(ctx: llama_context_p, /): ... # LLAMA_API void llama_perf_context_reset( struct llama_context * ctx); @@ -4257,8 +4183,7 @@ def llama_perf_context_print(ctx: llama_context_p, /): [llama_context_p_ctypes], None, ) -def llama_perf_context_reset(ctx: llama_context_p, /): - ... +def llama_perf_context_reset(ctx: llama_context_p, /): ... # // NOTE: the following work only with samplers constructed via llama_sampler_chain_init @@ -4268,8 +4193,7 @@ def llama_perf_context_reset(ctx: llama_context_p, /): [llama_sampler_p_ctypes], llama_perf_sampler_data, ) -def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: - ... +def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: ... # LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); @@ -4278,8 +4202,7 @@ def llama_perf_sampler(chain: llama_sampler_p, /) -> llama_perf_sampler_data: [llama_sampler_p_ctypes], None, ) -def llama_perf_sampler_print(chain: llama_sampler_p, /): - ... +def llama_perf_sampler_print(chain: llama_sampler_p, /): ... # LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); @@ -4288,8 +4211,7 @@ def llama_perf_sampler_print(chain: llama_sampler_p, /): [llama_sampler_p_ctypes], None, ) -def llama_perf_sampler_reset(chain: llama_sampler_p, /): - ... +def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... # // @@ -4298,7 +4220,10 @@ def llama_perf_sampler_reset(chain: llama_sampler_p, /): # // function that returns whether or not a given tensor contains trainable parameters # typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata); -llama_opt_param_filter = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p) +llama_opt_param_filter = ctypes.CFUNCTYPE( + ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p +) + # // always returns true # LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata); @@ -4307,8 +4232,9 @@ def llama_perf_sampler_reset(chain: llama_sampler_p, /): [ctypes.c_void_p, ctypes.c_void_p], ctypes.c_bool, ) -def llama_opt_param_filter_all(tensor: ctypes.c_void_p, userdata: ctypes.c_void_p, /) -> bool: - ... +def llama_opt_param_filter_all( + tensor: ctypes.c_void_p, userdata: ctypes.c_void_p, / +) -> bool: ... # struct llama_opt_params { @@ -4317,6 +4243,7 @@ def llama_opt_param_filter_all(tensor: ctypes.c_void_p, userdata: ctypes.c_void_ # llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters # void * param_filter_ud; // userdata for determining which tensors contain trainable parameters + # ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters # void * get_opt_pars_ud; // userdata for calculating optimizer parameters # }; @@ -4325,7 +4252,10 @@ class llama_opt_params(ctypes.Structure): ("n_ctx_train", ctypes.c_uint32), ("param_filter", llama_opt_param_filter), ("param_filter_ud", ctypes.c_void_p), - ("get_opt_pars", ctypes.c_void_p), # ggml_opt_get_optimizer_params - not implemented here + ( + "get_opt_pars", + ctypes.c_void_p, + ), # ggml_opt_get_optimizer_params - not implemented here ("get_opt_pars_ud", ctypes.c_void_p), ] @@ -4336,8 +4266,9 @@ class llama_opt_params(ctypes.Structure): [llama_context_p_ctypes, llama_model_p_ctypes, llama_opt_params], None, ) -def llama_opt_init(lctx: llama_context_p, model: llama_model_p, lopt_params: llama_opt_params, /): - ... +def llama_opt_init( + lctx: llama_context_p, model: llama_model_p, lopt_params: llama_opt_params, / +): ... # LLAMA_API void llama_opt_epoch( @@ -4353,7 +4284,7 @@ def llama_opt_init(lctx: llama_context_p, model: llama_model_p, lopt_params: lla [ llama_context_p_ctypes, ctypes.c_void_p, # ggml_opt_dataset_t - ctypes.c_void_p, # ggml_opt_result_t + ctypes.c_void_p, # ggml_opt_result_t ctypes.c_void_p, # ggml_opt_result_t ctypes.c_int64, ctypes.c_void_p, # ggml_opt_epoch_callback @@ -4370,5 +4301,4 @@ def llama_opt_epoch( callback_train: ctypes.c_void_p, callback_eval: ctypes.c_void_p, /, -): - ... +): ... diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index b95c77ab5e..ba34dda831 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -297,7 +297,7 @@ def opt_repetitions(up_to_n, prefix_with_sep=False): if max_items is not None: result += opt_repetitions(max_items - min_items, prefix_with_sep=min_items > 0) else: - item_operator = f'({separator_rule + " " if separator_rule else ""}{item_rule})' + item_operator = f"({separator_rule + ' ' if separator_rule else ''}{item_rule})" if min_items == 0 and separator_rule: result = f"({item_rule} {item_operator}*)?" @@ -450,9 +450,9 @@ def visit(n: dict): ref = n.get("$ref") if ref is not None and ref not in self._refs: if ref.startswith("https://"): - assert ( - self._allow_fetch - ), "Fetching remote schemas is not allowed (use --allow-fetch for force)" + assert self._allow_fetch, ( + "Fetching remote schemas is not allowed (use --allow-fetch for force)" + ) import requests frag_split = ref.split("#") @@ -475,9 +475,9 @@ def visit(n: dict): raise ValueError(f"Unsupported ref {ref}") for sel in ref.split("#")[-1].split("/")[1:]: - assert ( - target is not None and sel in target - ), f"Error resolving ref {ref}: {sel} not in {target}" + assert target is not None and sel in target, ( + f"Error resolving ref {ref}: {sel} not in {target}" + ) target = target[sel] self._refs[ref] = target @@ -492,7 +492,7 @@ def visit(n: dict): def _generate_union_rule(self, name, alt_schemas): return " | ".join( ( - self.visit(alt_schema, f'{name}{"-" if name else "alternative-"}{i}') + self.visit(alt_schema, f"{name}{'-' if name else 'alternative-'}{i}") for i, alt_schema in enumerate(alt_schemas) ) ) @@ -510,9 +510,9 @@ def _visit_pattern(self, pattern, name): we define sub-rules to keep the output lean. """ - assert pattern.startswith("^") and pattern.endswith( - "$" - ), 'Pattern must start with "^" and end with "$"' + assert pattern.startswith("^") and pattern.endswith("$"), ( + 'Pattern must start with "^" and end with "$"' + ) pattern = pattern[1:-1] sub_rule_ids = {} @@ -566,15 +566,15 @@ def join_seq(): elif c == "(": i += 1 if i < length: - assert ( - pattern[i] != "?" - ), f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' + assert pattern[i] != "?", ( + f'Unsupported pattern syntax "{pattern[i]}" at index {i} of /{pattern}/' + ) seq.append((f"({to_rule(transform())})", False)) elif c == ")": i += 1 - assert ( - start > 0 and pattern[start - 1] == "(" - ), f"Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}" + assert start > 0 and pattern[start - 1] == "(", ( + f"Unbalanced parentheses; start = {start}, i = {i}, pattern = {pattern}" + ) return join_seq() elif c == "[": square_brackets = c @@ -586,9 +586,9 @@ def join_seq(): else: square_brackets += pattern[i] i += 1 - assert ( - i < length - ), f"Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}" + assert i < length, ( + f"Unbalanced square brackets; start = {start}, i = {i}, pattern = {pattern}" + ) square_brackets += "]" i += 1 seq.append((square_brackets, False)) @@ -604,9 +604,9 @@ def join_seq(): while i < length and pattern[i] != "}": curly_brackets += pattern[i] i += 1 - assert ( - i < length - ), f"Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}" + assert i < length, ( + f"Unbalanced curly brackets; start = {start}, i = {i}, pattern = {pattern}" + ) curly_brackets += "}" i += 1 nums = [s.strip() for s in curly_brackets[1:-1].split(",")] @@ -777,13 +777,13 @@ def add_component(comp_schema, is_required): rule_name, '"[" space ' + ' "," space '.join( - self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') + self.visit(item, f"{name}{'-' if name else ''}tuple-{i}") for i, item in enumerate(items) ) + ' "]" space', ) else: - item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') + item_rule_name = self.visit(items, f"{name}{'-' if name else ''}item") min_items = schema.get("minItems", 0) max_items = schema.get("maxItems") return self._add_rule( @@ -873,17 +873,17 @@ def _build_object_rule( prop_kv_rule_names = {} for prop_name, prop_schema in properties: prop_rule_name = self.visit( - prop_schema, f'{name}{"-" if name else ""}{prop_name}' + prop_schema, f"{name}{'-' if name else ''}{prop_name}" ) prop_kv_rule_names[prop_name] = self._add_rule( - f'{name}{"-" if name else ""}{prop_name}-kv', + f"{name}{'-' if name else ''}{prop_name}-kv", rf'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}', ) required_props = [k for k in sorted_props if k in required] optional_props = [k for k in sorted_props if k not in required] if additional_properties == True or isinstance(additional_properties, dict): - sub_name = f'{name}{"-" if name else ""}additional' + sub_name = f"{name}{'-' if name else ''}additional" value_rule = self.visit( {} if additional_properties == True else additional_properties, f"{sub_name}-value", @@ -908,7 +908,7 @@ def get_recursive_refs(ks, first_is_optional): kv_rule_name = prop_kv_rule_names[k] if k == "*": res = self._add_rule( - f'{name}{"-" if name else ""}additional-kvs', + f"{name}{'-' if name else ''}additional-kvs", f'{kv_rule_name} ( "," space ' + kv_rule_name + " )*", ) elif first_is_optional: @@ -917,7 +917,7 @@ def get_recursive_refs(ks, first_is_optional): res = kv_rule_name if len(rest) > 0: res += " " + self._add_rule( - f'{name}{"-" if name else ""}{k}-rest', + f"{name}{'-' if name else ''}{k}-rest", get_recursive_refs(rest, first_is_optional=True), ) return res diff --git a/llama_cpp/llava_cpp.py b/llama_cpp/llava_cpp.py index d9dfaf5fd0..3dc96d2f25 100644 --- a/llama_cpp/llava_cpp.py +++ b/llama_cpp/llava_cpp.py @@ -36,7 +36,11 @@ # Specify the base name of the shared library to load _libllava_base_name = "llava" _libllava_override_path = os.environ.get("LLAVA_CPP_LIB") -_libllava_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _libllava_override_path is None else pathlib.Path() +_libllava_base_path = ( + pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" + if _libllava_override_path is None + else pathlib.Path() +) # Load the library _libllava = load_shared_library(_libllava_base_name, _libllava_base_path) @@ -73,8 +77,7 @@ class llava_image_embed(Structure): ) def llava_validate_embed_size( ctx_llama: llama_cpp.llama_context_p, ctx_clip: clip_ctx_p, / -) -> bool: - ... +) -> bool: ... # /** build an image embed from image file bytes */ @@ -90,8 +93,7 @@ def llava_image_embed_make_with_bytes( image_bytes: CtypesArray[c_uint8], image_bytes_length: Union[c_int, int], /, -) -> "_Pointer[llava_image_embed]": - ... +) -> "_Pointer[llava_image_embed]": ... # /** build an image embed from a path to an image filename */ @@ -103,15 +105,13 @@ def llava_image_embed_make_with_bytes( ) def llava_image_embed_make_with_filename( ctx_clip: clip_ctx_p, n_threads: Union[c_int, int], image_path: bytes, / -) -> "_Pointer[llava_image_embed]": - ... +) -> "_Pointer[llava_image_embed]": ... # LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed); # /** free an embedding made with llava_image_embed_make_* */ @ctypes_function("llava_image_embed_free", [POINTER(llava_image_embed)], None) -def llava_image_embed_free(embed: "_Pointer[llava_image_embed]", /): - ... +def llava_image_embed_free(embed: "_Pointer[llava_image_embed]", /): ... # /** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */ @@ -132,8 +132,7 @@ def llava_eval_image_embed( n_batch: Union[c_int, int], n_past: "_Pointer[c_int]", /, -) -> bool: - ... +) -> bool: ... ################################################ @@ -146,13 +145,10 @@ def llava_eval_image_embed( @ctypes_function("clip_model_load", [c_char_p, c_int], clip_ctx_p_ctypes) def clip_model_load( fname: bytes, verbosity: Union[c_int, int], / -) -> Optional[clip_ctx_p]: - ... +) -> Optional[clip_ctx_p]: ... # /** free mmproj model */ # CLIP_API void clip_free(struct clip_ctx * ctx); @ctypes_function("clip_free", [clip_ctx_p_ctypes], None) -def clip_free(ctx: clip_ctx_p, /): - ... - +def clip_free(ctx: clip_ctx_p, /): ... diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index a45f8f4064..41753a7f60 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -39,7 +39,11 @@ # Specify the base name of the shared library to load _libmtmd_base_name = "mtmd" _libmtmd_override_path = os.environ.get("MTMD_CPP_LIB") -_libmtmd_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _libmtmd_override_path is None else pathlib.Path() +_libmtmd_base_path = ( + pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" + if _libmtmd_override_path is None + else pathlib.Path() +) # Load the library _libmtmd = load_shared_library(_libmtmd_base_name, _libmtmd_base_path) @@ -71,6 +75,7 @@ MTMD_INPUT_CHUNK_TYPE_IMAGE = 1 MTMD_INPUT_CHUNK_TYPE_AUDIO = 2 + # Structures class mtmd_context_params(Structure): _fields_ = [ @@ -82,6 +87,7 @@ class mtmd_context_params(Structure): ("media_marker", c_char_p), ] + class mtmd_input_text(Structure): _fields_ = [ ("text", c_char_p), @@ -89,19 +95,21 @@ class mtmd_input_text(Structure): ("parse_special", c_bool), ] + ################################################ # mtmd.h functions ################################################ + # MTMD_API const char * mtmd_default_marker(void); @ctypes_function("mtmd_default_marker", [], c_char_p) -def mtmd_default_marker() -> bytes: - ... +def mtmd_default_marker() -> bytes: ... + # MTMD_API struct mtmd_context_params mtmd_context_params_default(void); @ctypes_function("mtmd_context_params_default", [], mtmd_context_params) -def mtmd_context_params_default() -> mtmd_context_params: - ... +def mtmd_context_params_default() -> mtmd_context_params: ... + # MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname, # const struct llama_model * text_model, @@ -109,70 +117,68 @@ def mtmd_context_params_default() -> mtmd_context_params: @ctypes_function( "mtmd_init_from_file", [c_char_p, llama_cpp.llama_model_p_ctypes, mtmd_context_params], - mtmd_context_p_ctypes + mtmd_context_p_ctypes, ) def mtmd_init_from_file( mmproj_fname: bytes, text_model: llama_cpp.llama_model_p, ctx_params: mtmd_context_params, /, -) -> Optional[mtmd_context_p]: - ... +) -> Optional[mtmd_context_p]: ... + # MTMD_API void mtmd_free(mtmd_context * ctx); @ctypes_function("mtmd_free", [mtmd_context_p_ctypes], None) -def mtmd_free(ctx: mtmd_context_p, /): - ... +def mtmd_free(ctx: mtmd_context_p, /): ... + # MTMD_API bool mtmd_support_vision(mtmd_context * ctx); @ctypes_function("mtmd_support_vision", [mtmd_context_p_ctypes], c_bool) -def mtmd_support_vision(ctx: mtmd_context_p, /) -> bool: - ... +def mtmd_support_vision(ctx: mtmd_context_p, /) -> bool: ... + # MTMD_API mtmd_bitmap * mtmd_bitmap_init(uint32_t nx, uint32_t ny, const unsigned char * data); @ctypes_function( - "mtmd_bitmap_init", - [c_uint32, c_uint32, POINTER(c_uint8)], - mtmd_bitmap_p_ctypes + "mtmd_bitmap_init", [c_uint32, c_uint32, POINTER(c_uint8)], mtmd_bitmap_p_ctypes ) def mtmd_bitmap_init( nx: Union[c_uint32, int], ny: Union[c_uint32, int], data: CtypesArray[c_uint8], /, -) -> Optional[mtmd_bitmap_p]: - ... +) -> Optional[mtmd_bitmap_p]: ... + # MTMD_API void mtmd_bitmap_free(mtmd_bitmap * bitmap); @ctypes_function("mtmd_bitmap_free", [mtmd_bitmap_p_ctypes], None) -def mtmd_bitmap_free(bitmap: mtmd_bitmap_p, /): - ... +def mtmd_bitmap_free(bitmap: mtmd_bitmap_p, /): ... + # MTMD_API mtmd_input_chunks * mtmd_input_chunks_init(void); @ctypes_function("mtmd_input_chunks_init", [], mtmd_input_chunks_p_ctypes) -def mtmd_input_chunks_init() -> Optional[mtmd_input_chunks_p]: - ... +def mtmd_input_chunks_init() -> Optional[mtmd_input_chunks_p]: ... + # MTMD_API void mtmd_input_chunks_free(mtmd_input_chunks * chunks); @ctypes_function("mtmd_input_chunks_free", [mtmd_input_chunks_p_ctypes], None) -def mtmd_input_chunks_free(chunks: mtmd_input_chunks_p, /): - ... +def mtmd_input_chunks_free(chunks: mtmd_input_chunks_p, /): ... + # MTMD_API size_t mtmd_input_chunks_size(const mtmd_input_chunks * chunks); @ctypes_function("mtmd_input_chunks_size", [mtmd_input_chunks_p_ctypes], c_size_t) -def mtmd_input_chunks_size(chunks: mtmd_input_chunks_p, /) -> int: - ... +def mtmd_input_chunks_size(chunks: mtmd_input_chunks_p, /) -> int: ... + # MTMD_API const mtmd_input_chunk * mtmd_input_chunks_get(const mtmd_input_chunks * chunks, size_t idx); @ctypes_function( "mtmd_input_chunks_get", [mtmd_input_chunks_p_ctypes, c_size_t], - mtmd_input_chunk_p_ctypes + mtmd_input_chunk_p_ctypes, ) def mtmd_input_chunks_get( chunks: mtmd_input_chunks_p, idx: Union[c_size_t, int], / -) -> Optional[mtmd_input_chunk_p]: - ... +) -> Optional[mtmd_input_chunk_p]: ... + # MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx, # mtmd_input_chunks * output, @@ -197,52 +203,53 @@ def mtmd_tokenize( bitmaps: CtypesArray[mtmd_bitmap_p_ctypes], n_bitmaps: Union[c_size_t, int], /, -) -> int: - ... +) -> int: ... + # MTMD_API size_t mtmd_input_chunk_get_n_tokens(const mtmd_input_chunk * chunk); @ctypes_function("mtmd_input_chunk_get_n_tokens", [mtmd_input_chunk_p_ctypes], c_size_t) -def mtmd_input_chunk_get_n_tokens(chunk: mtmd_input_chunk_p, /) -> int: - ... +def mtmd_input_chunk_get_n_tokens(chunk: mtmd_input_chunk_p, /) -> int: ... + # MTMD_API enum mtmd_input_chunk_type mtmd_input_chunk_get_type(const mtmd_input_chunk * chunk); @ctypes_function("mtmd_input_chunk_get_type", [mtmd_input_chunk_p_ctypes], c_int) -def mtmd_input_chunk_get_type(chunk: mtmd_input_chunk_p, /) -> int: - ... +def mtmd_input_chunk_get_type(chunk: mtmd_input_chunk_p, /) -> int: ... + # MTMD_API const llama_token * mtmd_input_chunk_get_tokens_text(const mtmd_input_chunk * chunk, size_t * n_tokens_output); @ctypes_function( "mtmd_input_chunk_get_tokens_text", [mtmd_input_chunk_p_ctypes, POINTER(c_size_t)], - POINTER(llama_cpp.llama_token) + POINTER(llama_cpp.llama_token), ) def mtmd_input_chunk_get_tokens_text( chunk: mtmd_input_chunk_p, n_tokens_output: "_Pointer[c_size_t]", / -) -> Optional["_Pointer[llama_cpp.llama_token]"]: - ... +) -> Optional["_Pointer[llama_cpp.llama_token]"]: ... + ################################################ # mtmd-helper.h functions ################################################ + # MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len); @ctypes_function( "mtmd_helper_bitmap_init_from_buf", [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t], - mtmd_bitmap_p_ctypes + mtmd_bitmap_p_ctypes, ) def mtmd_helper_bitmap_init_from_buf( ctx: mtmd_context_p, buf: CtypesArray[c_uint8], length: Union[c_size_t, int], /, -) -> Optional[mtmd_bitmap_p]: - ... +) -> Optional[mtmd_bitmap_p]: ... + # MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); @ctypes_function("mtmd_helper_get_n_tokens", [mtmd_input_chunks_p_ctypes], c_size_t) -def mtmd_helper_get_n_tokens(chunks: mtmd_input_chunks_p, /) -> int: - ... +def mtmd_helper_get_n_tokens(chunks: mtmd_input_chunks_p, /) -> int: ... + # MTMD_API int32_t mtmd_helper_eval_chunk_single(mtmd_context * ctx, # struct llama_context * lctx, @@ -276,5 +283,4 @@ def mtmd_helper_eval_chunk_single( logits_last: Union[c_bool, bool], new_n_past: "_Pointer[llama_cpp.llama_pos]", /, -) -> int: - ... +) -> int: ... diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py index 5120f24162..f776fe159c 100644 --- a/llama_cpp/server/app.py +++ b/llama_cpp/server/app.py @@ -125,9 +125,9 @@ def create_app( server_settings = ServerSettings.model_validate(settings) model_settings = [ModelSettings.model_validate(settings)] - assert ( - server_settings is not None and model_settings is not None - ), "server_settings and model_settings must be provided together" + assert server_settings is not None and model_settings is not None, ( + "server_settings and model_settings must be provided together" + ) set_server_settings(server_settings) middleware = [Middleware(RawContextMiddleware, plugins=(RequestIdPlugin(),))] diff --git a/llama_cpp/server/cli.py b/llama_cpp/server/cli.py index 3dd0076767..8ed0290637 100644 --- a/llama_cpp/server/cli.py +++ b/llama_cpp/server/cli.py @@ -14,7 +14,9 @@ def _get_base_type(annotation: Type[Any]) -> Type[Any]: elif getattr(annotation, "__origin__", None) is Union: assert hasattr(annotation, "__args__") and len(annotation.__args__) >= 1 # type: ignore non_optional_args: List[Type[Any]] = [ - arg for arg in annotation.__args__ if arg is not type(None) # type: ignore + arg + for arg in annotation.__args__ + if arg is not type(None) # type: ignore ] if non_optional_args: return _get_base_type(non_optional_args[0]) diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 11bd363b56..9e59e85633 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -186,18 +186,18 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: clip_model_path=settings.clip_model_path, verbose=settings.verbose ) elif settings.chat_format == "hf-autotokenizer": - assert ( - settings.hf_pretrained_model_name_or_path is not None - ), "hf_pretrained_model_name_or_path must be set for hf-autotokenizer" + assert settings.hf_pretrained_model_name_or_path is not None, ( + "hf_pretrained_model_name_or_path must be set for hf-autotokenizer" + ) chat_handler = ( llama_cpp.llama_chat_format.hf_autotokenizer_to_chat_completion_handler( settings.hf_pretrained_model_name_or_path ) ) elif settings.chat_format == "hf-tokenizer-config": - assert ( - settings.hf_tokenizer_config_path is not None - ), "hf_tokenizer_config_path must be set for hf-tokenizer-config" + assert settings.hf_tokenizer_config_path is not None, ( + "hf_tokenizer_config_path must be set for hf-tokenizer-config" + ) chat_handler = llama_cpp.llama_chat_format.hf_tokenizer_config_to_chat_completion_handler( json.load(open(settings.hf_tokenizer_config_path)) ) diff --git a/pyproject.toml b/pyproject.toml index f5ae7b59c7..e0b0dc520c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ test = [ "huggingface-hub>=0.23.0" ] dev = [ - "black>=23.3.0", + "ruff>=0.15.7", "twine>=4.0.2", "mkdocs>=1.4.3", "mkdocstrings[python]>=0.22.0", @@ -78,5 +78,16 @@ Issues = "https://github.com/abetlen/llama-cpp-python/issues" Documentation = "https://llama-cpp-python.readthedocs.io/en/latest/" Changelog = "https://llama-cpp-python.readthedocs.io/en/latest/changelog/" +[tool.ruff] +target-version = "py38" +line-length = 88 +required-version = ">=0.15.7" +src = ["llama_cpp", "tests"] +extend-exclude = ["vendor", "examples/notebooks"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9"] +ignore = ["E712"] + [tool.pytest.ini_options] testpaths = "tests" diff --git a/tests/test_llama.py b/tests/test_llama.py index 0a1a9f5ad3..964b0895cc 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -66,6 +66,7 @@ def llama_cpp_model_path(): def test_real_model(llama_cpp_model_path): import os + assert os.path.exists(llama_cpp_model_path) params = llama_cpp.llama_model_default_params() @@ -114,6 +115,7 @@ def test_real_model(llama_cpp_model_path): output_text = model.detokenize(output, special=True) assert output_text == b" over the lazy dog" + def test_real_llama(llama_cpp_model_path): model = llama_cpp.Llama( llama_cpp_model_path, @@ -132,11 +134,10 @@ def test_real_llama(llama_cpp_model_path): top_k=50, top_p=0.9, temperature=0.8, - seed=1337 + seed=1337, ) assert output["choices"][0]["text"] == " over the lazy dog" - output = model.create_completion( "The capital of france is paris, 'true' or 'false'?:\n", max_tokens=4, @@ -146,20 +147,19 @@ def test_real_llama(llama_cpp_model_path): seed=1337, grammar=llama_cpp.LlamaGrammar.from_string(""" root ::= "true" | "false" -""") +"""), ) assert output["choices"][0]["text"] == "true" suffix = b"rot" tokens = model.tokenize(suffix, add_bos=True, special=True) + def logit_processor_func(input_ids, logits): for token in tokens: logits[token] *= 1000 return logits - logit_processors = llama_cpp.LogitsProcessorList( - [logit_processor_func] - ) + logit_processors = llama_cpp.LogitsProcessorList([logit_processor_func]) output = model.create_completion( "The capital of france is par", @@ -168,7 +168,7 @@ def logit_processor_func(input_ids, logits): top_p=0.9, temperature=0.8, seed=1337, - logits_processor=logit_processors + logits_processor=logit_processors, ) assert output["choices"][0]["text"].lower().startswith("rot") @@ -184,7 +184,7 @@ def logit_processor_func(input_ids, logits): temperature=0.8, grammar=llama_cpp.LlamaGrammar.from_string(""" root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" -""") +"""), ) number_1 = output["choices"][0]["text"] @@ -196,7 +196,7 @@ def logit_processor_func(input_ids, logits): temperature=0.8, grammar=llama_cpp.LlamaGrammar.from_string(""" root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" -""") +"""), ) number_2 = output["choices"][0]["text"] @@ -210,7 +210,7 @@ def logit_processor_func(input_ids, logits): temperature=0.8, grammar=llama_cpp.LlamaGrammar.from_string(""" root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" -""") +"""), ) number_3 = output["choices"][0]["text"] @@ -228,7 +228,7 @@ def test_real_llama_embeddings(llama_cpp_model_path): n_threads_batch=multiprocessing.cpu_count(), logits_all=False, flash_attn=True, - embedding=True + embedding=True, ) # Smoke test for now model.embed("Hello World") diff --git a/tests/test_llama_chat_format.py b/tests/test_llama_chat_format.py index f031bf72b7..18c7279cf0 100644 --- a/tests/test_llama_chat_format.py +++ b/tests/test_llama_chat_format.py @@ -10,13 +10,20 @@ from llama_cpp.llama_chat_format import hf_tokenizer_config_to_chat_formatter + def test_mistral_instruct(): chat_template = "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}" chat_formatter = jinja2.Template(chat_template) messages = [ - llama_types.ChatCompletionRequestUserMessage(role="user", content="Instruction"), - llama_types.ChatCompletionRequestAssistantMessage(role="assistant", content="Model answer"), - llama_types.ChatCompletionRequestUserMessage(role="user", content="Follow-up instruction"), + llama_types.ChatCompletionRequestUserMessage( + role="user", content="Instruction" + ), + llama_types.ChatCompletionRequestAssistantMessage( + role="assistant", content="Model answer" + ), + llama_types.ChatCompletionRequestUserMessage( + role="user", content="Follow-up instruction" + ), ] response = llama_chat_format.format_mistral_instruct( messages=messages, @@ -77,13 +84,11 @@ def test_mistral_instruct(): def test_hf_tokenizer_config_str_to_chat_formatter(): tokenizer_config = json.loads(mistral_7b_tokenizer_config) - chat_formatter = hf_tokenizer_config_to_chat_formatter( - tokenizer_config - ) + chat_formatter = hf_tokenizer_config_to_chat_formatter(tokenizer_config) chat_formatter_respoonse = chat_formatter( messages=[ ChatCompletionRequestUserMessage(role="user", content="Hello, world!"), ] ) - assert chat_formatter_respoonse.prompt == ("[INST] Hello, world! [/INST]" "") + assert chat_formatter_respoonse.prompt == ("[INST] Hello, world! [/INST]") diff --git a/tests/test_llama_speculative.py b/tests/test_llama_speculative.py index b5d450567b..d28c9ca9c6 100644 --- a/tests/test_llama_speculative.py +++ b/tests/test_llama_speculative.py @@ -2,15 +2,20 @@ from llama_cpp.llama_speculative import LlamaPromptLookupDecoding + def test_find_candidate_pred_tokens(): find_candidate_pred_tokens = LlamaPromptLookupDecoding.find_candidate_pred_tokens # Test Case 1: Matching ngram is found input_ids1 = np.array([1, 2, 3, 1, 2, 3, 1, 2, 3]) - result1 = find_candidate_pred_tokens(input_ids1, max_ngram_size=3, num_pred_tokens=2) + result1 = find_candidate_pred_tokens( + input_ids1, max_ngram_size=3, num_pred_tokens=2 + ) assert np.array_equal(result1, np.array([1, 2])) # Test Case 2: Matching ngram is not found input_ids2 = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9]) - result2 = find_candidate_pred_tokens(input_ids2, max_ngram_size=3, num_pred_tokens=2) + result2 = find_candidate_pred_tokens( + input_ids2, max_ngram_size=3, num_pred_tokens=2 + ) assert np.array_equal(result2, np.array([])) From 18aa31ef1e6150cf1b24b81583f5254aabe00199 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 22 Mar 2026 18:58:55 -0700 Subject: [PATCH 440/561] feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 (#2151) * Update llama.cpp and sync bindings * Clean up binding compatibility shims * Remove flash attention property shim * Remove mtmd verbosity shim * Add docstrings for new bindings * Format Ruff files and add changelog entry --- CHANGELOG.md | 1 + CMakeLists.txt | 20 + Makefile | 2 - llama_cpp/_internals.py | 8 +- llama_cpp/llama.py | 22 +- llama_cpp/llama_chat_format.py | 9 +- llama_cpp/llama_cpp.py | 881 +++++++++++++++++++++------------ llama_cpp/mtmd_cpp.py | 96 +++- tests/test_llama.py | 2 +- vendor/llama.cpp | 2 +- 10 files changed, 714 insertions(+), 329 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7044f44d65..e7506eed57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 and sync Python bindings by @abetlen in #2151 - chore(dev): Add Ruff-based formatting and a safe lint baseline, and run it in CI for pull requests and pushes to `main` - fix(ci): Run macOS CI on supported Apple Silicon and Intel runners by @abetlen in #2150 - fix(ci): Use the `hf` CLI instead of the deprecated `huggingface-cli` name in GitHub Actions and docs by @abetlen in #2149 diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b06d98b32..9b2744cdce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,6 +153,26 @@ if (LLAMA_BUILD) add_compile_definitions(GGML_USE_METAL) endif() + # Upstream mtmd expects LLAMA_INSTALL_VERSION to be set by llama.cpp's + # top-level CMakeLists.txt. When we include tools/mtmd directly from the + # Python package build, that directory scope is skipped. + if (NOT DEFINED LLAMA_INSTALL_VERSION OR "${LLAMA_INSTALL_VERSION}" STREQUAL "") + set(LLAMA_INSTALL_VERSION 0.0.0) + find_package(Git QUIET) + if (Git_FOUND) + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-list --count HEAD + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/vendor/llama.cpp + OUTPUT_VARIABLE LLAMA_MTMD_BUILD_NUMBER + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE LLAMA_MTMD_BUILD_NUMBER_RESULT + ) + if (LLAMA_MTMD_BUILD_NUMBER_RESULT EQUAL 0) + set(LLAMA_INSTALL_VERSION 0.0.${LLAMA_MTMD_BUILD_NUMBER}) + endif() + endif() + endif() + # Building llava add_subdirectory(vendor/llama.cpp/tools/mtmd) diff --git a/Makefile b/Makefile index 8e6cae2c1f..db45246c79 100644 --- a/Makefile +++ b/Makefile @@ -82,8 +82,6 @@ run-server: python3 -m llama_cpp.server --model ${MODEL} clean: - - cd vendor/llama.cpp && make clean - - cd vendor/llama.cpp && rm libllama.so - rm -rf _skbuild - rm llama_cpp/lib/*.so - rm llama_cpp/lib/*.dylib diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index b520b7ea5f..d6258d2244 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -2,6 +2,7 @@ import os import ctypes +import warnings from typing import ( Dict, @@ -699,8 +700,11 @@ def add_dist(self, seed: int): llama_cpp.llama_sampler_chain_add(self.sampler, sampler) def add_softmax(self): - sampler = llama_cpp.llama_sampler_init_softmax() - llama_cpp.llama_sampler_chain_add(self.sampler, sampler) + warnings.warn( + "add_softmax is deprecated; llama_sampler_init_dist now samples directly from logits", + DeprecationWarning, + stacklevel=2, + ) def add_top_k(self, k: int): sampler = llama_cpp.llama_sampler_init_top_k(k) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 21a7430a01..1609ad16bf 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -341,7 +341,11 @@ def __init__( self._logits_all = logits_all if draft_model is None else True self.context_params.embeddings = embedding # TODO: Rename to embeddings self.context_params.offload_kqv = offload_kqv - self.context_params.flash_attn = flash_attn + self.context_params.flash_attn_type = ( + llama_cpp.LLAMA_FLASH_ATTN_TYPE_ENABLED + if flash_attn + else llama_cpp.LLAMA_FLASH_ATTN_TYPE_DISABLED + ) if op_offload is not None: self.context_params.op_offload = op_offload @@ -431,9 +435,9 @@ def free_lora_adapter(): self._stack.callback(free_lora_adapter) - if llama_cpp.llama_set_adapter_lora( - self._ctx.ctx, self._lora_adapter, self.lora_scale - ): + adapters = (llama_cpp.llama_adapter_lora_p_ctypes * 1)(self._lora_adapter) + scales = (ctypes.c_float * 1)(self.lora_scale) + if llama_cpp.llama_set_adapters_lora(self._ctx.ctx, adapters, 1, scales): raise RuntimeError( f"Failed to set LoRA adapter from lora path: {self.lora_path}" ) @@ -726,7 +730,6 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): sampler.add_grammar(self._model, grammar) if temp < 0.0: - sampler.add_softmax() sampler.add_dist(self._seed) elif temp == 0.0: sampler.add_greedy() @@ -1042,7 +1045,7 @@ def embed( data: Union[List[List[float]], List[List[List[float]]]] = [] def decode_batch(seq_sizes: List[int]): - llama_cpp.llama_kv_self_clear(self._ctx.ctx) + self._ctx.kv_cache_clear() self._ctx.decode(self._batch) self._batch.reset() @@ -1113,7 +1116,7 @@ def decode_batch(seq_sizes: List[int]): output = data[0] if isinstance(input, str) else data - llama_cpp.llama_kv_self_clear(self._ctx.ctx) + self._ctx.kv_cache_clear() self.reset() if return_count: @@ -2100,7 +2103,10 @@ def __getstate__(self): logits_all=self._logits_all, embedding=self.context_params.embeddings, offload_kqv=self.context_params.offload_kqv, - flash_attn=self.context_params.flash_attn, + flash_attn=( + self.context_params.flash_attn_type + == llama_cpp.LLAMA_FLASH_ATTN_TYPE_ENABLED + ), op_offload=self.context_params.op_offload, swa_full=self.context_params.swa_full, # Sampling Params diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 8e8ac7bb30..d7910e9847 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2755,7 +2755,14 @@ def _init_mtmd_context(self, llama_model: llama.Llama): ctx_params.use_gpu = True # TODO: Make this configurable ctx_params.print_timings = self.verbose ctx_params.n_threads = llama_model.n_threads - ctx_params.verbosity = 2 if self.verbose else 0 # GGML_LOG_LEVEL_INFO = 2 + ctx_params.flash_attn_type = ( + llama_cpp.LLAMA_FLASH_ATTN_TYPE_ENABLED + if ( + llama_model.context_params.flash_attn_type + == llama_cpp.LLAMA_FLASH_ATTN_TYPE_ENABLED + ) + else llama_cpp.LLAMA_FLASH_ATTN_TYPE_DISABLED + ) # Initialize mtmd context self.mtmd_ctx = self._mtmd_cpp.mtmd_init_from_file( diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index f13af67f3b..e51492c56e 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -3,6 +3,7 @@ import os import ctypes import pathlib +import warnings from typing import ( Callable, @@ -77,6 +78,8 @@ # GGML_TYPE_I64 = 27, # GGML_TYPE_F64 = 28, # GGML_TYPE_IQ1_M = 29, +# GGML_TYPE_MXFP4 = 39, +# GGML_TYPE_NVFP4 = 40, # GGML_TYPE_COUNT, # }; GGML_TYPE_F32 = 0 @@ -107,7 +110,9 @@ GGML_TYPE_I64 = 27 GGML_TYPE_F64 = 28 GGML_TYPE_IQ1_M = 29 -GGML_TYPE_COUNT = 30 +GGML_TYPE_MXFP4 = 39 +GGML_TYPE_NVFP4 = 40 +GGML_TYPE_COUNT = 41 # from ggml-backend.h # typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); @@ -173,6 +178,10 @@ llama_kv_cache_p = NewType("llama_kv_cache_p", int) llama_kv_cache_p_ctypes = ctypes.c_void_p +# struct gguf_context; +gguf_context_p = NewType("gguf_context_p", int) +gguf_context_p_ctypes = ctypes.c_void_p + # typedef int32_t llama_pos; llama_pos = ctypes.c_int32 # typedef int32_t llama_token; @@ -292,12 +301,14 @@ # LLAMA_ROPE_TYPE_NORM = 0, # LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, # LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE, +# LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE, # LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION, # }; LLAMA_ROPE_TYPE_NONE = -1 LLAMA_ROPE_TYPE_NORM = 0 LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2 LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE = 8 +LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE = 40 LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION = 24 @@ -386,6 +397,7 @@ # LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors # LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors # LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -425,6 +437,7 @@ LLAMA_FTYPE_MOSTLY_TQ1_0 = 36 LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 +LLAMA_FTYPE_MOSTLY_NVFP4 = 39 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { @@ -467,6 +480,16 @@ LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1 +# enum llama_flash_attn_type { +# LLAMA_FLASH_ATTN_TYPE_AUTO = -1, +# LLAMA_FLASH_ATTN_TYPE_DISABLED = 0, +# LLAMA_FLASH_ATTN_TYPE_ENABLED = 1, +# }; +LLAMA_FLASH_ATTN_TYPE_AUTO = -1 +LLAMA_FLASH_ATTN_TYPE_DISABLED = 0 +LLAMA_FLASH_ATTN_TYPE_ENABLED = 1 + + # enum llama_split_mode { # LLAMA_SPLIT_MODE_NONE = 0, // single GPU # LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs @@ -619,6 +642,34 @@ class llama_batch(ctypes.Structure): LLAMA_KV_OVERRIDE_TYPE_STR = 3 +# enum llama_model_meta_key { +# LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE, +# LLAMA_MODEL_META_KEY_SAMPLING_TOP_K, +# LLAMA_MODEL_META_KEY_SAMPLING_TOP_P, +# LLAMA_MODEL_META_KEY_SAMPLING_MIN_P, +# LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY, +# LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD, +# LLAMA_MODEL_META_KEY_SAMPLING_TEMP, +# LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N, +# LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT, +# LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT, +# LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU, +# LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA, +# }; +LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE = 0 +LLAMA_MODEL_META_KEY_SAMPLING_TOP_K = 1 +LLAMA_MODEL_META_KEY_SAMPLING_TOP_P = 2 +LLAMA_MODEL_META_KEY_SAMPLING_MIN_P = 3 +LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY = 4 +LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD = 5 +LLAMA_MODEL_META_KEY_SAMPLING_TEMP = 6 +LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N = 7 +LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT = 8 +LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT = 9 +LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU = 10 +LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA = 11 + + # struct llama_model_kv_override { # enum llama_model_kv_override_type tag; @@ -695,11 +746,14 @@ class llama_model_kv_override(ctypes.Structure): # // Keep the booleans together to avoid misalignment during copy-by-value. -# bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_mlock; // force system to keep model in RAM -# bool check_tensors; // validate model tensor data +# bool vocab_only; // only load the vocabulary, no weights +# bool use_mmap; // use mmap if possible +# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported +# bool use_mlock; // force system to keep model in RAM +# bool check_tensors; // validate model tensor data # bool use_extra_bufts; // use extra buffer types (used for weight repacking) +# bool no_host; // bypass host buffer allowing extra buffers to be used +# bool no_alloc; // only load metadata and simulate memory allocations # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -716,9 +770,12 @@ class llama_model_params(ctypes.Structure): kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights use_mmap (bool): use mmap if possible + use_direct_io (bool): use direct io, takes precedence over use_mmap when supported use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data - use_extra_bufts (bool): use extra buffer types (used for weight repacking)""" + use_extra_bufts (bool): use extra buffer types (used for weight repacking) + no_host (bool): bypass host buffer allowing extra buffers to be used + no_alloc (bool): only load metadata and simulate memory allocations""" if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused @@ -734,9 +791,12 @@ class llama_model_params(ctypes.Structure): kv_overrides: CtypesArray[llama_model_kv_override] vocab_only: bool use_mmap: bool + use_direct_io: bool use_mlock: bool check_tensors: bool use_extra_bufts: bool + no_host: bool + no_alloc: bool _fields_ = [ ("devices", ctypes.c_void_p), # NOTE: unnused @@ -750,9 +810,27 @@ class llama_model_params(ctypes.Structure): ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("vocab_only", ctypes.c_bool), ("use_mmap", ctypes.c_bool), + ("use_direct_io", ctypes.c_bool), ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), ("use_extra_bufts", ctypes.c_bool), + ("no_host", ctypes.c_bool), + ("no_alloc", ctypes.c_bool), + ] + + +# struct llama_sampler_seq_config { +# llama_seq_id seq_id; +# struct llama_sampler * sampler; +# }; +class llama_sampler_seq_config(ctypes.Structure): + if TYPE_CHECKING: + seq_id: int + sampler: ctypes.c_void_p + + _fields_ = [ + ("seq_id", llama_seq_id), + ("sampler", ctypes.c_void_p), ] @@ -769,6 +847,7 @@ class llama_model_params(ctypes.Structure): # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` # enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id # enum llama_attention_type attention_type; // attention type to use for embeddings +# enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention # // ref: https://github.com/ggml-org/llama.cpp/pull/2054 # float rope_freq_base; // RoPE base frequency, 0 = from model @@ -796,7 +875,6 @@ class llama_model_params(ctypes.Structure): # // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value. # bool embeddings; // if true, extract embeddings (together with logits) # bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU -# bool flash_attn; // use flash attention [EXPERIMENTAL] # bool no_perf; // measure performance timings # bool op_offload; // offload host tensor operations to device # bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) @@ -805,6 +883,8 @@ class llama_model_params(ctypes.Structure): # bool kv_unified; // use a unified buffer across the input sequences when computing the attention # // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix # // ref: https://github.com/ggml-org/llama.cpp/pull/14363 +# struct llama_sampler_seq_config * samplers; +# size_t n_samplers; # }; class llama_context_params(ctypes.Structure): """Parameters for llama_context @@ -819,6 +899,7 @@ class llama_context_params(ctypes.Structure): rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type` pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) attention_type (int): attention type to use for embeddings + flash_attn_type (int): when to enable flash attention rope_freq_base (float): RoPE base frequency, 0 = from model rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model @@ -835,11 +916,12 @@ class llama_context_params(ctypes.Structure): abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU - flash_attn (bool): whether to use flash attention no_perf (bool): whether to measure performance timings op_offload (bool): offload host tensor operations to device swa_full (bool): use full-size SWA cache kv_unified (bool): use a unified buffer across the input sequences when computing the attention + samplers (ctypes.POINTER(llama_sampler_seq_config)): backend sampler chain configuration + n_samplers (int): number of backend sampler chain configurations """ if TYPE_CHECKING: @@ -852,6 +934,7 @@ class llama_context_params(ctypes.Structure): rope_scaling_type: int pooling_type: int attention_type: int + flash_attn_type: int rope_freq_base: float rope_freq_scale: float yarn_ext_factor: float @@ -868,11 +951,12 @@ class llama_context_params(ctypes.Structure): abort_callback_data: ctypes.c_void_p embeddings: bool offload_kqv: bool - flash_attn: bool no_perf: bool op_offload: bool swa_full: bool kv_unified: bool + samplers: ctypes.POINTER(llama_sampler_seq_config) + n_samplers: int _fields_ = [ ("n_ctx", ctypes.c_uint32), @@ -884,6 +968,7 @@ class llama_context_params(ctypes.Structure): ("rope_scaling_type", ctypes.c_int), ("pooling_type", ctypes.c_int), ("attention_type", ctypes.c_int), + ("flash_attn_type", ctypes.c_int), ("rope_freq_base", ctypes.c_float), ("rope_freq_scale", ctypes.c_float), ("yarn_ext_factor", ctypes.c_float), @@ -900,11 +985,12 @@ class llama_context_params(ctypes.Structure): ("abort_callback_data", ctypes.c_void_p), ("embeddings", ctypes.c_bool), ("offload_kqv", ctypes.c_bool), - ("flash_attn", ctypes.c_bool), ("no_perf", ctypes.c_bool), ("op_offload", ctypes.c_bool), ("swa_full", ctypes.c_bool), ("kv_unified", ctypes.c_bool), + ("samplers", ctypes.POINTER(llama_sampler_seq_config)), + ("n_samplers", ctypes.c_size_t), ] @@ -935,6 +1021,7 @@ class llama_context_params(ctypes.Structure): # bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored # bool pure; // quantize all tensors to the default type # bool keep_split; // quantize to the same number of shards +# bool dry_run; // calculate and show the final quantization size without performing quantization # void * imatrix; // pointer to importance matrix data # void * kv_overrides; // pointer to vector containing overrides # void * tensor_types; // pointer to vector containing tensor types @@ -953,6 +1040,7 @@ class llama_model_quantize_params(ctypes.Structure): only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored pure (bool): quantize all tensors to the default type keep_split (bool): quantize to the same number of shards + dry_run (bool): calculate and show the final quantization size without performing quantization imatrix (ctypes.c_void_p): pointer to importance matrix data kv_overrides (ctypes.c_void_p): pointer to vector containing overrides tensor_types (ctypes.c_void_p): pointer to vector containing tensor types @@ -969,6 +1057,7 @@ class llama_model_quantize_params(ctypes.Structure): only_copy: bool pure: bool keep_split: bool + dry_run: bool imatrix: ctypes.c_void_p kv_overrides: ctypes.c_void_p tensor_types: ctypes.c_void_p @@ -984,6 +1073,7 @@ class llama_model_quantize_params(ctypes.Structure): ("only_copy", ctypes.c_bool), ("pure", ctypes.c_bool), ("keep_split", ctypes.c_bool), + ("dry_run", ctypes.c_bool), ("imatrix", ctypes.c_void_p), ("kv_overrides", ctypes.c_void_p), ("tensor_types", ctypes.c_void_p), @@ -1095,6 +1185,13 @@ def llama_model_quantize_default_params() -> llama_model_quantize_params: ... +# LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type); +@ctypes_function("llama_flash_attn_type_name", [ctypes.c_int], ctypes.c_char_p) +def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: + """Get the flash attention type name.""" + ... + + # // Initialize the llama + ggml backend # // If numa is true, use NUMA optimizations # // Call once at the start of the program @@ -1249,6 +1346,36 @@ def llama_free_model(model: llama_model_p, /): ... def llama_model_free(model: llama_model_p, /): ... +# typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata); +llama_model_set_tensor_data_t = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p) + + +# LLAMA_API struct llama_model * llama_model_init_from_user( +# struct gguf_context * metadata, +# llama_model_set_tensor_data_t set_tensor_data, +# void * set_tensor_data_ud, +# struct llama_model_params params); +@ctypes_function( + "llama_model_init_from_user", + [ + gguf_context_p_ctypes, + llama_model_set_tensor_data_t, + ctypes.c_void_p, + llama_model_params, + ], + llama_model_p_ctypes, +) +def llama_model_init_from_user( + metadata: gguf_context_p, + set_tensor_data: llama_model_set_tensor_data_t, + set_tensor_data_ud: ctypes.c_void_p, + params: llama_model_params, + /, +) -> Optional[llama_model_p]: + """Initialize a model from user-provided metadata and tensor data.""" + ... + + # LLAMA_API struct llama_context * llama_init_from_model( # struct llama_model * model, # struct llama_context_params params); @@ -1288,6 +1415,54 @@ def llama_free(ctx: llama_context_p, /): ... +# enum llama_params_fit_status { +# LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0, +# LLAMA_PARAMS_FIT_STATUS_FAILURE = 1, +# LLAMA_PARAMS_FIT_STATUS_ERROR = 2, +# }; +LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0 +LLAMA_PARAMS_FIT_STATUS_FAILURE = 1 +LLAMA_PARAMS_FIT_STATUS_ERROR = 2 + + +# LLAMA_API enum llama_params_fit_status llama_params_fit( +# const char * path_model, +# struct llama_model_params * mparams, +# struct llama_context_params * cparams, +# float * tensor_split, +# struct llama_model_tensor_buft_override * tensor_buft_overrides, +# size_t * margins, +# uint32_t n_ctx_min, +# enum ggml_log_level log_level); +@ctypes_function( + "llama_params_fit", + [ + ctypes.c_char_p, + ctypes.POINTER(llama_model_params), + ctypes.POINTER(llama_context_params), + ctypes.POINTER(ctypes.c_float), + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_size_t), + ctypes.c_uint32, + ctypes.c_int, + ], + ctypes.c_int, +) +def llama_params_fit( + path_model: bytes, + mparams: CtypesPointerOrRef[llama_model_params], + cparams: CtypesPointerOrRef[llama_context_params], + tensor_split: Optional[CtypesPointer[ctypes.c_float]], + tensor_buft_overrides: ctypes.c_void_p, + margins: Optional[CtypesPointer[ctypes.c_size_t]], + n_ctx_min: int, + log_level: int, + /, +) -> int: + """Fit model and context parameters for a model path.""" + ... + + # LLAMA_API int64_t llama_time_us(void); @ctypes_function( "llama_time_us", @@ -1307,6 +1482,13 @@ def llama_max_devices() -> int: ... def llama_max_parallel_sequences() -> int: ... +# LLAMA_API size_t llama_max_tensor_buft_overrides(void); +@ctypes_function("llama_max_tensor_buft_overrides", [], ctypes.c_size_t) +def llama_max_tensor_buft_overrides() -> int: + """Get the maximum number of tensor buffer type overrides.""" + ... + + # LLAMA_API bool llama_supports_mmap (void); @ctypes_function("llama_supports_mmap", [], ctypes.c_bool) def llama_supports_mmap() -> bool: ... @@ -1332,6 +1514,13 @@ def llama_supports_rpc() -> bool: ... def llama_n_ctx(ctx: llama_context_p, /) -> int: ... +# LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx); +@ctypes_function("llama_n_ctx_seq", [llama_context_p_ctypes], ctypes.c_uint32) +def llama_n_ctx_seq(ctx: llama_context_p, /) -> int: + """Get the context size per sequence.""" + ... + + # LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); @ctypes_function("llama_n_batch", [llama_context_p_ctypes], ctypes.c_uint32) def llama_n_batch(ctx: llama_context_p, /) -> int: ... @@ -1389,17 +1578,6 @@ def llama_get_memory(ctx: llama_context_p, /) -> Optional[llama_memory_t]: def llama_pooling_type(ctx: llama_context_p, /) -> int: ... -# DEPRECATED(LLAMA_API struct llama_kv_cache * llama_get_kv_self(struct llama_context * ctx), "use llama_get_memory instead"); -@ctypes_function( - "llama_get_kv_self", - [llama_context_p_ctypes], - llama_kv_cache_p_ctypes, -) -def llama_get_kv_self(ctx: llama_context_p, /) -> Optional[llama_kv_cache_p]: - """Get the KV cache for self-attention (DEPRECATED)""" - ... - - # LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); @ctypes_function("llama_model_get_vocab", [llama_model_p_ctypes], llama_vocab_p_ctypes) def llama_model_get_vocab(model: llama_model_p, /) -> Optional[llama_vocab_p]: ... @@ -1420,6 +1598,20 @@ def llama_model_n_ctx_train(model: llama_model_p, /) -> int: ... def llama_model_n_embd(model: llama_model_p, /) -> int: ... +# LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model); +@ctypes_function("llama_model_n_embd_inp", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_embd_inp(model: llama_model_p, /) -> int: + """Get the model input embedding size.""" + ... + + +# LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model); +@ctypes_function("llama_model_n_embd_out", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_embd_out(model: llama_model_p, /) -> int: + """Get the model output embedding size.""" + ... + + # LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model); @ctypes_function("llama_model_n_layer", [llama_model_p_ctypes], ctypes.c_int32) def llama_model_n_layer(model: llama_model_p, /) -> int: ... @@ -1515,6 +1707,14 @@ def llama_model_meta_count(model: llama_model_p, /) -> int: ... +# // Get sampling metadata key name. Returns nullptr if the key is invalid +# LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key); +@ctypes_function("llama_model_meta_key_str", [ctypes.c_int], ctypes.c_char_p) +def llama_model_meta_key_str(key: int, /) -> Optional[bytes]: + """Get sampling metadata key name. Returns None if the key is invalid.""" + ... + + # // Get metadata key name by index # LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size); @ctypes_function( @@ -1647,6 +1847,14 @@ def llama_model_is_recurrent(model: llama_model_p, /) -> bool: ... +# // Returns true if the model is hybrid (like Jamba, Granite, etc.) +# LLAMA_API bool llama_model_is_hybrid(const struct llama_model * model); +@ctypes_function("llama_model_is_hybrid", [llama_model_p_ctypes], ctypes.c_bool) +def llama_model_is_hybrid(model: llama_model_p, /) -> bool: + """Returns true if the model is hybrid (like Jamba, Granite, etc.)""" + ... + + # // Returns true if the model is diffusion-based (like LLaDA, Dream, etc.) # LLAMA_API bool llama_model_is_diffusion(const struct llama_model * model); @ctypes_function("llama_model_is_diffusion", [llama_model_p_ctypes], ctypes.c_bool) @@ -1698,6 +1906,85 @@ def llama_adapter_lora_init( ) -> Optional[llama_adapter_lora_p]: ... +# // Get metadata value as a string by key name +# LLAMA_API int32_t llama_adapter_meta_val_str(const struct llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size); +@ctypes_function( + "llama_adapter_meta_val_str", + [ + llama_adapter_lora_p_ctypes, + ctypes.c_char_p, + ctypes.c_char_p, + ctypes.c_size_t, + ], + ctypes.c_int32, +) +def llama_adapter_meta_val_str( + adapter: llama_adapter_lora_p, + key: bytes, + buf: Union[bytes, CtypesArray[ctypes.c_char]], + buf_size: int, + /, +) -> int: + """Get adapter metadata value as a string by key name.""" + ... + + +# // Get the number of metadata key/value pairs +# LLAMA_API int32_t llama_adapter_meta_count(const struct llama_adapter_lora * adapter); +@ctypes_function( + "llama_adapter_meta_count", [llama_adapter_lora_p_ctypes], ctypes.c_int32 +) +def llama_adapter_meta_count(adapter: llama_adapter_lora_p, /) -> int: + """Get the number of adapter metadata key/value pairs.""" + ... + + +# // Get metadata key name by index +# LLAMA_API int32_t llama_adapter_meta_key_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size); +@ctypes_function( + "llama_adapter_meta_key_by_index", + [ + llama_adapter_lora_p_ctypes, + ctypes.c_int32, + ctypes.c_char_p, + ctypes.c_size_t, + ], + ctypes.c_int32, +) +def llama_adapter_meta_key_by_index( + adapter: llama_adapter_lora_p, + i: int, + buf: Union[bytes, CtypesArray[ctypes.c_char]], + buf_size: int, + /, +) -> int: + """Get adapter metadata key name by index.""" + ... + + +# // Get metadata value as a string by index +# LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size); +@ctypes_function( + "llama_adapter_meta_val_str_by_index", + [ + llama_adapter_lora_p_ctypes, + ctypes.c_int32, + ctypes.c_char_p, + ctypes.c_size_t, + ], + ctypes.c_int32, +) +def llama_adapter_meta_val_str_by_index( + adapter: llama_adapter_lora_p, + i: int, + buf: Union[bytes, CtypesArray[ctypes.c_char]], + buf_size: int, + /, +) -> int: + """Get adapter metadata value as a string by index.""" + ... + + # // Manually free a LoRA adapter # // Note: loaded adapters will be free when the associated model is deleted # LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter); @@ -1709,56 +1996,75 @@ def llama_adapter_lora_init( def llama_adapter_lora_free(adapter: llama_adapter_lora_p, /): ... -# // The following functions operate on a llama_context, hence the naming: llama_verb_... +# // Get the invocation tokens if the current lora is an alora +# LLAMA_API uint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter); +@ctypes_function( + "llama_adapter_get_alora_n_invocation_tokens", + [llama_adapter_lora_p_ctypes], + ctypes.c_uint64, +) +def llama_adapter_get_alora_n_invocation_tokens( + adapter: llama_adapter_lora_p, / +) -> int: + """Get the invocation token count if the current LoRA is an aLoRA.""" + ... -# // Add a loaded LoRA adapter to given context -# // This will not modify model's weight -# LLAMA_API int32_t llama_set_adapter_lora( -# struct llama_context * ctx, -# struct llama_adapter_lora * adapter, -# float scale); +# LLAMA_API const llama_token * llama_adapter_get_alora_invocation_tokens (const struct llama_adapter_lora * adapter); @ctypes_function( - "llama_set_adapter_lora", - [llama_context_p_ctypes, llama_adapter_lora_p_ctypes, ctypes.c_float], - ctypes.c_int32, + "llama_adapter_get_alora_invocation_tokens", + [llama_adapter_lora_p_ctypes], + ctypes.POINTER(llama_token), ) -def llama_set_adapter_lora( - ctx: llama_context_p, adapter: llama_adapter_lora_p, scale: float, / -) -> int: - """Add a loaded LoRA adapter to given context - This will not modify model's weight""" +def llama_adapter_get_alora_invocation_tokens( + adapter: llama_adapter_lora_p, / +) -> Optional[CtypesPointer[llama_token]]: + """Get the invocation tokens if the current LoRA is an aLoRA.""" ... -# // Remove a specific LoRA adapter from given context -# // Return -1 if the adapter is not present in the context -# LLAMA_API int32_t llama_rm_adapter_lora( +# // The following functions operate on a llama_context, hence the naming: llama_verb_... + + +# // Set LoRa adapters on the context. Will only modify if the adapters currently in context are different. +# LLAMA_API int32_t llama_set_adapters_lora( # struct llama_context * ctx, -# struct llama_adapter_lora * adapter); +# struct llama_adapter_lora ** adapters, +# size_t n_adapters, +# float * scales); @ctypes_function( - "llama_rm_adapter_lora", - [llama_context_p_ctypes, llama_adapter_lora_p_ctypes], + "llama_set_adapters_lora", + [ + llama_context_p_ctypes, + ctypes.POINTER(llama_adapter_lora_p_ctypes), + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_float), + ], ctypes.c_int32, ) -def llama_rm_adapter_lora( - ctx: llama_context_p, adapter: llama_adapter_lora_p, / +def llama_set_adapters_lora( + ctx: llama_context_p, + adapters: Optional[CtypesArray[llama_adapter_lora_p_ctypes]], + n_adapters: int, + scales: Optional[CtypesArray[ctypes.c_float]], + /, ) -> int: - """Remove a specific LoRA adapter from given context - Return -1 if the adapter is not present in the context""" + """Set LoRA adapters on the context if they differ from the current adapters.""" ... -# // Remove all LoRA adapters from given context -# LLAMA_API void llama_clear_adapter_lora(struct llama_context * ctx); -@ctypes_function( - "llama_clear_adapter_lora", - [llama_context_p_ctypes], - None, -) -def llama_clear_adapter_lora(ctx: llama_context_p, /): - """Remove all LoRA adapters from given context""" - ... +# Deprecated compatibility wrapper for the renamed llama_set_adapters_lora(). +def llama_set_adapter_lora( + ctx: llama_context_p, adapter: llama_adapter_lora_p, scale: float, / +) -> int: + warnings.warn( + "llama_set_adapter_lora is deprecated; use llama_set_adapters_lora instead", + DeprecationWarning, + stacklevel=2, + ) + adapters = (llama_adapter_lora_p_ctypes * 1)(adapter) + scales = (ctypes.c_float * 1)(scale) + return llama_set_adapters_lora(ctx, adapters, 1, scales) # // Apply a loaded control vector to a llama_context, or if data is NULL, clear @@ -1767,7 +2073,7 @@ def llama_clear_adapter_lora(ctx: llama_context_p, /): # // to an n_embd x n_layers buffer starting from layer 1. # // il_start and il_end are the layer range the vector should apply to (both inclusive) # // See llama_control_vector_load in common to load a control vector. -# LLAMA_API int32_t llama_apply_adapter_cvec( +# LLAMA_API int32_t llama_set_adapter_cvec( # struct llama_context * ctx, # const float * data, # size_t len, @@ -1775,7 +2081,7 @@ def llama_clear_adapter_lora(ctx: llama_context_p, /): # int32_t il_start, # int32_t il_end); @ctypes_function( - "llama_apply_adapter_cvec", + "llama_set_adapter_cvec", [ llama_context_p_ctypes, ctypes.POINTER(ctypes.c_float), @@ -1786,7 +2092,7 @@ def llama_clear_adapter_lora(ctx: llama_context_p, /): ], ctypes.c_int32, ) -def llama_apply_adapter_cvec( +def llama_set_adapter_cvec( ctx: llama_context_p, data: CtypesPointerOrRef[ctypes.c_float], len: int, @@ -1804,6 +2110,24 @@ def llama_apply_adapter_cvec( ... +# Deprecated compatibility wrapper for the renamed llama_set_adapter_cvec(). +def llama_apply_adapter_cvec( + ctx: llama_context_p, + data: CtypesPointerOrRef[ctypes.c_float], + len: int, + n_embd: int, + il_start: int, + il_end: int, + /, +) -> int: + warnings.warn( + "llama_apply_adapter_cvec is deprecated; use llama_set_adapter_cvec instead", + DeprecationWarning, + stacklevel=2, + ) + return llama_set_adapter_cvec(ctx, data, len, n_embd, il_start, il_end) + + # // # // Memory # // @@ -2018,251 +2342,6 @@ def llama_memory_can_shift(mem: llama_memory_t, /) -> bool: ... -# // -# // KV cache for self-attention (TODO: deprecate in favor of llama_memory) -# // - - -# // Returns the number of tokens in the KV cache (slow, use only for debug) -# // If a KV cell has multiple sequences assigned to it, it will be counted multiple times -# DEPRECATED(LLAMA_API int32_t llama_kv_self_n_tokens(const struct llama_context * ctx), -# "Use llama_kv_self_seq_pos_max() and llama_kv_self_seq_pos_min() instead (https://github.com/ggml-org/llama.cpp/issues/13793)"); -@ctypes_function("llama_kv_self_n_tokens", [llama_context_p_ctypes], ctypes.c_int32) -def llama_kv_self_n_tokens(ctx: llama_context_p, /) -> int: - """Returns the number of tokens in the KV cache (slow, use only for debug) (DEPRECATED)""" - ... - - -# // Returns the number of used KV cells (i.e. have at least one sequence assigned to them) -# DEPRECATED(LLAMA_API int32_t llama_kv_self_used_cells(const struct llama_context * ctx), -# "Use llama_kv_self_seq_pos_max() and llama_kv_self_seq_pos_min() instead (https://github.com/ggml-org/llama.cpp/issues/13793)"); -@ctypes_function("llama_kv_self_used_cells", [llama_context_p_ctypes], ctypes.c_int32) -def llama_kv_self_used_cells(ctx: llama_context_p, /) -> int: - """Returns the number of used KV cells (DEPRECATED)""" - ... - - -# // Clear the KV cache - both cell info is erased and KV data is zeroed -# DEPRECATED(LLAMA_API void llama_kv_self_clear( -# struct llama_context * ctx), -# "Use llama_memory_clear() instead"); -@ctypes_function("llama_kv_self_clear", [llama_context_p_ctypes], None) -def llama_kv_self_clear(ctx: llama_context_p, /): - """Clear the KV cache (DEPRECATED)""" - ... - - -# // Removes all tokens that belong to the specified sequence and have positions in [p0, p1) -# // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails -# // seq_id < 0 : match any sequence -# // p0 < 0 : [0, p1] -# // p1 < 0 : [p0, inf) -# DEPRECATED(LLAMA_API bool llama_kv_self_seq_rm( -# struct llama_context * ctx, -# llama_seq_id seq_id, -# llama_pos p0, -# llama_pos p1), -# "Use llama_memory_seq_rm() instead"); -@ctypes_function( - "llama_kv_self_seq_rm", - [ - llama_context_p_ctypes, - llama_seq_id, - llama_pos, - llama_pos, - ], - ctypes.c_bool, -) -def llama_kv_self_seq_rm( - ctx: llama_context_p, - seq_id: Union[llama_seq_id, int], - p0: Union[llama_pos, int], - p1: Union[llama_pos, int], - /, -) -> bool: - """Remove tokens from KV cache (DEPRECATED)""" - ... - - -# // Copy all tokens that belong to the specified sequence to another sequence -# // Note that this does not allocate extra KV cache memory - it simply assigns the tokens to the new sequence -# // p0 < 0 : [0, p1] -# // p1 < 0 : [p0, inf) -# DEPRECATED(LLAMA_API void llama_kv_self_seq_cp( -# struct llama_context * ctx, -# llama_seq_id seq_id_src, -# llama_seq_id seq_id_dst, -# llama_pos p0, -# llama_pos p1), -# "Use llama_memory_seq_cp() instead"); -@ctypes_function( - "llama_kv_self_seq_cp", - [ - llama_context_p_ctypes, - llama_seq_id, - llama_seq_id, - llama_pos, - llama_pos, - ], - None, -) -def llama_kv_self_seq_cp( - ctx: llama_context_p, - seq_id_src: Union[llama_seq_id, int], - seq_id_dst: Union[llama_seq_id, int], - p0: Union[llama_pos, int], - p1: Union[llama_pos, int], - /, -): - """Copy tokens in KV cache (DEPRECATED)""" - ... - - -# // Removes all tokens that do not belong to the specified sequence -# DEPRECATED(LLAMA_API void llama_kv_self_seq_keep( -# struct llama_context * ctx, -# llama_seq_id seq_id), -# "Use llama_memory_seq_keep() instead"); -@ctypes_function("llama_kv_self_seq_keep", [llama_context_p_ctypes, llama_seq_id], None) -def llama_kv_self_seq_keep(ctx: llama_context_p, seq_id: Union[llama_seq_id, int], /): - """Keep only specified sequence in KV cache (DEPRECATED)""" - ... - - -# // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1) -# // If the KV cache is RoPEd, the KV data is updated accordingly: -# // - lazily on next llama_decode() -# // p0 < 0 : [0, p1] -# // p1 < 0 : [p0, inf) -# DEPRECATED(LLAMA_API void llama_kv_self_seq_add( -# struct llama_context * ctx, -# llama_seq_id seq_id, -# llama_pos p0, -# llama_pos p1, -# llama_pos delta), -# "Use llama_memory_seq_add() instead"); -@ctypes_function( - "llama_kv_self_seq_add", - [ - llama_context_p_ctypes, - llama_seq_id, - llama_pos, - llama_pos, - llama_pos, - ], - None, -) -def llama_kv_self_seq_add( - ctx: llama_context_p, - seq_id: Union[llama_seq_id, int], - p0: Union[llama_pos, int], - p1: Union[llama_pos, int], - delta: Union[llama_pos, int], - /, -): - """Add delta to sequence positions in KV cache (DEPRECATED)""" - ... - - -# // Integer division of the positions by factor of `d > 1` -# // If the KV cache is RoPEd, the KV data is updated accordingly: -# // - lazily on next llama_decode() -# // p0 < 0 : [0, p1] -# // p1 < 0 : [p0, inf) -# DEPRECATED(LLAMA_API void llama_kv_self_seq_div( -# struct llama_context * ctx, -# llama_seq_id seq_id, -# llama_pos p0, -# llama_pos p1, -# int d), -# "Use llama_memory_seq_div() instead"); -@ctypes_function( - "llama_kv_self_seq_div", - [ - llama_context_p_ctypes, - llama_seq_id, - llama_pos, - llama_pos, - ctypes.c_int, - ], - None, -) -def llama_kv_self_seq_div( - ctx: llama_context_p, - seq_id: Union[llama_seq_id, int], - p0: Union[llama_pos, int], - p1: Union[llama_pos, int], - d: Union[ctypes.c_int, int], - /, -): - """Divide sequence positions in KV cache (DEPRECATED)""" - ... - - -# // Returns the smallest position present in the KV cache for the specified sequence -# // This is typically non-zero only for SWA caches -# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the KV cache -# // Return -1 if the sequence is empty -# DEPRECATED(LLAMA_API llama_pos llama_kv_self_seq_pos_min( -# struct llama_context * ctx, -# llama_seq_id seq_id), -# "Use llama_memory_seq_pos_min() instead"); -@ctypes_function( - "llama_kv_self_seq_pos_min", [llama_context_p_ctypes, llama_seq_id], llama_pos -) -def llama_kv_self_seq_pos_min( - ctx: llama_context_p, seq_id: Union[llama_seq_id, int], / -) -> int: - """Returns the smallest position in KV cache for sequence (DEPRECATED)""" - ... - - -# // Returns the largest position present in the KV cache for the specified sequence -# // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the KV cache -# // Return -1 if the sequence is empty -# DEPRECATED(LLAMA_API llama_pos llama_kv_self_seq_pos_max( -# struct llama_context * ctx, -# llama_seq_id seq_id), -# "Use llama_memory_seq_pos_max() instead"); -@ctypes_function( - "llama_kv_self_seq_pos_max", [llama_context_p_ctypes, llama_seq_id], llama_pos -) -def llama_kv_self_seq_pos_max( - ctx: llama_context_p, seq_id: Union[llama_seq_id, int], / -) -> int: - """Returns the largest position in KV cache for sequence (DEPRECATED)""" - ... - - -# // Defragment the KV cache -# // This will be applied: -# // - lazily on next llama_decode() -# DEPRECATED(LLAMA_API void llama_kv_self_defrag(struct llama_context * ctx), -# "simply remove this call, the context will automatically decide when to do a defragmentation based on 'defrag_thold'"); -@ctypes_function("llama_kv_self_defrag", [llama_context_p_ctypes], None) -def llama_kv_self_defrag(ctx: llama_context_p, /): - """Defragment the KV cache (DEPRECATED)""" - ... - - -# // Check if the context supports KV cache shifting -# DEPRECATED(LLAMA_API bool llama_kv_self_can_shift(const struct llama_context * ctx), -# "use llama_memory_can_shift() instead"); -@ctypes_function("llama_kv_self_can_shift", [llama_context_p_ctypes], ctypes.c_bool) -def llama_kv_self_can_shift(ctx: llama_context_p, /) -> bool: - """Check if the context supports KV cache shifting (DEPRECATED)""" - ... - - -# // Apply the KV cache updates (such as K-shifts, defragmentation, etc.) -# DEPRECATED(LLAMA_API void llama_kv_self_update(struct llama_context * ctx), -# "simply remove this call, updates are applied lazily on the next llama_decode()"); -@ctypes_function("llama_kv_self_update", [llama_context_p_ctypes], None) -def llama_kv_self_update(ctx: llama_context_p, /): - """Apply the KV cache updates (DEPRECATED)""" - ... - - # // # // State / sessions # // @@ -2914,6 +2993,100 @@ def llama_get_embeddings_seq( ... +# // Get the backend sampled token for the ith token. +# // Returns LLAMA_TOKEN_NULL if no token was sampled. +# LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i); +@ctypes_function( + "llama_get_sampled_token_ith", [llama_context_p_ctypes, ctypes.c_int32], llama_token +) +def llama_get_sampled_token_ith( + ctx: llama_context_p, i: Union[ctypes.c_int32, int], / +) -> int: + """Get the backend sampled token for the ith token.""" + ... + + +# // Get the backend sampled probabilities for the ith token +# LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i); +@ctypes_function( + "llama_get_sampled_probs_ith", + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(ctypes.c_float), +) +def llama_get_sampled_probs_ith( + ctx: llama_context_p, i: Union[ctypes.c_int32, int], / +) -> Optional[CtypesPointer[ctypes.c_float]]: + """Get the backend sampled probabilities for the ith token.""" + ... + + +# LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i); +@ctypes_function( + "llama_get_sampled_probs_count_ith", + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.c_uint32, +) +def llama_get_sampled_probs_count_ith( + ctx: llama_context_p, i: Union[ctypes.c_int32, int], / +) -> int: + """Get the backend sampled probability count for the ith token.""" + ... + + +# // Get the backend sampled logits for the ith token +# LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i); +@ctypes_function( + "llama_get_sampled_logits_ith", + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(ctypes.c_float), +) +def llama_get_sampled_logits_ith( + ctx: llama_context_p, i: Union[ctypes.c_int32, int], / +) -> Optional[CtypesPointer[ctypes.c_float]]: + """Get the backend sampled logits for the ith token.""" + ... + + +# LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i); +@ctypes_function( + "llama_get_sampled_logits_count_ith", + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.c_uint32, +) +def llama_get_sampled_logits_count_ith( + ctx: llama_context_p, i: Union[ctypes.c_int32, int], / +) -> int: + """Get the backend sampled logit count for the ith token.""" + ... + + +# // Get the backend sampled candidates for the ith token +# LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i); +@ctypes_function( + "llama_get_sampled_candidates_ith", + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(llama_token), +) +def llama_get_sampled_candidates_ith( + ctx: llama_context_p, i: Union[ctypes.c_int32, int], / +) -> Optional[CtypesPointer[llama_token]]: + """Get the backend sampled candidates for the ith token.""" + ... + + +# LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i); +@ctypes_function( + "llama_get_sampled_candidates_count_ith", + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.c_uint32, +) +def llama_get_sampled_candidates_count_ith( + ctx: llama_context_p, i: Union[ctypes.c_int32, int], / +) -> int: + """Get the backend sampled candidate count for the ith token.""" + ... + + # // # // Vocab # // @@ -3558,6 +3731,21 @@ def llama_chat_builtin_templates( llama_sampler_context_t = ctypes.c_void_p +# struct llama_sampler_data { +# struct ggml_tensor * logits; +# struct ggml_tensor * probs; +# struct ggml_tensor * sampled; +# struct ggml_tensor * candidates; +# }; +class llama_sampler_data(ctypes.Structure): + _fields_ = [ + ("logits", ctypes.c_void_p), + ("probs", ctypes.c_void_p), + ("sampled", ctypes.c_void_p), + ("candidates", ctypes.c_void_p), + ] + + # // user code can implement the interface below in order to create custom llama_sampler # struct llama_sampler_i { # const char * (*name) (const struct llama_sampler * smpl); // can be NULL @@ -3598,6 +3786,24 @@ class llama_sampler(ctypes.Structure): llama_sampler_i_reset = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) llama_sampler_i_clone = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, llama_sampler_p_ctypes) llama_sampler_i_free = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i_backend_init = ctypes.CFUNCTYPE( + ctypes.c_bool, llama_sampler_p_ctypes, ctypes.c_void_p +) +llama_sampler_i_backend_accept = ctypes.CFUNCTYPE( + None, + llama_sampler_p_ctypes, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_void_p, +) +llama_sampler_i_backend_apply = ctypes.CFUNCTYPE( + None, + llama_sampler_p_ctypes, + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.POINTER(llama_sampler_data), +) +llama_sampler_i_backend_set_input = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) llama_sampler_i._fields_ = [ ("name", llama_sampler_i_name), @@ -3606,9 +3812,27 @@ class llama_sampler(ctypes.Structure): ("reset", llama_sampler_i_reset), ("clone", llama_sampler_i_clone), ("free", llama_sampler_i_free), + ("backend_init", llama_sampler_i_backend_init), + ("backend_accept", llama_sampler_i_backend_accept), + ("backend_apply", llama_sampler_i_backend_apply), + ("backend_set_input", llama_sampler_i_backend_set_input), ] +# // attach a sampler to the context +# LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl); +@ctypes_function( + "llama_set_sampler", + [llama_context_p_ctypes, llama_seq_id, llama_sampler_p_ctypes], + ctypes.c_bool, +) +def llama_set_sampler( + ctx: llama_context_p, seq_id: Union[llama_seq_id, int], smpl: llama_sampler_p, / +) -> bool: + """Attach a sampler to the context.""" + ... + + # // mirror of llama_sampler_i: # LLAMA_API struct llama_sampler * llama_sampler_init (const struct llama_sampler_i * iface, llama_sampler_context_t ctx); @ctypes_function( @@ -3748,14 +3972,6 @@ def llama_sampler_init_greedy() -> llama_sampler_p: ... def llama_sampler_init_dist(seed: int) -> llama_sampler_p: ... -# /// @details Sorts candidate tokens by their logits in descending order and calculate probabilities based on logits. -# /// NOTE: Avoid using on the full vocabulary as the sorting can become slow. For example, apply top-k or top-p sampling first. -# DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_softmax (void), -# "will be removed in the future (see https://github.com/ggml-org/llama.cpp/pull/9896#discussion_r1800920915)"); -@ctypes_function("llama_sampler_init_softmax", [], llama_sampler_p_ctypes) -def llama_sampler_init_softmax() -> llama_sampler_p: ... - - # /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 # /// Setting k <= 0 makes this a noop # LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); @@ -4005,6 +4221,22 @@ def llama_sampler_init_dry( ) -> llama_sampler_p: ... +# LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p( +# float target, +# float decay, +# uint32_t seed); +@ctypes_function( + "llama_sampler_init_adaptive_p", + [ctypes.c_float, ctypes.c_float, ctypes.c_uint32], + llama_sampler_p_ctypes, +) +def llama_sampler_init_adaptive_p( + target: float, decay: float, seed: int, / +) -> llama_sampler_p: + """Initialize an adaptive-p sampler.""" + ... + + # LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( # int32_t n_vocab, # int32_t n_logit_bias, @@ -4102,10 +4334,26 @@ def llama_print_system_info() -> bytes: ... # // Set callback for all future logging events. # // If this is not called, or NULL is supplied, everything is output on stderr. +# // The logger state is global so these functions are NOT thread safe. +# LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data); +@ctypes_function( + "llama_log_get", + [ctypes.POINTER(llama_log_callback), ctypes.POINTER(ctypes.c_void_p)], + None, +) +def llama_log_get( + log_callback: CtypesPointerOrRef[llama_log_callback], + user_data: CtypesPointerOrRef[ctypes.c_void_p], + /, +): + """Get the current logging callback and user data.""" + ... + + # LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function( "llama_log_set", - [ctypes.c_void_p, ctypes.c_void_p], + [llama_log_callback, ctypes.c_void_p], None, ) def llama_log_set( @@ -4214,6 +4462,13 @@ def llama_perf_sampler_print(chain: llama_sampler_p, /): ... def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... +# // print a breakdown of per-device memory use via LLAMA_LOG: +@ctypes_function("llama_memory_breakdown_print", [llama_context_p_ctypes], None) +def llama_memory_breakdown_print(ctx: llama_context_p, /): + """Print a breakdown of per-device memory use.""" + ... + + # // # // training # // diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 41753a7f60..787683179c 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import warnings from ctypes import ( c_bool, c_char_p, @@ -78,13 +79,31 @@ # Structures class mtmd_context_params(Structure): + if TYPE_CHECKING: + use_gpu: bool + print_timings: bool + n_threads: int + image_marker: Optional[bytes] + media_marker: Optional[bytes] + flash_attn_type: int + warmup: bool + image_min_tokens: int + image_max_tokens: int + cb_eval: llama_cpp.ggml_backend_sched_eval_callback + cb_eval_user_data: c_void_p + _fields_ = [ ("use_gpu", c_bool), ("print_timings", c_bool), ("n_threads", c_int), - ("verbosity", c_int), # ggml_log_level ("image_marker", c_char_p), ("media_marker", c_char_p), + ("flash_attn_type", c_int), + ("warmup", c_bool), + ("image_min_tokens", c_int), + ("image_max_tokens", c_int), + ("cb_eval", llama_cpp.ggml_backend_sched_eval_callback), + ("cb_eval_user_data", c_void_p), ] @@ -132,11 +151,49 @@ def mtmd_init_from_file( def mtmd_free(ctx: mtmd_context_p, /): ... +# MTMD_API bool mtmd_decode_use_non_causal(mtmd_context * ctx); +@ctypes_function("mtmd_decode_use_non_causal", [mtmd_context_p_ctypes], c_bool) +def mtmd_decode_use_non_causal(ctx: mtmd_context_p, /) -> bool: + """Check whether MTMD decoding uses non-causal attention.""" + ... + + +# MTMD_API bool mtmd_decode_use_mrope(mtmd_context * ctx); +@ctypes_function("mtmd_decode_use_mrope", [mtmd_context_p_ctypes], c_bool) +def mtmd_decode_use_mrope(ctx: mtmd_context_p, /) -> bool: + """Check whether MTMD decoding uses mRoPE.""" + ... + + # MTMD_API bool mtmd_support_vision(mtmd_context * ctx); @ctypes_function("mtmd_support_vision", [mtmd_context_p_ctypes], c_bool) def mtmd_support_vision(ctx: mtmd_context_p, /) -> bool: ... +# MTMD_API bool mtmd_support_audio(mtmd_context * ctx); +@ctypes_function("mtmd_support_audio", [mtmd_context_p_ctypes], c_bool) +def mtmd_support_audio(ctx: mtmd_context_p, /) -> bool: + """Check whether MTMD supports audio.""" + ... + + +# MTMD_API int mtmd_get_audio_sample_rate(mtmd_context * ctx); +@ctypes_function("mtmd_get_audio_sample_rate", [mtmd_context_p_ctypes], c_int) +def mtmd_get_audio_sample_rate(ctx: mtmd_context_p, /) -> int: + """Get the MTMD audio sample rate.""" + ... + + +# Deprecated compatibility wrapper for the renamed mtmd_get_audio_sample_rate(). +def mtmd_get_audio_bitrate(ctx: mtmd_context_p, /) -> int: + warnings.warn( + "mtmd_get_audio_bitrate is deprecated; use mtmd_get_audio_sample_rate instead", + DeprecationWarning, + stacklevel=2, + ) + return mtmd_get_audio_sample_rate(ctx) + + # MTMD_API mtmd_bitmap * mtmd_bitmap_init(uint32_t nx, uint32_t ny, const unsigned char * data); @ctypes_function( "mtmd_bitmap_init", [c_uint32, c_uint32, POINTER(c_uint8)], mtmd_bitmap_p_ctypes @@ -149,6 +206,21 @@ def mtmd_bitmap_init( ) -> Optional[mtmd_bitmap_p]: ... +# MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data); +@ctypes_function( + "mtmd_bitmap_init_from_audio", + [c_size_t, POINTER(c_float)], + mtmd_bitmap_p_ctypes, +) +def mtmd_bitmap_init_from_audio( + n_samples: Union[c_size_t, int], + data: CtypesArray[c_float], + /, +) -> Optional[mtmd_bitmap_p]: + """Initialize an MTMD bitmap from audio samples.""" + ... + + # MTMD_API void mtmd_bitmap_free(mtmd_bitmap * bitmap); @ctypes_function("mtmd_bitmap_free", [mtmd_bitmap_p_ctypes], None) def mtmd_bitmap_free(bitmap: mtmd_bitmap_p, /): ... @@ -284,3 +356,25 @@ def mtmd_helper_eval_chunk_single( new_n_past: "_Pointer[llama_cpp.llama_pos]", /, ) -> int: ... + + +# MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); +@ctypes_function( + "mtmd_log_set", + [llama_cpp.llama_log_callback, c_void_p], + None, +) +def mtmd_log_set(log_callback, user_data: c_void_p, /): + """Set the MTMD logging callback.""" + ... + + +# MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data); +@ctypes_function( + "mtmd_helper_log_set", + [llama_cpp.llama_log_callback, c_void_p], + None, +) +def mtmd_helper_log_set(log_callback, user_data: c_void_p, /): + """Set the MTMD helper logging callback.""" + ... diff --git a/tests/test_llama.py b/tests/test_llama.py index 964b0895cc..619c7378de 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -83,7 +83,7 @@ def test_real_model(llama_cpp_model_path): cparams.n_threads = multiprocessing.cpu_count() cparams.n_threads_batch = multiprocessing.cpu_count() cparams.logits_all = False - cparams.flash_attn = True + cparams.flash_attn_type = llama_cpp.LLAMA_FLASH_ATTN_TYPE_ENABLED context = internals.LlamaContext(model=model, params=cparams) tokens = model.tokenize(b"Hello, world!", add_bos=True, special=True) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4227c9be42..49bfddeca1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4227c9be4268ac844921b90f31595f81236bd317 +Subproject commit 49bfddeca18e62fa3d39114a23e9fcbdf8a22388 From e1f8ac0e135201c4a6de7c828331a2f078e9763a Mon Sep 17 00:00:00 2001 From: Bruno Verachten Date: Mon, 23 Mar 2026 03:27:22 +0100 Subject: [PATCH 441/561] ci: add riscv64 wheel builds to release workflow (#2139) * ci: add riscv64 wheel builds to release workflow Add a build_wheels_riscv64 job mirroring the existing arm64 QEMU-based build. Uses cibuildwheel with QEMU emulation for linux/riscv64, targeting CPython 3.10-3.14 on manylinux. Closes #2138 * ci: use cibuildwheel 3.1.2 for riscv64 wheels * docs: update changelog for riscv64 wheel PR --------- Co-authored-by: abetlen --- .github/workflows/build-and-release.yaml | 31 +++++++++++++++++++++++- CHANGELOG.md | 1 + 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 7eaf017fbc..0121febe8a 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -85,6 +85,35 @@ jobs: name: wheels_arm64 path: ./wheelhouse/*.whl + build_wheels_riscv64: + name: Build riscv64 wheels + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/riscv64 + + - name: Build wheels + uses: pypa/cibuildwheel@v3.1.2 + env: + CIBW_SKIP: "*musllinux* pp*" + CIBW_REPAIR_WHEEL_COMMAND: "" + CIBW_ARCHS: "riscv64" + CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*" + with: + output-dir: wheelhouse + + - name: Upload wheels as artifacts + uses: actions/upload-artifact@v4 + with: + name: wheels_riscv64 + path: ./wheelhouse/*.whl + build_sdist: name: Build source distribution runs-on: ubuntu-latest @@ -129,7 +158,7 @@ jobs: release: name: Release - needs: [build_wheels, build_wheels_arm64, build_sdist] + needs: [build_wheels, build_wheels_arm64, build_wheels_riscv64, build_sdist] runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index e7506eed57..94666cec1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - chore(dev): Add Ruff-based formatting and a safe lint baseline, and run it in CI for pull requests and pushes to `main` - fix(ci): Run macOS CI on supported Apple Silicon and Intel runners by @abetlen in #2150 - fix(ci): Use the `hf` CLI instead of the deprecated `huggingface-cli` name in GitHub Actions and docs by @abetlen in #2149 +- ci: add riscv64 wheel builds to release workflow by @gounthar in #2139 ## [0.3.16] From 11e7a55af76303d42f2bc5a79ed0babbc89652dd Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 22 Mar 2026 22:33:31 -0700 Subject: [PATCH 442/561] fix: Qwen 3.5 support (#2152) * fix: handle Qwen 3.5 hybrid prefix reuse * test: fix Qwen runtime unit mocks * test: drop Qwen runtime unit tests * docs: credit Qwen fix contributors in changelog * docs/tests: update default Qwen model to 3.5 0.8B * test: rebaseline Qwen 3.5 outputs * test: stabilize low-level Qwen sampling check * test: tighten Qwen 3.5 completion prompts --- .github/workflows/test.yaml | 4 ++-- CHANGELOG.md | 1 + README.md | 6 +++--- examples/gradio_chat/local.py | 6 +++--- examples/hf_pull/main.py | 6 +++--- llama_cpp/_internals.py | 4 ++-- llama_cpp/llama.py | 19 +++++++++++++------ tests/test_llama.py | 33 ++++++++++++++++++++------------- 8 files changed, 47 insertions(+), 32 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index af4cacac40..8a6845ff24 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,8 +8,8 @@ on: - main env: - REPO_ID: Qwen/Qwen2-0.5B-Instruct-GGUF - MODEL_FILE: qwen2-0_5b-instruct-q8_0.gguf + REPO_ID: lmstudio-community/Qwen3.5-0.8B-GGUF + MODEL_FILE: Qwen3.5-0.8B-Q8_0.gguf jobs: download-model: diff --git a/CHANGELOG.md b/CHANGELOG.md index 94666cec1d..4153406c10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 and sync Python bindings by @abetlen in #2151 +- fix: Handle Qwen 3.5 hybrid prefix reuse by @codavidgarcia and @r-dh in #2152 - chore(dev): Add Ruff-based formatting and a safe lint baseline, and run it in CI for pull requests and pushes to `main` - fix(ci): Run macOS CI on supported Apple Silicon and Intel runners by @abetlen in #2150 - fix(ci): Use the `hf` CLI instead of the deprecated `huggingface-cli` name in GitHub Actions and docs by @abetlen in #2149 diff --git a/README.md b/README.md index b57c958076..8ba4dbb5ea 100644 --- a/README.md +++ b/README.md @@ -322,8 +322,8 @@ You'll need to install the `huggingface-hub` package to use this feature (`pip i ```python llm = Llama.from_pretrained( - repo_id="Qwen/Qwen2-0.5B-Instruct-GGUF", - filename="*q8_0.gguf", + repo_id="lmstudio-community/Qwen3.5-0.8B-GGUF", + filename="*Q8_0.gguf", verbose=False ) ``` @@ -685,7 +685,7 @@ For possible options, see [llama_cpp/llama_chat_format.py](llama_cpp/llama_chat_ If you have `huggingface-hub` installed, you can also use the `--hf_model_repo_id` flag to load a model from the Hugging Face Hub. ```bash -python3 -m llama_cpp.server --hf_model_repo_id Qwen/Qwen2-0.5B-Instruct-GGUF --model '*q8_0.gguf' +python3 -m llama_cpp.server --hf_model_repo_id lmstudio-community/Qwen3.5-0.8B-GGUF --model '*Q8_0.gguf' ``` ### Web Server Features diff --git a/examples/gradio_chat/local.py b/examples/gradio_chat/local.py index e16bf234ab..871d8b09b5 100644 --- a/examples/gradio_chat/local.py +++ b/examples/gradio_chat/local.py @@ -4,10 +4,10 @@ import gradio as gr llama = llama_cpp.Llama.from_pretrained( - repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF", - filename="*q8_0.gguf", + repo_id="lmstudio-community/Qwen3.5-0.8B-GGUF", + filename="*Q8_0.gguf", tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained( - "Qwen/Qwen1.5-0.5B" + "Qwen/Qwen3.5-0.8B" ), verbose=False, ) diff --git a/examples/hf_pull/main.py b/examples/hf_pull/main.py index dfed17516d..a9ca424d19 100644 --- a/examples/hf_pull/main.py +++ b/examples/hf_pull/main.py @@ -3,10 +3,10 @@ llama = llama_cpp.Llama.from_pretrained( - repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF", - filename="*q8_0.gguf", + repo_id="lmstudio-community/Qwen3.5-0.8B-GGUF", + filename="*Q8_0.gguf", tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained( - "Qwen/Qwen1.5-0.5B" + "Qwen/Qwen3.5-0.8B" ), verbose=False, ) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index d6258d2244..6862135aa7 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -291,10 +291,10 @@ def kv_cache_clear(self): assert self.memory is not None, "Memory is not initialized" llama_cpp.llama_memory_clear(self.memory, True) - def kv_cache_seq_rm(self, seq_id: int, p0: int, p1: int): + def kv_cache_seq_rm(self, seq_id: int, p0: int, p1: int) -> bool: assert self.memory is not None, "Memory is not initialized" seq_id = seq_id if seq_id >= 0 else 0 - llama_cpp.llama_memory_seq_rm(self.memory, seq_id, p0, p1) + return llama_cpp.llama_memory_seq_rm(self.memory, seq_id, p0, p1) def kv_cache_seq_cp(self, seq_id_src: int, seq_id_dst: int, p0: int, p1: int): assert self.memory is not None, "Memory is not initialized" diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 1609ad16bf..88bc2e5bb8 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -891,13 +891,20 @@ def generate( else: break if longest_prefix > 0: - reset = False - tokens = tokens[longest_prefix:] - self.n_tokens = longest_prefix - if self.verbose: + if self._ctx.kv_cache_seq_rm(-1, longest_prefix, -1): + reset = False + tokens = tokens[longest_prefix:] + self.n_tokens = longest_prefix + if self.verbose: + print( + f"Llama.generate: {longest_prefix} prefix-match hit, " + f"remaining {len(tokens)} prompt tokens to eval", + file=sys.stderr, + ) + elif self.verbose: print( - f"Llama.generate: {longest_prefix} prefix-match hit, " - f"remaining {len(tokens)} prompt tokens to eval", + f"Llama.generate: {longest_prefix} prefix-match found " + f"but partial kv removal not supported, re-evaluating full prompt", file=sys.stderr, ) diff --git a/tests/test_llama.py b/tests/test_llama.py index 619c7378de..1a70c74d43 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -58,8 +58,8 @@ def test_llama_cpp_tokenization(): @pytest.fixture def llama_cpp_model_path(): - repo_id = "Qwen/Qwen2-0.5B-Instruct-GGUF" - filename = "qwen2-0_5b-instruct-q8_0.gguf" + repo_id = "lmstudio-community/Qwen3.5-0.8B-GGUF" + filename = "Qwen3.5-0.8B-Q8_0.gguf" model_path = hf_hub_download(repo_id, filename) return model_path @@ -88,9 +88,14 @@ def test_real_model(llama_cpp_model_path): context = internals.LlamaContext(model=model, params=cparams) tokens = model.tokenize(b"Hello, world!", add_bos=True, special=True) - assert tokens == [9707, 11, 1879, 0] + assert tokens == [9419, 11, 1814, 0] - tokens = model.tokenize(b"The quick brown fox jumps", add_bos=True, special=True) + tokens = model.tokenize( + b"The quick brown fox jumps over the lazy dog. The quick brown fox jumps ", + add_bos=True, + special=True, + ) + prompt_token_count = len(tokens) batch = internals.LlamaBatch(n_tokens=len(tokens), embd=0, n_seq_max=1) @@ -111,9 +116,11 @@ def test_real_model(llama_cpp_model_path): tokens = [token_id] result += tokens - output = result[5:] + output = result[prompt_token_count:] output_text = model.detokenize(output, special=True) - assert output_text == b" over the lazy dog" + # Low-level sampling output varies across CPU and Metal backends. + assert len(output) == 4 + assert output_text def test_real_llama(llama_cpp_model_path): @@ -129,14 +136,14 @@ def test_real_llama(llama_cpp_model_path): ) output = model.create_completion( - "The quick brown fox jumps", - max_tokens=4, + "The quick brown fox jumps over the lazy dog. The quick brown fox", + max_tokens=6, top_k=50, top_p=0.9, - temperature=0.8, + temperature=0.0, seed=1337, ) - assert output["choices"][0]["text"] == " over the lazy dog" + assert output["choices"][0]["text"] == " jumps over the lazy dog." output = model.create_completion( "The capital of france is paris, 'true' or 'false'?:\n", @@ -181,7 +188,7 @@ def logit_processor_func(input_ids, logits): max_tokens=4, top_k=50, top_p=0.9, - temperature=0.8, + temperature=1.0, grammar=llama_cpp.LlamaGrammar.from_string(""" root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" """), @@ -193,7 +200,7 @@ def logit_processor_func(input_ids, logits): max_tokens=4, top_k=50, top_p=0.9, - temperature=0.8, + temperature=1.0, grammar=llama_cpp.LlamaGrammar.from_string(""" root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" """), @@ -207,7 +214,7 @@ def logit_processor_func(input_ids, logits): max_tokens=4, top_k=50, top_p=0.9, - temperature=0.8, + temperature=1.0, grammar=llama_cpp.LlamaGrammar.from_string(""" root ::= "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "10" """), From a6b180724a641750aeaefa51692fe12ee8c4d54f Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 22 Mar 2026 23:28:55 -0700 Subject: [PATCH 443/561] chore: Bump version (#2153) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4153406c10..d1195cc2ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.17] + - feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 and sync Python bindings by @abetlen in #2151 - fix: Handle Qwen 3.5 hybrid prefix reuse by @codavidgarcia and @r-dh in #2152 - chore(dev): Add Ruff-based formatting and a safe lint baseline, and run it in CI for pull requests and pushes to `main` diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index c1dde70468..a7c40478bc 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.16" +__version__ = "0.3.17" From f0391c5ea7159b4c4916d9f4aced2f982adbd1f4 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 24 Mar 2026 00:59:19 -0700 Subject: [PATCH 444/561] fix(ci): release wheel workflow (#2154) * fix(ci): harden release wheel workflow * fix(ci): document and pin release wheel baselines * fix(ci): speed up release arch builds * fix(ci): split riscv64 by python version * fix(ci): sanitize riscv64 artifact names --- .github/workflows/build-and-release.yaml | 49 +++++++++++++++++++----- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 0121febe8a..3a9e6f3698 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -46,6 +46,13 @@ jobs: env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" + # Skip cibuildwheel's default i686 sidecar and keep Linux release + # wheels on a portable x86_64 CPU baseline. + CIBW_ARCHS_LINUX: "auto64" + CIBW_ENVIRONMENT_LINUX: CMAKE_ARGS="-DGGML_NATIVE=off" + # Keep macOS release wheels on a portable CPU baseline instead of + # inheriting the hosted runner's native flags. + CIBW_ENVIRONMENT_MACOS: CMAKE_ARGS="-DGGML_NATIVE=off" with: package-dir: . output-dir: wheelhouse @@ -57,24 +64,21 @@ jobs: build_wheels_arm64: name: Build arm64 wheels - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 with: submodules: "recursive" - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: linux/arm64 - - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" CIBW_ARCHS: "aarch64" - CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DCMAKE_CROSSCOMPILING=ON" + # Keep native arm64 builds on a portable CPU baseline instead of + # tuning wheels to the hosted runner. + CIBW_ENVIRONMENT: CMAKE_ARGS="-DGGML_NATIVE=off" CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" with: output-dir: wheelhouse @@ -86,8 +90,27 @@ jobs: path: ./wheelhouse/*.whl build_wheels_riscv64: - name: Build riscv64 wheels + name: Build riscv64 wheels (${{ matrix.shard.name }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: + - name: cp310 + build: "cp310-*" + artifact: wheels_riscv64_cp310 + - name: cp311 + build: "cp311-*" + artifact: wheels_riscv64_cp311 + - name: cp312 + build: "cp312-*" + artifact: wheels_riscv64_cp312 + - name: cp313 + build: "cp313-*" + artifact: wheels_riscv64_cp313 + - name: cp314 + build: "cp314-*" + artifact: wheels_riscv64_cp314 steps: - uses: actions/checkout@v4 with: @@ -104,14 +127,19 @@ jobs: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" CIBW_ARCHS: "riscv64" - CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*" + # Build riscv64 wheels against a conservative baseline instead of + # enabling RVV-related extensions from the build container. + CIBW_ENVIRONMENT: CMAKE_ARGS="-DGGML_NATIVE=off -DGGML_RVV=off -DGGML_RV_ZFH=off -DGGML_RV_ZVFH=off -DGGML_RV_ZICBOP=off -DGGML_RV_ZIHINTPAUSE=off" + # Split the emulated riscv64 build into one Python version per job + # to minimize wall-clock time without changing the release artifacts. + CIBW_BUILD: ${{ matrix.shard.build }} with: output-dir: wheelhouse - name: Upload wheels as artifacts uses: actions/upload-artifact@v4 with: - name: wheels_riscv64 + name: ${{ matrix.shard.artifact }} path: ./wheelhouse/*.whl build_sdist: @@ -159,6 +187,7 @@ jobs: release: name: Release needs: [build_wheels, build_wheels_arm64, build_wheels_riscv64, build_sdist] + if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: From 909ebf1246a52c15ebc95460c7e5957e3b64711e Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 24 Mar 2026 01:00:50 -0700 Subject: [PATCH 445/561] fix(ci): cuda wheel workflow (#2155) * fix(ci): harden cuda wheel workflow * fix(ci): pin cuda toolkit versions accurately * fix(ci): resolve exact cuda toolkit installs * fix(ci): align cuda toolkit roots and tags * fix(ci): pin cuda packages to nvidia label * fix(ci): allow cuda solver to mix non-cuda deps --- .github/workflows/build-wheels-cuda.yaml | 57 +++++++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 07b30cfc06..b8d6c9dcec 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -99,21 +99,63 @@ jobs: MAMBA_NO_LOW_SPEED_LIMIT: "1" run: | $cudaVersion = $env:CUDAVER - mamba install -y 'cuda' -c nvidia/label/cuda-$cudaVersion + $cudaChannel = "nvidia/label/cuda-$cudaVersion" + if ($IsLinux) { + # Keep nvcc, cudart, and headers on the same NVIDIA label so the + # detected toolkit version matches the published wheel tag. + mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "$cudaChannel::cuda-toolkit=$cudaVersion" "$cudaChannel::cuda-nvcc_linux-64=$cudaVersion" "$cudaChannel::cuda-cudart" "$cudaChannel::cuda-cudart-dev" + } else { + mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "$cudaChannel::cuda-toolkit=$cudaVersion" + } + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } python -m pip install build wheel - name: Build Wheel run: | - $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','') $env:CUDA_PATH = $env:CONDA_PREFIX $env:CUDA_HOME = $env:CONDA_PREFIX $env:CUDA_TOOLKIT_ROOT_DIR = $env:CONDA_PREFIX + $cudaHostCompilerArg = '' + $env:CMAKE_ARGS = '' if ($IsLinux) { - $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH + if (Test-Path '/usr/bin/g++-12') { + $env:CC = '/usr/bin/gcc-12' + $env:CXX = '/usr/bin/g++-12' + $env:CUDAHOSTCXX = '/usr/bin/g++-12' + $cudaHostCompilerArg = " -DCMAKE_CUDA_HOST_COMPILER=$env:CUDAHOSTCXX" + } + if (Test-Path (Join-Path $env:CONDA_PREFIX 'include/cuda_runtime.h')) { + $env:CUDAToolkit_ROOT = $env:CONDA_PREFIX + $env:CUDA_TOOLKIT_ROOT_DIR = $env:CONDA_PREFIX + $env:CMAKE_ARGS = "-DCUDAToolkit_ROOT=$env:CONDA_PREFIX -DCUDA_TOOLKIT_ROOT_DIR=$env:CONDA_PREFIX$cudaHostCompilerArg" + $env:CPATH = "$env:CONDA_PREFIX/include:$env:CPATH" + $env:CPLUS_INCLUDE_PATH = "$env:CONDA_PREFIX/include:$env:CPLUS_INCLUDE_PATH" + $env:LIBRARY_PATH = "$env:CONDA_PREFIX/lib:$env:LIBRARY_PATH" + $env:LD_LIBRARY_PATH = "$env:CONDA_PREFIX/lib:$env:LD_LIBRARY_PATH" + } else { + $env:CMAKE_ARGS = $cudaHostCompilerArg.Trim() + } + } + $nvccPath = Join-Path $env:CONDA_PREFIX 'bin/nvcc' + if (-not (Test-Path $nvccPath)) { + $nvccPath = Join-Path $env:CONDA_PREFIX 'targets/x86_64-linux/bin/nvcc' + } + if (-not (Test-Path $nvccPath)) { + throw 'Failed to find nvcc in the conda environment' + } + $env:CUDACXX = $nvccPath + $env:PATH = "$(Split-Path $nvccPath):$env:PATH" + $nvccVersion = ((& $nvccPath --version) | Select-String 'release ([0-9]+\.[0-9]+)').Matches[0].Groups[1].Value + if (-not $nvccVersion) { + throw 'Failed to detect the installed CUDA toolkit version' } + $cudaTagVersion = $nvccVersion.Replace('.','') $env:VERBOSE = '1' - $env:CMAKE_ARGS = '-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=all' - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS" + # Keep a portable SM set, including sm_70, instead of CMake's `all`, + # which now pulls in future targets the hosted-runner toolchains cannot assemble. + $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70;75;80;86;89;90 -DCMAKE_CUDA_FLAGS=--allow-unsupported-compiler $env:CMAKE_ARGS" # if ($env:AVXVER -eq 'AVX') { $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' # } @@ -124,10 +166,11 @@ jobs: # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' # } python -m build --wheel - # write the build tag to the output - Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV + # Publish tags that reflect the actual installed toolkit version. + Write-Output "CUDA_VERSION=$cudaTagVersion" >> $env:GITHUB_ENV - uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') with: files: dist/* # Set tag_name to -cu From ccc6bc0454b2d73431a419620aad92fda1aba162 Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 24 Mar 2026 01:02:14 -0700 Subject: [PATCH 446/561] fix(ci): docker build workflow (#2156) * fix(ci): harden docker build workflow * docs: update changelog for ci workflows --- .github/workflows/build-docker.yaml | 11 ++++++++++- CHANGELOG.md | 4 ++++ docker/simple/Dockerfile | 3 ++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index b290f6273f..c656958479 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -16,6 +16,15 @@ jobs: with: submodules: "recursive" + - name: Set image tag + run: | + if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + image_tag="${GITHUB_REF_NAME}" + else + image_tag="${GITHUB_REF_NAME//\//-}" + fi + echo "IMAGE_TAG=$image_tag" >> "$GITHUB_ENV" + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -40,7 +49,7 @@ jobs: platforms: linux/amd64,linux/arm64 tags: | ghcr.io/abetlen/llama-cpp-python:latest - ghcr.io/abetlen/llama-cpp-python:${{ github.ref_name }} + ghcr.io/abetlen/llama-cpp-python:${{ env.IMAGE_TAG }} build-args: | BUILDKIT_INLINE_CACHE=1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d1195cc2ae..b476131099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(ci): Build Docker images from the checked-out source and sanitize branch tags by @abetlen in #2156 +- fix(ci): Fix the CUDA wheel workflow and keep release tags aligned with the built toolkit by @abetlen in #2155 +- fix(ci): Speed up release wheel builds by moving arm64 off QEMU and parallelizing riscv64 by @abetlen in #2154 + ## [0.3.17] - feat: Update llama.cpp to ggerganov/llama.cpp@49bfddeca18e62fa3d39114a23e9fcbdf8a22388 and sync Python bindings by @abetlen in #2151 diff --git a/docker/simple/Dockerfile b/docker/simple/Dockerfile index 06483d44e5..bad4f456ff 100644 --- a/docker/simple/Dockerfile +++ b/docker/simple/Dockerfile @@ -6,6 +6,7 @@ FROM ${IMAGE} # Re-declare the ARG after FROM ARG IMAGE +ARG CMAKE_ARGS="-DGGML_NATIVE=off" # Update and upgrade the existing packages RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ @@ -26,7 +27,7 @@ RUN python3 -m pip install --upgrade pip RUN python3 -m pip install --upgrade pip pytest cmake scikit-build setuptools fastapi uvicorn sse-starlette pydantic-settings starlette-context -RUN pip install llama-cpp-python --verbose; +RUN CMAKE_ARGS="${CMAKE_ARGS}" pip install . --verbose # Set environment variable for the host ENV HOST=0.0.0.0 From 7b38c3122d2ff3ad23e1502de045807836ced4a7 Mon Sep 17 00:00:00 2001 From: Victor Biederbeck Date: Tue, 24 Mar 2026 02:50:15 -0700 Subject: [PATCH 447/561] feat: expose attention_type parameter in Llama.__init__ (#2143) * feat: expose attention_type parameter in Llama.__init__ * docs: preserve attention_type in pickled state * docs: update changelog for attention_type --------- Co-authored-by: Victor Biederbeck Co-authored-by: abetlen --- CHANGELOG.md | 1 + llama_cpp/llama.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b476131099..de4f070ffc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: Expose `attention_type` in `Llama.__init__` for non-causal embedding models by @jamesbiederbeck in #2143 - fix(ci): Build Docker images from the checked-out source and sanitize branch tags by @abetlen in #2156 - fix(ci): Fix the CUDA wheel workflow and keep release tags aligned with the built toolkit by @abetlen in #2155 - fix(ci): Speed up release wheel builds by moving arm64 off QEMU and parallelizing riscv64 by @abetlen in #2154 diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 88bc2e5bb8..ad484c4d5f 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -81,6 +81,7 @@ def __init__( int ] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED, + attention_type: int = llama_cpp.LLAMA_ATTENTION_TYPE_UNSPECIFIED, rope_freq_base: float = 0.0, rope_freq_scale: float = 0.0, yarn_ext_factor: float = -1.0, @@ -163,6 +164,7 @@ def __init__( n_threads_batch: Number of threads to use for batch processing rope_scaling_type: RoPE scaling type, from `enum llama_rope_scaling_type`. ref: https://github.com/ggerganov/llama.cpp/pull/2054 pooling_type: Pooling type, from `enum llama_pooling_type`. + attention_type: Attention type, from `enum llama_attention_type`. rope_freq_base: RoPE base frequency, 0 = from model rope_freq_scale: RoPE frequency scaling factor, 0 = from model yarn_ext_factor: YaRN extrapolation mix factor, negative = from model @@ -319,6 +321,7 @@ def __init__( else llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED ) self.context_params.pooling_type = pooling_type + self.context_params.attention_type = attention_type self.context_params.rope_freq_base = ( rope_freq_base if rope_freq_base != 0.0 else 0 ) @@ -2100,6 +2103,7 @@ def __getstate__(self): n_threads_batch=self.context_params.n_threads_batch, rope_scaling_type=self.context_params.rope_scaling_type, pooling_type=self.context_params.pooling_type, + attention_type=self.context_params.attention_type, rope_freq_base=self.context_params.rope_freq_base, rope_freq_scale=self.context_params.rope_freq_scale, yarn_ext_factor=self.context_params.yarn_ext_factor, From d6f46a50d6b4cda10460c05e2acdbaec74428c1b Mon Sep 17 00:00:00 2001 From: Andrei Date: Tue, 24 Mar 2026 02:56:01 -0700 Subject: [PATCH 448/561] chore: bump version (#2157) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de4f070ffc..4118f48485 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.18] + - feat: Expose `attention_type` in `Llama.__init__` for non-causal embedding models by @jamesbiederbeck in #2143 - fix(ci): Build Docker images from the checked-out source and sanitize branch tags by @abetlen in #2156 - fix(ci): Fix the CUDA wheel workflow and keep release tags aligned with the built toolkit by @abetlen in #2155 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index a7c40478bc..bdaefb9e01 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.17" +__version__ = "0.3.18" From 5f9c231ce165126f38c8897fd760ecd7ef79f9fd Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 25 Mar 2026 01:56:18 -0700 Subject: [PATCH 449/561] fix(ci): reduce CUDA binary wheel size only including cubins for current arches and one PTX target for forward compatibility (#2158) * fix(ci): shrink CUDA wheel fatbins * docs: update changelog for cuda wheel size fix --- .github/workflows/build-wheels-cuda.yaml | 7 ++++--- CHANGELOG.md | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index b8d6c9dcec..17daaa12a2 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -153,9 +153,10 @@ jobs: } $cudaTagVersion = $nvccVersion.Replace('.','') $env:VERBOSE = '1' - # Keep a portable SM set, including sm_70, instead of CMake's `all`, - # which now pulls in future targets the hosted-runner toolchains cannot assemble. - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70;75;80;86;89;90 -DCMAKE_CUDA_FLAGS=--allow-unsupported-compiler $env:CMAKE_ARGS" + # Build real cubins for the supported GPUs, including sm_70, and keep + # one forward-compatible PTX target instead of embedding PTX for every + # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit. + $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=--allow-unsupported-compiler $env:CMAKE_ARGS" # if ($env:AVXVER -eq 'AVX') { $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' # } diff --git a/CHANGELOG.md b/CHANGELOG.md index 4118f48485..f4a0b55d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(ci): Shrink CUDA wheel fatbins so CUDA releases stay under GitHub's asset size limit by @abetlen in #2158 + ## [0.3.18] - feat: Expose `attention_type` in `Llama.__init__` for non-causal embedding models by @jamesbiederbeck in #2143 From ac59e5a5ae8d331d80f30d3ddfc50195061637f5 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 25 Mar 2026 15:03:57 -0700 Subject: [PATCH 450/561] fix: handle embedding models without KV memory (#2160) * Fix embedding models without KV memory * Add changelog entry for embedding memory fix --- CHANGELOG.md | 1 + llama_cpp/_internals.py | 4 +++- tests/test_llama.py | 16 ++++++++++++---- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4a0b55d33..d2e4937c9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: Handle embedding models without KV memory and test embeddings with a real GGUF embedding model by @abetlen in #2160 - fix(ci): Shrink CUDA wheel fatbins so CUDA releases stay under GitHub's asset size limit by @abetlen in #2158 ## [0.3.18] diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 6862135aa7..9e9bcd407a 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -288,7 +288,9 @@ def pooling_type(self) -> int: return llama_cpp.llama_pooling_type(self.ctx) def kv_cache_clear(self): - assert self.memory is not None, "Memory is not initialized" + # Embedding models with non-causal attention may not allocate memory. + if self.memory is None: + return llama_cpp.llama_memory_clear(self.memory, True) def kv_cache_seq_rm(self, seq_id: int, p0: int, p1: int) -> bool: diff --git a/tests/test_llama.py b/tests/test_llama.py index 1a70c74d43..23928fff66 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -64,6 +64,14 @@ def llama_cpp_model_path(): return model_path +@pytest.fixture +def llama_cpp_embedding_model_path(): + repo_id = "CompendiumLabs/bge-small-en-v1.5-gguf" + filename = "bge-small-en-v1.5-q4_k_m.gguf" + model_path = hf_hub_download(repo_id, filename) + return model_path + + def test_real_model(llama_cpp_model_path): import os @@ -225,9 +233,9 @@ def logit_processor_func(input_ids, logits): assert number_1 == number_3 -def test_real_llama_embeddings(llama_cpp_model_path): +def test_real_llama_embeddings(llama_cpp_embedding_model_path): model = llama_cpp.Llama( - llama_cpp_model_path, + llama_cpp_embedding_model_path, n_ctx=32, n_batch=32, n_ubatch=32, @@ -237,5 +245,5 @@ def test_real_llama_embeddings(llama_cpp_model_path): flash_attn=True, embedding=True, ) - # Smoke test for now - model.embed("Hello World") + embedding = model.embed("Hello World") + assert len(embedding) > 0 From c670222c8379608aa22ac81e5f6a813620187a26 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 25 Mar 2026 15:21:44 -0700 Subject: [PATCH 451/561] feat: Update llama.cpp to ggerganov/llama.cpp@c0159f9c1f874da15e94f371d136f5920b4b5335 (#2161) * Update llama.cpp to c0159f9c1 * Add changelog entry for llama.cpp update --- CHANGELOG.md | 1 + llama_cpp/llama_cpp.py | 16 ++++++++++++++++ vendor/llama.cpp | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2e4937c9c..03240454f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: Update llama.cpp to ggerganov/llama.cpp@c0159f9c1f874da15e94f371d136f5920b4b5335 by @abetlen in #2161 - fix: Handle embedding models without KV memory and test embeddings with a real GGUF embedding model by @abetlen in #2160 - fix(ci): Shrink CUDA wheel fatbins so CUDA releases stay under GitHub's asset size limit by @abetlen in #2158 diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index e51492c56e..5a6c06b077 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1314,6 +1314,22 @@ def llama_model_load_from_splits( ... +# // Load a model from an open FILE pointer +# LLAMA_API struct llama_model * llama_model_load_from_file_ptr( +# FILE * file, +# struct llama_model_params params); +@ctypes_function( + "llama_model_load_from_file_ptr", + [ctypes.c_void_p, llama_model_params], + llama_model_p_ctypes, +) +def llama_model_load_from_file_ptr( + file: ctypes.c_void_p, params: llama_model_params, / +) -> Optional[llama_model_p]: + """Load a model from an open FILE pointer.""" + ... + + # LLAMA_API void llama_model_save_to_file( # const struct llama_model * model, # const char * path_model); diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 49bfddeca1..c0159f9c1f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 49bfddeca18e62fa3d39114a23e9fcbdf8a22388 +Subproject commit c0159f9c1f874da15e94f371d136f5920b4b5335 From f54421ba71db942b262a28762bc9e035a5d4d349 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 25 Mar 2026 15:28:12 -0700 Subject: [PATCH 452/561] Bump version to 0.3.19 (#2162) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03240454f6..d1efdc5e95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.19] + - feat: Update llama.cpp to ggerganov/llama.cpp@c0159f9c1f874da15e94f371d136f5920b4b5335 by @abetlen in #2161 - fix: Handle embedding models without KV memory and test embeddings with a real GGUF embedding model by @abetlen in #2160 - fix(ci): Shrink CUDA wheel fatbins so CUDA releases stay under GitHub's asset size limit by @abetlen in #2158 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index bdaefb9e01..72388c4e5d 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.18" +__version__ = "0.3.19" From fcd932a1574b7b6fdfc6c2d652f10f3af66995b5 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 28 Mar 2026 21:36:13 -0700 Subject: [PATCH 453/561] fix(ci): publish distinct manylinux and musllinux cpu wheels (#2165) * fix(ci): publish distinct manylinux and musllinux cpu wheels * docs: add changelog entry for linux wheel repair fix --- .github/workflows/build-and-release.yaml | 5 ++++- CHANGELOG.md | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 3a9e6f3698..6cbac0cb1c 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -44,8 +44,11 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: - # disable repair + # Keep repair disabled by default for non-Linux platforms in this job. CIBW_REPAIR_WHEEL_COMMAND: "" + # Linux needs auditwheel repair so manylinux and musllinux wheels are + # published with distinct platform tags instead of generic linux tags. + CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair -w {dest_dir} {wheel}" # Skip cibuildwheel's default i686 sidecar and keep Linux release # wheels on a portable x86_64 CPU baseline. CIBW_ARCHS_LINUX: "auto64" diff --git a/CHANGELOG.md b/CHANGELOG.md index d1efdc5e95..7bcad0a473 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(ci): Publish distinct manylinux and musllinux CPU wheels by @abetlen in #2165 + ## [0.3.19] - feat: Update llama.cpp to ggerganov/llama.cpp@c0159f9c1f874da15e94f371d136f5920b4b5335 by @abetlen in #2161 From 7613aca61259820ab550626384af52eed56a731f Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 29 Mar 2026 00:31:03 -0700 Subject: [PATCH 454/561] ci: publish release wheels as py3-none (#2166) * ci: publish CPU wheels as py3-none * docs: add changelog entry for py3-none wheel tags --- CHANGELOG.md | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bcad0a473..3b4c13ee34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- ci: Publish release wheels as `py3-none` by @Bing-su in #2166 - fix(ci): Publish distinct manylinux and musllinux CPU wheels by @abetlen in #2165 ## [0.3.19] diff --git a/pyproject.toml b/pyproject.toml index e0b0dc520c..b5998dd1c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ all = [ [tool.scikit-build] wheel.packages = ["llama_cpp"] +wheel.py-api = "py3" cmake.verbose = true cmake.minimum-version = "3.21" minimum-version = "0.5.1" From 7257ba95fbbf65201fd5bf4b7f0bdd1c701e1345 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 29 Mar 2026 23:17:22 -0700 Subject: [PATCH 455/561] feat(server): add model-load chat_template_kwargs (#2168) --- CHANGELOG.md | 1 + docs/server.md | 25 +++++++++++++++++++++++++ llama_cpp/llama_chat_format.py | 7 ++++++- llama_cpp/server/cli.py | 34 ++++++++++++++++++++++++++++++++-- llama_cpp/server/model.py | 15 +++++++++++++++ llama_cpp/server/settings.py | 6 +++++- 6 files changed, 84 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b4c13ee34..e577324db5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat(server): Add model-load `chat_template_kwargs` support and document the CLI/config usage by @abetlen in #2168 - ci: Publish release wheels as `py3-none` by @Bing-su in #2166 - fix(ci): Publish distinct manylinux and musllinux CPU wheels by @abetlen in #2165 diff --git a/docs/server.md b/docs/server.md index cd6f86c513..9c09a1f1cf 100644 --- a/docs/server.md +++ b/docs/server.md @@ -22,6 +22,15 @@ The server can then be started by running the following command: python3 -m llama_cpp.server --model ``` +You can also pass chat-template kwargs at model load time from the CLI: + +```bash +python3 -m llama_cpp.server \ + --model \ + --chat_format chatml \ + --chat_template_kwargs '{"enable_thinking": true}' +``` + ### Server options For a full list of options, run: @@ -147,6 +156,22 @@ The server supports routing requests to multiple models based on the `model` par At the moment only a single model is loaded into memory at, the server will automatically load and unload models as needed. +For a single-model config, `chat_template_kwargs` can be set directly on the model entry: + +```json +{ + "models": [ + { + "model": "models/Qwen3.5-0.8B/qwen3.5-0.8b-q8_0.gguf", + "chat_format": "chatml", + "chat_template_kwargs": { + "enable_thinking": true + } + } + ] +} +``` + ```json { "host": "0.0.0.0", diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index d7910e9847..1024fb85b9 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -243,6 +243,7 @@ def raise_exception(message: str): tools=tools, tool_choice=tool_choice, strftime_now=self.strftime_now, + **kwargs, ) stopping_criteria = None @@ -617,6 +618,7 @@ def chat_completion_handler( function_call=function_call, tools=tools, tool_choice=tool_choice, + **kwargs, ) prompt = llama.tokenize( result.prompt.encode("utf-8"), @@ -734,7 +736,9 @@ def format_autotokenizer( **kwargs: Any, ) -> ChatFormatterResponse: tokenizer.use_default_system_prompt = False # type: ignore - prompt: str = tokenizer.apply_chat_template(messages, tokenize=False) # type: ignore + prompt: str = tokenizer.apply_chat_template( # type: ignore + messages, tokenize=False, **kwargs + ) assert isinstance(prompt, str) # Return formatted prompt and eos token by default return ChatFormatterResponse( @@ -791,6 +795,7 @@ def format_tokenizer_config( messages=messages, bos_token=bos_token, eos_token=eos_token, + **kwargs, ) return ChatFormatterResponse( prompt=prompt, stop=[eos_token, bos_token], added_special=True diff --git a/llama_cpp/server/cli.py b/llama_cpp/server/cli.py index 8ed0290637..171b8db30c 100644 --- a/llama_cpp/server/cli.py +++ b/llama_cpp/server/cli.py @@ -1,8 +1,9 @@ from __future__ import annotations import argparse +import json -from typing import List, Literal, Union, Any, Type, TypeVar +from typing import List, Literal, Union, Any, Type, TypeVar, Dict from pydantic import BaseModel @@ -40,6 +41,17 @@ def _contains_list_type(annotation: Type[Any] | None) -> bool: return False +def _contains_dict_type(annotation: Type[Any] | None) -> bool: + origin = getattr(annotation, "__origin__", None) + + if origin is dict or origin is Dict: + return True + elif origin in (Literal, Union): + return any(_contains_dict_type(arg) for arg in annotation.__args__) # type: ignore + else: + return False + + def _parse_bool_arg(arg: str | bytes | bool) -> bool: if isinstance(arg, bytes): arg = arg.decode("utf-8") @@ -57,6 +69,16 @@ def _parse_bool_arg(arg: str | bytes | bool) -> bool: raise ValueError(f"Invalid boolean argument: {arg}") +def _parse_json_object_arg(arg: str | bytes) -> dict[str, Any]: + if isinstance(arg, bytes): + arg = arg.decode("utf-8") + + value = json.loads(arg) + if not isinstance(value, dict): + raise ValueError(f"Invalid JSON object argument: {arg}") + return value + + def add_args_from_model(parser: argparse.ArgumentParser, model: Type[BaseModel]): """Add arguments from a pydantic model to an argparse parser.""" @@ -68,7 +90,15 @@ def add_args_from_model(parser: argparse.ArgumentParser, model: Type[BaseModel]) _get_base_type(field.annotation) if field.annotation is not None else str ) list_type = _contains_list_type(field.annotation) - if base_type is not bool: + dict_type = _contains_dict_type(field.annotation) + if dict_type: + parser.add_argument( + f"--{name}", + dest=name, + type=_parse_json_object_arg, + help=description, + ) + elif base_type is not bool: parser.add_argument( f"--{name}", dest=name, diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 9e59e85633..3922ce5df3 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -299,6 +299,21 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: # Misc verbose=settings.verbose, ) + if settings.chat_template_kwargs: + base_chat_handler = ( + _model.chat_handler + or _model._chat_handlers.get(_model.chat_format) + or llama_cpp.llama_chat_format.get_chat_completion_handler( + _model.chat_format + ) + ) + + def chat_handler_with_kwargs(*args, **kwargs): + return base_chat_handler( + *args, **{**settings.chat_template_kwargs, **kwargs} + ) + + _model.chat_handler = chat_handler_with_kwargs if settings.cache: if settings.cache_type == "disk": if settings.verbose: diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 13c9512419..3c2bb7fd07 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -2,7 +2,7 @@ import multiprocessing -from typing import Optional, List, Literal, Union, Dict, cast +from typing import Any, Optional, List, Literal, Union, Dict, cast from typing_extensions import Self from pydantic import Field, model_validator @@ -131,6 +131,10 @@ class ModelSettings(BaseSettings): default=None, description="Chat format to use.", ) + chat_template_kwargs: Optional[Dict[str, Any]] = Field( + default=None, + description="Extra keyword arguments forwarded to chat templates at model load time. Matches llama.cpp server `chat_template_kwargs`.", + ) clip_model_path: Optional[str] = Field( default=None, description="Path to a CLIP model to use for multi-modal chat completion.", From 100b275c88763e2fada18d009e9bca8287cdb7c9 Mon Sep 17 00:00:00 2001 From: Andrei Date: Thu, 2 Apr 2026 22:44:24 -0700 Subject: [PATCH 456/561] feat: Update llama.cpp to ggerganov/llama.cpp@f49e9178767d557a522618b16ce8694f9ddac628 (#2169) --- llama_cpp/llama_cpp.py | 417 +++++++++++++++++++++++++++++++++++++++-- llama_cpp/mtmd_cpp.py | 26 ++- vendor/llama.cpp | 2 +- 3 files changed, 423 insertions(+), 22 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 5a6c06b077..204f1e6262 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -45,6 +45,14 @@ ctypes_function = ctypes_function_for_shared_library(_lib) +def _warn_deprecated(symbol: str, hint: str) -> None: + warnings.warn( + f"{symbol} is deprecated; {hint}", + DeprecationWarning, + stacklevel=2, + ) + + # from ggml.h # // NOTE: always add types at the end of the enum to keep backward compatibility # enum ggml_type { @@ -711,6 +719,43 @@ class llama_model_kv_override(ctypes.Structure): value: Union[int, float, bool, bytes] +# struct llama_model_tensor_override { +# const char * pattern; +# enum ggml_type type; +# }; +class llama_model_tensor_override(ctypes.Structure): + """Override the quantization type for tensors matching a pattern.""" + + _fields_ = [ + ("pattern", ctypes.c_char_p), + ("type", ctypes.c_int), + ] + + if TYPE_CHECKING: + pattern: Optional[bytes] + type: int + + +# struct llama_model_imatrix_data { +# const char * name; +# const float * data; +# size_t size; +# }; +class llama_model_imatrix_data(ctypes.Structure): + """Importance matrix data for a tensor used during quantization.""" + + _fields_ = [ + ("name", ctypes.c_char_p), + ("data", ctypes.POINTER(ctypes.c_float)), + ("size", ctypes.c_size_t), + ] + + if TYPE_CHECKING: + name: Optional[bytes] + data: CtypesPointer[ctypes.c_float] + size: int + + # struct llama_model_tensor_buft_override { # const char * pattern; # ggml_backend_buffer_type_t buft; @@ -1022,10 +1067,10 @@ class llama_context_params(ctypes.Structure): # bool pure; // quantize all tensors to the default type # bool keep_split; // quantize to the same number of shards # bool dry_run; // calculate and show the final quantization size without performing quantization -# void * imatrix; // pointer to importance matrix data -# void * kv_overrides; // pointer to vector containing overrides -# void * tensor_types; // pointer to vector containing tensor types -# void * prune_layers; // pointer to vector containing layer indices to prune +# const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data +# const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides +# const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides +# const int32_t * prune_layers; // pointer to layer indices to prune # } llama_model_quantize_params; class llama_model_quantize_params(ctypes.Structure): """Parameters for llama_model_quantize @@ -1041,10 +1086,10 @@ class llama_model_quantize_params(ctypes.Structure): pure (bool): quantize all tensors to the default type keep_split (bool): quantize to the same number of shards dry_run (bool): calculate and show the final quantization size without performing quantization - imatrix (ctypes.c_void_p): pointer to importance matrix data - kv_overrides (ctypes.c_void_p): pointer to vector containing overrides - tensor_types (ctypes.c_void_p): pointer to vector containing tensor types - prune_layers (ctypes.c_void_p): pointer to vector containing layer indices to prune + imatrix (ctypes.Array[llama_model_imatrix_data]): pointer to importance matrix data + kv_overrides (ctypes.Array[llama_model_kv_override]): pointer to kv overrides + tt_overrides (ctypes.Array[llama_model_tensor_override]): pointer to tensor overrides + prune_layers (ctypes.Array[ctypes.c_int32]): pointer to layer indices to prune """ if TYPE_CHECKING: @@ -1058,10 +1103,10 @@ class llama_model_quantize_params(ctypes.Structure): pure: bool keep_split: bool dry_run: bool - imatrix: ctypes.c_void_p - kv_overrides: ctypes.c_void_p - tensor_types: ctypes.c_void_p - prune_layers: ctypes.c_void_p + imatrix: CtypesPointer[llama_model_imatrix_data] + kv_overrides: CtypesPointer[llama_model_kv_override] + tt_overrides: CtypesPointer[llama_model_tensor_override] + prune_layers: CtypesPointer[ctypes.c_int32] _fields_ = [ ("nthread", ctypes.c_int32), @@ -1074,10 +1119,10 @@ class llama_model_quantize_params(ctypes.Structure): ("pure", ctypes.c_bool), ("keep_split", ctypes.c_bool), ("dry_run", ctypes.c_bool), - ("imatrix", ctypes.c_void_p), - ("kv_overrides", ctypes.c_void_p), - ("tensor_types", ctypes.c_void_p), - ("prune_layers", ctypes.c_void_p), + ("imatrix", ctypes.POINTER(llama_model_imatrix_data)), + ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), + ("tt_overrides", ctypes.POINTER(llama_model_tensor_override)), + ("prune_layers", ctypes.POINTER(ctypes.c_int32)), ] @@ -1272,6 +1317,19 @@ def llama_load_model_from_file( ) -> Optional[llama_model_p]: ... +_llama_load_model_from_file = llama_load_model_from_file + + +def llama_load_model_from_file( + path_model: bytes, params: llama_model_params, / +) -> Optional[llama_model_p]: + _warn_deprecated( + "llama_load_model_from_file", + "use llama_model_load_from_file instead", + ) + return _llama_load_model_from_file(path_model, params) + + # // Load the model from a file # // If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf # // If the split file name does not follow this pattern, use llama_model_load_from_splits @@ -1353,6 +1411,14 @@ def llama_model_save_to_file(model: llama_model_p, path_model: bytes, /): def llama_free_model(model: llama_model_p, /): ... +_llama_free_model = llama_free_model + + +def llama_free_model(model: llama_model_p, /): + _warn_deprecated("llama_free_model", "use llama_model_free instead") + return _llama_free_model(model) + + # LLAMA_API void llama_model_free(struct llama_model * model); @ctypes_function( "llama_model_free", @@ -1419,6 +1485,19 @@ def llama_new_context_with_model( ) -> Optional[llama_context_p]: ... +_llama_new_context_with_model = llama_new_context_with_model + + +def llama_new_context_with_model( + model: llama_model_p, params: llama_context_params, / +) -> Optional[llama_context_p]: + _warn_deprecated( + "llama_new_context_with_model", + "use llama_init_from_model instead", + ) + return _llama_new_context_with_model(model, params) + + # // Frees all allocated memory # LLAMA_API void llama_free(struct llama_context * ctx); @ctypes_function( @@ -1557,26 +1636,66 @@ def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... def llama_n_ctx_train(model: llama_model_p, /) -> int: ... +_llama_n_ctx_train = llama_n_ctx_train + + +def llama_n_ctx_train(model: llama_model_p, /) -> int: + _warn_deprecated("llama_n_ctx_train", "use llama_model_n_ctx_train instead") + return _llama_n_ctx_train(model) + + # DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); @ctypes_function("llama_n_embd", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_embd(model: llama_model_p, /) -> int: ... +_llama_n_embd = llama_n_embd + + +def llama_n_embd(model: llama_model_p, /) -> int: + _warn_deprecated("llama_n_embd", "use llama_model_n_embd instead") + return _llama_n_embd(model) + + # DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); @ctypes_function("llama_n_layer", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_layer(model: llama_model_p, /) -> int: ... +_llama_n_layer = llama_n_layer + + +def llama_n_layer(model: llama_model_p, /) -> int: + _warn_deprecated("llama_n_layer", "use llama_model_n_layer instead") + return _llama_n_layer(model) + + # DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); @ctypes_function("llama_n_head", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_head(model: llama_model_p, /) -> int: ... +_llama_n_head = llama_n_head + + +def llama_n_head(model: llama_model_p, /) -> int: + _warn_deprecated("llama_n_head", "use llama_model_n_head instead") + return _llama_n_head(model) + + # DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); @ctypes_function("llama_n_vocab", [llama_vocab_p_ctypes], ctypes.c_int32) def llama_n_vocab(model: llama_vocab_p, /) -> int: ... +_llama_n_vocab = llama_n_vocab + + +def llama_n_vocab(model: llama_vocab_p, /) -> int: + _warn_deprecated("llama_n_vocab", "use llama_vocab_n_tokens instead") + return _llama_n_vocab(model) + + # LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); @ctypes_function("llama_get_model", [llama_context_p_ctypes], llama_model_p_ctypes) def llama_get_model(ctx: llama_context_p, /) -> Optional[llama_model_p]: ... @@ -2381,6 +2500,14 @@ def llama_get_state_size(ctx: llama_context_p, /) -> int: ... +_llama_get_state_size = llama_get_state_size + + +def llama_get_state_size(ctx: llama_context_p, /) -> int: + _warn_deprecated("llama_get_state_size", "use llama_state_get_size instead") + return _llama_get_state_size(ctx) + + # // Copies the state to the specified destination address. # // Destination needs to have allocated enough memory. # // Returns the number of bytes copied @@ -2428,6 +2555,16 @@ def llama_copy_state_data( ... +_llama_copy_state_data = llama_copy_state_data + + +def llama_copy_state_data( + ctx: llama_context_p, dst: CtypesArray[ctypes.c_uint8], / +) -> int: + _warn_deprecated("llama_copy_state_data", "use llama_state_get_data instead") + return _llama_copy_state_data(ctx, dst) + + # // Set the state reading from the specified address # // Returns the number of bytes read # LLAMA_API size_t llama_state_set_data( @@ -2466,6 +2603,16 @@ def llama_set_state_data( ... +_llama_set_state_data = llama_set_state_data + + +def llama_set_state_data( + ctx: llama_context_p, src: CtypesArray[ctypes.c_uint8], / +) -> int: + _warn_deprecated("llama_set_state_data", "use llama_state_set_data instead") + return _llama_set_state_data(ctx, src) + + # Save/load session file # LLAMA_API bool llama_state_load_file( # struct llama_context * ctx, @@ -2522,6 +2669,23 @@ def llama_load_session_file( ) -> bool: ... +_llama_load_session_file = llama_load_session_file + + +def llama_load_session_file( + ctx: llama_context_p, + path_session: bytes, + tokens_out: CtypesArray[llama_token], + n_token_capacity: Union[ctypes.c_size_t, int], + n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], + /, +) -> bool: + _warn_deprecated("llama_load_session_file", "use llama_state_load_file instead") + return _llama_load_session_file( + ctx, path_session, tokens_out, n_token_capacity, n_token_count_out + ) + + # LLAMA_API bool llama_state_save_file( # struct llama_context * ctx, # const char * path_session, @@ -2571,6 +2735,20 @@ def llama_save_session_file( ) -> bool: ... +_llama_save_session_file = llama_save_session_file + + +def llama_save_session_file( + ctx: llama_context_p, + path_session: bytes, + tokens: CtypesArray[llama_token], + n_token_count: Union[ctypes.c_size_t, int], + /, +) -> bool: + _warn_deprecated("llama_save_session_file", "use llama_state_save_file instead") + return _llama_save_session_file(ctx, path_session, tokens, n_token_count) + + # // Get the exact size needed to copy the state of a single sequence # LLAMA_API size_t llama_state_seq_get_size( # struct llama_context * ctx, @@ -3300,6 +3478,16 @@ def llama_token_get_text( ) -> bytes: ... +_llama_token_get_text = llama_token_get_text + + +def llama_token_get_text( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> bytes: + _warn_deprecated("llama_token_get_text", "use llama_vocab_get_text instead") + return _llama_token_get_text(vocab, token) + + # DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); @ctypes_function( "llama_token_get_score", @@ -3311,6 +3499,16 @@ def llama_token_get_score( ) -> float: ... +_llama_token_get_score = llama_token_get_score + + +def llama_token_get_score( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> float: + _warn_deprecated("llama_token_get_score", "use llama_vocab_get_score instead") + return _llama_token_get_score(vocab, token) + + # DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); @ctypes_function( "llama_token_get_attr", @@ -3322,6 +3520,16 @@ def llama_token_get_attr( ) -> int: ... +_llama_token_get_attr = llama_token_get_attr + + +def llama_token_get_attr( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> int: + _warn_deprecated("llama_token_get_attr", "use llama_vocab_get_attr instead") + return _llama_token_get_attr(vocab, token) + + # DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); @ctypes_function( "llama_token_is_eog", @@ -3333,6 +3541,14 @@ def llama_token_is_eog( ) -> bool: ... +_llama_token_is_eog = llama_token_is_eog + + +def llama_token_is_eog(vocab: llama_vocab_p, token: Union[llama_token, int], /) -> bool: + _warn_deprecated("llama_token_is_eog", "use llama_vocab_is_eog instead") + return _llama_token_is_eog(vocab, token) + + # DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); @ctypes_function( "llama_token_is_control", @@ -3344,6 +3560,19 @@ def llama_token_is_control( ) -> bool: ... +_llama_token_is_control = llama_token_is_control + + +def llama_token_is_control( + vocab: llama_vocab_p, token: Union[llama_token, int], / +) -> bool: + _warn_deprecated( + "llama_token_is_control", + "use llama_vocab_is_control instead", + ) + return _llama_token_is_control(vocab, token) + + # DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); @ctypes_function( "llama_token_bos", @@ -3353,6 +3582,14 @@ def llama_token_is_control( def llama_token_bos(vocab: llama_vocab_p, /) -> int: ... +_llama_token_bos = llama_token_bos + + +def llama_token_bos(vocab: llama_vocab_p, /) -> int: + _warn_deprecated("llama_token_bos", "use llama_vocab_bos instead") + return _llama_token_bos(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); @ctypes_function( "llama_token_eos", @@ -3362,6 +3599,14 @@ def llama_token_bos(vocab: llama_vocab_p, /) -> int: ... def llama_token_eos(vocab: llama_vocab_p, /) -> int: ... +_llama_token_eos = llama_token_eos + + +def llama_token_eos(vocab: llama_vocab_p, /) -> int: + _warn_deprecated("llama_token_eos", "use llama_vocab_eos instead") + return _llama_token_eos(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); @ctypes_function( "llama_token_eot", @@ -3371,6 +3616,14 @@ def llama_token_eos(vocab: llama_vocab_p, /) -> int: ... def llama_token_eot(vocab: llama_vocab_p, /) -> int: ... +_llama_token_eot = llama_token_eot + + +def llama_token_eot(vocab: llama_vocab_p, /) -> int: + _warn_deprecated("llama_token_eot", "use llama_vocab_eot instead") + return _llama_token_eot(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); @ctypes_function( "llama_token_cls", @@ -3380,6 +3633,14 @@ def llama_token_eot(vocab: llama_vocab_p, /) -> int: ... def llama_token_cls(vocab: llama_vocab_p, /) -> int: ... +_llama_token_cls = llama_token_cls + + +def llama_token_cls(vocab: llama_vocab_p, /) -> int: + _warn_deprecated("llama_token_cls", "use llama_vocab_cls instead") + return _llama_token_cls(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); @ctypes_function( "llama_token_sep", @@ -3389,6 +3650,14 @@ def llama_token_cls(vocab: llama_vocab_p, /) -> int: ... def llama_token_sep(vocab: llama_vocab_p, /) -> int: ... +_llama_token_sep = llama_token_sep + + +def llama_token_sep(vocab: llama_vocab_p, /) -> int: + _warn_deprecated("llama_token_sep", "use llama_vocab_sep instead") + return _llama_token_sep(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); @ctypes_function( "llama_token_nl", @@ -3398,6 +3667,14 @@ def llama_token_sep(vocab: llama_vocab_p, /) -> int: ... def llama_token_nl(vocab: llama_vocab_p, /) -> int: ... +_llama_token_nl = llama_token_nl + + +def llama_token_nl(vocab: llama_vocab_p, /) -> int: + _warn_deprecated("llama_token_nl", "use llama_vocab_nl instead") + return _llama_token_nl(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); @ctypes_function( "llama_token_pad", @@ -3407,6 +3684,14 @@ def llama_token_nl(vocab: llama_vocab_p, /) -> int: ... def llama_token_pad(vocab: llama_vocab_p, /) -> int: ... +_llama_token_pad = llama_token_pad + + +def llama_token_pad(vocab: llama_vocab_p, /) -> int: + _warn_deprecated("llama_token_pad", "use llama_vocab_pad instead") + return _llama_token_pad(vocab) + + # DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); @ctypes_function( "llama_add_bos_token", @@ -3416,6 +3701,14 @@ def llama_token_pad(vocab: llama_vocab_p, /) -> int: ... def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: ... +_llama_add_bos_token = llama_add_bos_token + + +def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: + _warn_deprecated("llama_add_bos_token", "use llama_vocab_get_add_bos instead") + return _llama_add_bos_token(vocab) + + # DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); @ctypes_function( "llama_add_eos_token", @@ -3425,6 +3718,14 @@ def llama_add_bos_token(vocab: llama_vocab_p, /) -> bool: ... def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: ... +_llama_add_eos_token = llama_add_eos_token + + +def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: + _warn_deprecated("llama_add_eos_token", "use llama_vocab_get_add_eos instead") + return _llama_add_eos_token(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); @ctypes_function( "llama_token_fim_pre", @@ -3434,6 +3735,14 @@ def llama_add_eos_token(vocab: llama_vocab_p, /) -> bool: ... def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ... +_llama_token_fim_pre = llama_token_fim_pre + + +def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: + _warn_deprecated("llama_token_fim_pre", "use llama_vocab_fim_pre instead") + return _llama_token_fim_pre(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); @ctypes_function( "llama_token_fim_suf", @@ -3443,6 +3752,14 @@ def llama_token_fim_pre(vocab: llama_vocab_p, /) -> llama_token: ... def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ... +_llama_token_fim_suf = llama_token_fim_suf + + +def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: + _warn_deprecated("llama_token_fim_suf", "use llama_vocab_fim_suf instead") + return _llama_token_fim_suf(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); @ctypes_function( "llama_token_fim_mid", @@ -3452,6 +3769,14 @@ def llama_token_fim_suf(vocab: llama_vocab_p, /) -> llama_token: ... def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ... +_llama_token_fim_mid = llama_token_fim_mid + + +def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: + _warn_deprecated("llama_token_fim_mid", "use llama_vocab_fim_mid instead") + return _llama_token_fim_mid(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); @ctypes_function( "llama_token_fim_pad", @@ -3461,6 +3786,14 @@ def llama_token_fim_mid(vocab: llama_vocab_p, /) -> llama_token: ... def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ... +_llama_token_fim_pad = llama_token_fim_pad + + +def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: + _warn_deprecated("llama_token_fim_pad", "use llama_vocab_fim_pad instead") + return _llama_token_fim_pad(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); @ctypes_function( "llama_token_fim_rep", @@ -3470,6 +3803,14 @@ def llama_token_fim_pad(vocab: llama_vocab_p, /) -> llama_token: ... def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: ... +_llama_token_fim_rep = llama_token_fim_rep + + +def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: + _warn_deprecated("llama_token_fim_rep", "use llama_vocab_fim_rep instead") + return _llama_token_fim_rep(vocab) + + # DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); @ctypes_function( "llama_token_fim_sep", @@ -3479,6 +3820,14 @@ def llama_token_fim_rep(vocab: llama_vocab_p, /) -> llama_token: ... def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ... +_llama_token_fim_sep = llama_token_fim_sep + + +def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: + _warn_deprecated("llama_token_fim_sep", "use llama_vocab_fim_sep instead") + return _llama_token_fim_sep(vocab) + + # // CLS is equivalent to BOS # DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification # "use llama_vocab_bos instead"); @@ -3490,6 +3839,14 @@ def llama_token_fim_sep(vocab: llama_vocab_p, /) -> llama_token: ... def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: ... +_llama_vocab_cls = llama_vocab_cls + + +def llama_vocab_cls(vocab: llama_vocab_p, /) -> llama_token: + _warn_deprecated("llama_vocab_cls", "use llama_vocab_bos instead") + return _llama_vocab_cls(vocab) + + # // # // Tokenization # // @@ -4146,6 +4503,34 @@ def llama_sampler_init_grammar_lazy( ) -> llama_sampler_p: ... +_llama_sampler_init_grammar_lazy = llama_sampler_init_grammar_lazy + + +def llama_sampler_init_grammar_lazy( + vocab: llama_vocab_p, + grammar_str: bytes, + grammar_root: bytes, + trigger_words: CtypesArray[bytes], + num_trigger_words: int, + trigger_tokens: CtypesArray[llama_token], + num_trigger_tokens: int, + /, +) -> llama_sampler_p: + _warn_deprecated( + "llama_sampler_init_grammar_lazy", + "use llama_sampler_init_grammar_lazy_patterns instead", + ) + return _llama_sampler_init_grammar_lazy( + vocab, + grammar_str, + grammar_root, + trigger_words, + num_trigger_words, + trigger_tokens, + num_trigger_tokens, + ) + + # /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 # LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( # const struct llama_vocab * vocab, diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 787683179c..f28402775c 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -79,6 +79,12 @@ # Structures class mtmd_context_params(Structure): + """Context parameters for MTMD initialization. + + `image_marker` is deprecated upstream and kept for compatibility; use + `media_marker` for multimodal prompt placeholders. + """ + if TYPE_CHECKING: use_gpu: bool print_timings: bool @@ -108,6 +114,8 @@ class mtmd_context_params(Structure): class mtmd_input_text(Structure): + """Text input passed to `mtmd_tokenize`.""" + _fields_ = [ ("text", c_char_p), ("add_special", c_bool), @@ -122,12 +130,16 @@ class mtmd_input_text(Structure): # MTMD_API const char * mtmd_default_marker(void); @ctypes_function("mtmd_default_marker", [], c_char_p) -def mtmd_default_marker() -> bytes: ... +def mtmd_default_marker() -> bytes: + """Return the default media marker.""" + ... # MTMD_API struct mtmd_context_params mtmd_context_params_default(void); @ctypes_function("mtmd_context_params_default", [], mtmd_context_params) -def mtmd_context_params_default() -> mtmd_context_params: ... +def mtmd_context_params_default() -> mtmd_context_params: + """Return the default MTMD context parameters.""" + ... # MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname, @@ -143,7 +155,9 @@ def mtmd_init_from_file( text_model: llama_cpp.llama_model_p, ctx_params: mtmd_context_params, /, -) -> Optional[mtmd_context_p]: ... +) -> Optional[mtmd_context_p]: + """Initialize the MTMD context from a projector file. Returns None on failure.""" + ... # MTMD_API void mtmd_free(mtmd_context * ctx); @@ -167,7 +181,9 @@ def mtmd_decode_use_mrope(ctx: mtmd_context_p, /) -> bool: # MTMD_API bool mtmd_support_vision(mtmd_context * ctx); @ctypes_function("mtmd_support_vision", [mtmd_context_p_ctypes], c_bool) -def mtmd_support_vision(ctx: mtmd_context_p, /) -> bool: ... +def mtmd_support_vision(ctx: mtmd_context_p, /) -> bool: + """Check whether the current model supports vision input.""" + ... # MTMD_API bool mtmd_support_audio(mtmd_context * ctx); @@ -180,7 +196,7 @@ def mtmd_support_audio(ctx: mtmd_context_p, /) -> bool: # MTMD_API int mtmd_get_audio_sample_rate(mtmd_context * ctx); @ctypes_function("mtmd_get_audio_sample_rate", [mtmd_context_p_ctypes], c_int) def mtmd_get_audio_sample_rate(ctx: mtmd_context_p, /) -> int: - """Get the MTMD audio sample rate.""" + """Get the audio sample rate in Hz. Returns -1 if audio is not supported.""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c0159f9c1f..f49e917876 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c0159f9c1f874da15e94f371d136f5920b4b5335 +Subproject commit f49e9178767d557a522618b16ce8694f9ddac628 From 08e088cde15e0b56e46337d6f49000518c18c203 Mon Sep 17 00:00:00 2001 From: Andrei Date: Thu, 2 Apr 2026 23:22:02 -0700 Subject: [PATCH 457/561] fix(misc): replace deprecated llama.cpp references (#2170) * refactor: replace deprecated llama.cpp references * docs: update changelog for recent llama.cpp changes --- CHANGELOG.md | 2 ++ README.md | 16 ++++++---- examples/batch-processing/server.py | 4 +-- .../low_level_api/low_level_api_chat_cpp.py | 31 ++++++++++--------- .../low_level_api/low_level_api_llama_cpp.py | 26 +++++++++------- examples/notebooks/Batching.ipynb | 15 ++++----- llama_cpp/_internals.py | 6 ++-- llama_cpp/llama.py | 11 ++++--- 8 files changed, 63 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e577324db5..f5f3677e5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- refactor: Replace deprecated llama.cpp references in library, docs, and examples by @abetlen in #2170 +- feat: Update llama.cpp to ggerganov/llama.cpp@f49e9178767d557a522618b16ce8694f9ddac628 by @abetlen in #2169 - feat(server): Add model-load `chat_template_kwargs` support and document the CLI/config usage by @abetlen in #2168 - ci: Publish release wheels as `py3-none` by @Bing-su in #2166 - fix(ci): Publish distinct manylinux and musllinux CPU wheels by @abetlen in #2165 diff --git a/README.md b/README.md index 8ba4dbb5ea..69a0f82345 100644 --- a/README.md +++ b/README.md @@ -717,16 +717,20 @@ Below is a short example demonstrating how to use the low-level API to tokenize ```python import llama_cpp import ctypes -llama_cpp.llama_backend_init(False) # Must be called once at the start of each program -params = llama_cpp.llama_context_default_params() +llama_cpp.llama_backend_init() # Must be called once at the start of each program +model_params = llama_cpp.llama_model_default_params() +ctx_params = llama_cpp.llama_context_default_params() +prompt = b"Q: Name the planets in the solar system? A: " # use bytes for char * params -model = llama_cpp.llama_load_model_from_file(b"./models/7b/llama-model.gguf", params) -ctx = llama_cpp.llama_new_context_with_model(model, params) -max_tokens = params.n_ctx +model = llama_cpp.llama_model_load_from_file(b"./models/7b/llama-model.gguf", model_params) +ctx = llama_cpp.llama_init_from_model(model, ctx_params) +vocab = llama_cpp.llama_model_get_vocab(model) +max_tokens = ctx_params.n_ctx # use ctypes arrays for array params tokens = (llama_cpp.llama_token * int(max_tokens))() -n_tokens = llama_cpp.llama_tokenize(ctx, b"Q: Name the planets in the solar system? A: ", tokens, max_tokens, llama_cpp.c_bool(True)) +n_tokens = llama_cpp.llama_tokenize(vocab, prompt, len(prompt), tokens, max_tokens, True, False) llama_cpp.llama_free(ctx) +llama_cpp.llama_model_free(model) ``` Check out the [examples folder](examples/low_level_api) for more examples of using the low-level API. diff --git a/examples/batch-processing/server.py b/examples/batch-processing/server.py index 0b36746f91..2b6fa759e0 100644 --- a/examples/batch-processing/server.py +++ b/examples/batch-processing/server.py @@ -6,14 +6,14 @@ # path = b"../../models/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf" # model_params = llama_cpp.llama_model_default_params() -# model = llama_cpp.llama_load_model_from_file(path, model_params) +# model = llama_cpp.llama_model_load_from_file(path, model_params) # if model is None: # raise RuntimeError(f"Failed to load model from file: {path}") # ctx_params = llama_cpp.llama_context_default_params() -# ctx = llama_cpp.llama_new_context_with_model(model, ctx_params) +# ctx = llama_cpp.llama_init_from_model(model, ctx_params) # if ctx is None: # raise RuntimeError("Failed to create context") diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 39081be17a..20f7a158ac 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -79,19 +79,22 @@ def __init__(self, params: GptParams) -> None: self.lparams.use_mlock = self.params.use_mlock self.lparams.use_mmap = self.params.use_mmap - self.model = llama_cpp.llama_load_model_from_file( + self.model = llama_cpp.llama_model_load_from_file( self.params.model.encode("utf8"), self.lparams ) + self.vocab = llama_cpp.llama_model_get_vocab(self.model) # Context Params. self.cparams = llama_cpp.llama_context_default_params() - self.ctx = llama_cpp.llama_new_context_with_model(self.model, self.cparams) + self.ctx = llama_cpp.llama_init_from_model(self.model, self.cparams) if not self.ctx: raise RuntimeError(f"error: failed to load model '{self.params.model}'") if self.params.ignore_eos: - self.params.logit_bias[llama_cpp.llama_token_eos()] = -float("inf") + self.params.logit_bias[llama_cpp.llama_vocab_eos(self.vocab)] = -float( + "inf" + ) if len(self.params.lora_adapter) > 0: if ( @@ -153,7 +156,7 @@ def __init__(self, params: GptParams) -> None: _session_tokens = (llama_cpp.llama_token * (self.params.n_ctx))() _n_token_count_out = llama_cpp.c_size_t() if ( - llama_cpp.llama_load_session_file( + llama_cpp.llama_state_load_file( self.ctx, self.params.path_session.encode("utf8"), _session_tokens, @@ -314,7 +317,7 @@ def __init__(self, params: GptParams) -> None: def _tokenize(self, prompt, bos=True): _arr = (llama_cpp.llama_token * ((len(prompt) + 1) * 4))() _n = llama_cpp.llama_tokenize( - self.model, + self.vocab, prompt.encode("utf8", errors="ignore"), len(prompt), _arr, @@ -406,7 +409,7 @@ def generate(self): if len(self.embd_inp) <= self.input_consumed: # && !is_interacting # out of user input, sample next token top_k = ( - llama_cpp.llama_n_vocab(self.ctx) + llama_cpp.llama_vocab_n_tokens(self.vocab) if self.params.top_k <= 0 else self.params.top_k ) @@ -419,7 +422,7 @@ def generate(self): # optionally save the session on first sample (for faster prompt loading next time) if len(self.params.path_session) > 0 and self.need_to_save_session: self.need_to_save_session = False - llama_cpp.llama_save_session_file( + llama_cpp.llama_state_save_file( self.ctx, self.params.path_session.encode("utf8"), (llama_cpp.llama_token * len(self.session_tokens))( @@ -431,7 +434,7 @@ def generate(self): id = 0 logits = llama_cpp.llama_get_logits(self.ctx) - n_vocab = llama_cpp.llama_n_vocab(self.model) + n_vocab = llama_cpp.llama_vocab_n_tokens(self.vocab) # Apply params.logit_bias map for key, value in self.params.logit_bias.items(): @@ -448,7 +451,7 @@ def generate(self): ) # Apply penalties - nl_logit = logits[llama_cpp.llama_token_nl(self.ctx)] + nl_logit = logits[llama_cpp.llama_vocab_nl(self.vocab)] last_n_repeat = min(len(self.last_n_tokens), repeat_last_n, self.n_ctx) _arr = (llama_cpp.llama_token * last_n_repeat)( @@ -470,7 +473,7 @@ def generate(self): # last_n_repeat, llama_cpp.c_float(self.params.frequency_penalty), llama_cpp.c_float(self.params.presence_penalty)) if not self.params.penalize_nl: - logits[llama_cpp.llama_token_nl()] = nl_logit + logits[llama_cpp.llama_vocab_nl(self.vocab)] = nl_logit if self.params.temp <= 0: # Greedy sampling @@ -539,7 +542,7 @@ def generate(self): # replace end of text token with newline token when in interactive mode if ( - id == llama_cpp.llama_token_eos(self.ctx) + id == llama_cpp.llama_vocab_eos(self.vocab) and self.params.interactive and not self.params.instruct ): @@ -599,8 +602,8 @@ def generate(self): break # end of text token - if len(self.embd) > 0 and self.embd[-1] == llama_cpp.llama_token_eos( - self.ctx + if len(self.embd) > 0 and self.embd[-1] == llama_cpp.llama_vocab_eos( + self.vocab ): if not self.params.instruct: for i in self.llama_token_eot: @@ -636,7 +639,7 @@ def token_to_str(self, token_id: int) -> bytes: size = 32 buffer = (ctypes.c_char * size)() n = llama_cpp.llama_token_to_piece( - self.model, llama_cpp.llama_token(token_id), buffer, size + self.vocab, llama_cpp.llama_token(token_id), buffer, size, 0, False ) assert n <= size return bytes(buffer[:n]) diff --git a/examples/low_level_api/low_level_api_llama_cpp.py b/examples/low_level_api/low_level_api_llama_cpp.py index ba3545771d..9fb3424ec0 100644 --- a/examples/low_level_api/low_level_api_llama_cpp.py +++ b/examples/low_level_api/low_level_api_llama_cpp.py @@ -4,7 +4,7 @@ import llama_cpp -llama_cpp.llama_backend_init(numa=False) +llama_cpp.llama_backend_init() N_THREADS = multiprocessing.cpu_count() MODEL_PATH = os.environ.get("MODEL", "../models/7B/ggml-model.bin") @@ -13,8 +13,9 @@ lparams = llama_cpp.llama_model_default_params() cparams = llama_cpp.llama_context_default_params() -model = llama_cpp.llama_load_model_from_file(MODEL_PATH.encode("utf-8"), lparams) -ctx = llama_cpp.llama_new_context_with_model(model, cparams) +model = llama_cpp.llama_model_load_from_file(MODEL_PATH.encode("utf-8"), lparams) +ctx = llama_cpp.llama_init_from_model(model, cparams) +vocab = llama_cpp.llama_model_get_vocab(model) # determine the required inference memory per token: tmp = [0, 1, 2, 3] @@ -28,13 +29,13 @@ embd_inp = (llama_cpp.llama_token * (len(prompt) + 1))() n_of_tok = llama_cpp.llama_tokenize( - model=model, - text=bytes(str(prompt), "utf-8"), - text_len=len(embd_inp), + vocab=vocab, + text=prompt, + text_len=len(prompt), tokens=embd_inp, - n_max_tokens=len(embd_inp), - add_bos=False, - special=False, + n_tokens_max=len(embd_inp), + add_special=False, + parse_special=False, ) embd_inp = embd_inp[:n_of_tok] @@ -70,7 +71,7 @@ embd = [] if len(embd_inp) <= input_consumed: logits = llama_cpp.llama_get_logits(ctx) - n_vocab = llama_cpp.llama_n_vocab(model) + n_vocab = llama_cpp.llama_vocab_n_tokens(vocab) _arr = (llama_cpp.llama_token_data * n_vocab)( *[ @@ -114,7 +115,7 @@ size = 32 buffer = (ctypes.c_char * size)() n = llama_cpp.llama_token_to_piece( - model, llama_cpp.llama_token(id), buffer, size + vocab, llama_cpp.llama_token(id), buffer, size, 0, False ) assert n <= size print( @@ -123,7 +124,7 @@ flush=True, ) - if len(embd) > 0 and embd[-1] == llama_cpp.llama_token_eos(ctx): + if len(embd) > 0 and embd[-1] == llama_cpp.llama_vocab_eos(vocab): break print() @@ -131,3 +132,4 @@ llama_cpp.llama_print_timings(ctx) llama_cpp.llama_free(ctx) +llama_cpp.llama_model_free(model) diff --git a/examples/notebooks/Batching.ipynb b/examples/notebooks/Batching.ipynb index be7fe9b522..0b36cd9c3a 100644 --- a/examples/notebooks/Batching.ipynb +++ b/examples/notebooks/Batching.ipynb @@ -122,9 +122,10 @@ "source": [ "params = llama_cpp.llama_model_default_params()\n", "params.n_gpu_layers = 35\n", - "model = llama_cpp.llama_load_model_from_file(\n", + "model = llama_cpp.llama_model_load_from_file(\n", " b\"/workspaces/llama-cpp-python/mistral-7b-v0.1.Q2_K.gguf\", params\n", - ") # Update this to whatever" + ") # Update this to whatever\n", + "vocab = llama_cpp.llama_model_get_vocab(model)" ] }, { @@ -149,7 +150,7 @@ "\n", "tokens = (llama_cpp.llama_token * n_ctx)()\n", "tokens_len = llama_cpp.llama_tokenize(\n", - " model, prompt, len(prompt), tokens, len(tokens), True, True\n", + " vocab, prompt, len(prompt), tokens, len(tokens), True, True\n", ")\n", "print(tokens[:tokens_len])\n", "\n", @@ -188,7 +189,7 @@ "ctx_params.n_batch = max(n_len, n_parallel)\n", "ctx_params.n_threads = 1\n", "ctx_params.n_threads_batch = 1\n", - "ctx = llama_cpp.llama_new_context_with_model(model, ctx_params)" + "ctx = llama_cpp.llama_init_from_model(model, ctx_params)" ] }, { @@ -338,14 +339,14 @@ " # Sample the next token using the sampler chain\n", " new_token_id = llama_cpp.llama_sampler_sample(sampler_chain, ctx, -1)\n", "\n", - " if new_token_id == llama_cpp.llama_token_eos(ctx) or n_cur == n_len:\n", + " if new_token_id == llama_cpp.llama_vocab_eos(vocab) or n_cur == n_len:\n", " i_batch[i] = -1\n", " continue\n", "\n", " buf = (ctypes.c_char * 32)()\n", " \n", " # Convert token ID to text\n", - " outlen = llama_cpp.llama_token_to_piece(model, new_token_id, buf, len(buf), 0, False)\n", + " outlen = llama_cpp.llama_token_to_piece(vocab, new_token_id, buf, len(buf), 0, False)\n", " streams[i] += bytes(buf[:outlen]).decode(\"utf-8\")\n", "\n", " batch.token[batch.n_tokens] = new_token_id\n", @@ -411,7 +412,7 @@ "metadata": {}, "outputs": [], "source": [ - "llama_cpp.llama_free_model(model)" + "llama_cpp.llama_model_free(model)" ] }, { diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 9e9bcd407a..cde52c8c8d 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -135,7 +135,7 @@ def token_eos(self) -> int: return llama_cpp.llama_vocab_eos(self.vocab) def token_cls(self) -> int: - return llama_cpp.llama_vocab_cls(self.vocab) + return llama_cpp.llama_vocab_bos(self.vocab) def token_sep(self) -> int: return llama_cpp.llama_vocab_sep(self.vocab) @@ -317,9 +317,9 @@ def get_state_size(self) -> int: # TODO: set_state_data - # TODO: llama_load_session_file + # TODO: llama_state_load_file - # TODO: llama_save_session_file + # TODO: llama_state_save_file def decode(self, batch: LlamaBatch): return_code = llama_cpp.llama_decode( diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index ad484c4d5f..11fe169cf5 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1351,7 +1351,7 @@ def logit_bias_processor( logits_processor=logits_processor, grammar=grammar, ): - if llama_cpp.llama_token_is_eog(self._model.vocab, token): + if llama_cpp.llama_vocab_is_eog(self._model.vocab, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) finish_reason = "stop" break @@ -2148,13 +2148,13 @@ def __setstate__(self, state): def save_state(self) -> LlamaState: if self.verbose: print("Llama.save_state: saving llama state", file=sys.stderr) - state_size = llama_cpp.llama_get_state_size(self._ctx.ctx) + state_size = llama_cpp.llama_state_get_size(self._ctx.ctx) if self.verbose: print(f"Llama.save_state: got state size: {state_size}", file=sys.stderr) llama_state = (ctypes.c_uint8 * int(state_size))() if self.verbose: print("Llama.save_state: allocated state", file=sys.stderr) - n_bytes = llama_cpp.llama_copy_state_data(self._ctx.ctx, llama_state) + n_bytes = llama_cpp.llama_state_get_data(self._ctx.ctx, llama_state, state_size) if self.verbose: print(f"Llama.save_state: copied llama state: {n_bytes}", file=sys.stderr) if int(n_bytes) > int(state_size): @@ -2187,7 +2187,10 @@ def load_state(self, state: LlamaState) -> None: LLamaStateArrayType = ctypes.c_uint8 * state_size llama_state = LLamaStateArrayType.from_buffer_copy(state.llama_state) - if llama_cpp.llama_set_state_data(self._ctx.ctx, llama_state) != state_size: + if ( + llama_cpp.llama_state_set_data(self._ctx.ctx, llama_state, state_size) + != state_size + ): raise RuntimeError("Failed to set llama state data") def n_ctx(self) -> int: From 02d6beed816535cb2ce845ad8558d0c8662f8e7e Mon Sep 17 00:00:00 2001 From: Andrei Date: Thu, 2 Apr 2026 23:42:35 -0700 Subject: [PATCH 458/561] chore: bump version to 0.3.20 (#2171) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5f3677e5d..9bc8d6654b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.20] + - refactor: Replace deprecated llama.cpp references in library, docs, and examples by @abetlen in #2170 - feat: Update llama.cpp to ggerganov/llama.cpp@f49e9178767d557a522618b16ce8694f9ddac628 by @abetlen in #2169 - feat(server): Add model-load `chat_template_kwargs` support and document the CLI/config usage by @abetlen in #2168 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 72388c4e5d..83177c065d 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.19" +__version__ = "0.3.20" From 1bcc5bc7bfd971cb39c00b0475ada778b6af4951 Mon Sep 17 00:00:00 2001 From: Andrei Date: Wed, 8 Apr 2026 01:59:51 -0700 Subject: [PATCH 459/561] feat: Update llama.cpp to ggerganov/llama.cpp@3bd9aa1f9 (#2176) * feat: Update llama.cpp to ggerganov/llama.cpp@3bd9aa1f9 * docs: Update changelog for llama.cpp bump --- CHANGELOG.md | 2 ++ llama_cpp/llama_cpp.py | 8 ++++++-- vendor/llama.cpp | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bc8d6654b..fbe5b6b6fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: Update llama.cpp to ggerganov/llama.cpp@3bd9aa1f9 and sync Python bindings + ## [0.3.20] - refactor: Replace deprecated llama.cpp references in library, docs, and examples by @abetlen in #2170 diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 204f1e6262..0a66a5d85d 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -88,7 +88,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # GGML_TYPE_IQ1_M = 29, # GGML_TYPE_MXFP4 = 39, # GGML_TYPE_NVFP4 = 40, -# GGML_TYPE_COUNT, +# GGML_TYPE_Q1_0 = 41, +# GGML_TYPE_COUNT = 42, # }; GGML_TYPE_F32 = 0 GGML_TYPE_F16 = 1 @@ -120,7 +121,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: GGML_TYPE_IQ1_M = 29 GGML_TYPE_MXFP4 = 39 GGML_TYPE_NVFP4 = 40 -GGML_TYPE_COUNT = 41 +GGML_TYPE_Q1_0 = 41 +GGML_TYPE_COUNT = 42 # from ggml-backend.h # typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); @@ -406,6 +408,7 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors # LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors # LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -446,6 +449,7 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 LLAMA_FTYPE_MOSTLY_NVFP4 = 39 +LLAMA_FTYPE_MOSTLY_Q1_0 = 40 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f49e917876..3bd9aa1f92 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f49e9178767d557a522618b16ce8694f9ddac628 +Subproject commit 3bd9aa1f9250cd15f5371f3622d73d954b68a747 From 1b1a320de18c14c3915ba2df59eedb7c6e7cbe69 Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 13 Apr 2026 02:56:37 -0400 Subject: [PATCH 460/561] feat: Update llama.cpp to ggerganov/llama.cpp@227ed28e1 (#2182) --- llama_cpp/llama_cpp.py | 8 +- llama_cpp/mtmd_cpp.py | 267 +++++++++++++++++++++++++++++++++++++++++ vendor/llama.cpp | 2 +- 3 files changed, 273 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 0a66a5d85d..e445ed66ae 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -503,13 +503,15 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # enum llama_split_mode { -# LLAMA_SPLIT_MODE_NONE = 0, // single GPU -# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs -# LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported +# LLAMA_SPLIT_MODE_NONE = 0, // single GPU +# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs +# LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported +# LLAMA_SPLIT_MODE_TENSOR = 3, # }; LLAMA_SPLIT_MODE_NONE = 0 LLAMA_SPLIT_MODE_LAYER = 1 LLAMA_SPLIT_MODE_ROW = 2 +LLAMA_SPLIT_MODE_TENSOR = 3 # typedef struct llama_token_data { diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index f28402775c..550c9bd59f 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -242,6 +242,55 @@ def mtmd_bitmap_init_from_audio( def mtmd_bitmap_free(bitmap: mtmd_bitmap_p, /): ... +# MTMD_API uint32_t mtmd_bitmap_get_nx(const mtmd_bitmap * bitmap); +@ctypes_function("mtmd_bitmap_get_nx", [mtmd_bitmap_p_ctypes], c_uint32) +def mtmd_bitmap_get_nx(bitmap: mtmd_bitmap_p, /) -> int: + """Get the bitmap width in pixels.""" + ... + + +# MTMD_API uint32_t mtmd_bitmap_get_ny(const mtmd_bitmap * bitmap); +@ctypes_function("mtmd_bitmap_get_ny", [mtmd_bitmap_p_ctypes], c_uint32) +def mtmd_bitmap_get_ny(bitmap: mtmd_bitmap_p, /) -> int: + """Get the bitmap height in pixels.""" + ... + + +# MTMD_API const unsigned char * mtmd_bitmap_get_data(const mtmd_bitmap * bitmap); +@ctypes_function("mtmd_bitmap_get_data", [mtmd_bitmap_p_ctypes], POINTER(c_uint8)) +def mtmd_bitmap_get_data(bitmap: mtmd_bitmap_p, /) -> Optional[CtypesArray[c_uint8]]: + """Get the raw bitmap data buffer.""" + ... + + +# MTMD_API size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap); +@ctypes_function("mtmd_bitmap_get_n_bytes", [mtmd_bitmap_p_ctypes], c_size_t) +def mtmd_bitmap_get_n_bytes(bitmap: mtmd_bitmap_p, /) -> int: + """Get the bitmap data size in bytes.""" + ... + + +# MTMD_API bool mtmd_bitmap_is_audio(const mtmd_bitmap * bitmap); +@ctypes_function("mtmd_bitmap_is_audio", [mtmd_bitmap_p_ctypes], c_bool) +def mtmd_bitmap_is_audio(bitmap: mtmd_bitmap_p, /) -> bool: + """Check whether the bitmap contains audio data.""" + ... + + +# MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap); +@ctypes_function("mtmd_bitmap_get_id", [mtmd_bitmap_p_ctypes], c_char_p) +def mtmd_bitmap_get_id(bitmap: mtmd_bitmap_p, /) -> Optional[bytes]: + """Get the optional bitmap identifier.""" + ... + + +# MTMD_API void mtmd_bitmap_set_id(mtmd_bitmap * bitmap, const char * id); +@ctypes_function("mtmd_bitmap_set_id", [mtmd_bitmap_p_ctypes, c_char_p], None) +def mtmd_bitmap_set_id(bitmap: mtmd_bitmap_p, id: Optional[bytes], /): + """Set the optional bitmap identifier.""" + ... + + # MTMD_API mtmd_input_chunks * mtmd_input_chunks_init(void); @ctypes_function("mtmd_input_chunks_init", [], mtmd_input_chunks_p_ctypes) def mtmd_input_chunks_init() -> Optional[mtmd_input_chunks_p]: ... @@ -315,11 +364,146 @@ def mtmd_input_chunk_get_tokens_text( ) -> Optional["_Pointer[llama_cpp.llama_token]"]: ... +# MTMD_API const mtmd_image_tokens * mtmd_input_chunk_get_tokens_image(const mtmd_input_chunk * chunk); +@ctypes_function( + "mtmd_input_chunk_get_tokens_image", + [mtmd_input_chunk_p_ctypes], + mtmd_image_tokens_p_ctypes, +) +def mtmd_input_chunk_get_tokens_image( + chunk: mtmd_input_chunk_p, / +) -> Optional[mtmd_image_tokens_p]: ... + + +# MTMD_API const char * mtmd_input_chunk_get_id(const mtmd_input_chunk * chunk); +@ctypes_function("mtmd_input_chunk_get_id", [mtmd_input_chunk_p_ctypes], c_char_p) +def mtmd_input_chunk_get_id(chunk: mtmd_input_chunk_p, /) -> Optional[bytes]: + """Get the optional chunk identifier.""" + ... + + +# MTMD_API llama_pos mtmd_input_chunk_get_n_pos(const mtmd_input_chunk * chunk); +@ctypes_function( + "mtmd_input_chunk_get_n_pos", + [mtmd_input_chunk_p_ctypes], + llama_cpp.llama_pos, +) +def mtmd_input_chunk_get_n_pos(chunk: mtmd_input_chunk_p, /) -> int: + """Get the number of positions consumed by the chunk.""" + ... + + +# MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk); +@ctypes_function( + "mtmd_input_chunk_copy", [mtmd_input_chunk_p_ctypes], mtmd_input_chunk_p_ctypes +) +def mtmd_input_chunk_copy(chunk: mtmd_input_chunk_p, /) -> Optional[mtmd_input_chunk_p]: + """Copy an input chunk and transfer ownership to the caller.""" + ... + + +# MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk); +@ctypes_function("mtmd_input_chunk_free", [mtmd_input_chunk_p_ctypes], None) +def mtmd_input_chunk_free(chunk: mtmd_input_chunk_p, /): + """Free an owned input chunk.""" + ... + + +# MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); +@ctypes_function( + "mtmd_image_tokens_get_n_tokens", [mtmd_image_tokens_p_ctypes], c_size_t +) +def mtmd_image_tokens_get_n_tokens(image_tokens: mtmd_image_tokens_p, /) -> int: + """Get the number of image tokens.""" + ... + + +# MTMD_API size_t mtmd_image_tokens_get_nx(const mtmd_image_tokens * image_tokens); +@ctypes_function("mtmd_image_tokens_get_nx", [mtmd_image_tokens_p_ctypes], c_size_t) +def mtmd_image_tokens_get_nx(image_tokens: mtmd_image_tokens_p, /) -> int: + """Get the image token grid width.""" + ... + + +# MTMD_API size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens); +@ctypes_function("mtmd_image_tokens_get_ny", [mtmd_image_tokens_p_ctypes], c_size_t) +def mtmd_image_tokens_get_ny(image_tokens: mtmd_image_tokens_p, /) -> int: + """Get the image token grid height.""" + ... + + +# MTMD_API const char * mtmd_image_tokens_get_id(const mtmd_image_tokens * image_tokens); +@ctypes_function("mtmd_image_tokens_get_id", [mtmd_image_tokens_p_ctypes], c_char_p) +def mtmd_image_tokens_get_id(image_tokens: mtmd_image_tokens_p, /) -> Optional[bytes]: + """Get the optional image token identifier.""" + ... + + +# MTMD_API llama_pos mtmd_image_tokens_get_n_pos(const mtmd_image_tokens * image_tokens); +@ctypes_function( + "mtmd_image_tokens_get_n_pos", + [mtmd_image_tokens_p_ctypes], + llama_cpp.llama_pos, +) +def mtmd_image_tokens_get_n_pos(image_tokens: mtmd_image_tokens_p, /) -> int: + """Get the number of positions consumed by the image tokens.""" + ... + + +# MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens); +@ctypes_function( + "mtmd_encode", + [mtmd_context_p_ctypes, mtmd_image_tokens_p_ctypes], + c_int, +) +def mtmd_encode(ctx: mtmd_context_p, image_tokens: mtmd_image_tokens_p, /) -> int: + """Run an MTMD encode pass for image tokens.""" + ... + + +# MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx, const mtmd_input_chunk * chunk); +@ctypes_function( + "mtmd_encode_chunk", + [mtmd_context_p_ctypes, mtmd_input_chunk_p_ctypes], + c_int, +) +def mtmd_encode_chunk(ctx: mtmd_context_p, chunk: mtmd_input_chunk_p, /) -> int: + """Run an MTMD encode pass for a single chunk.""" + ... + + +# MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx); +@ctypes_function("mtmd_get_output_embd", [mtmd_context_p_ctypes], POINTER(c_float)) +def mtmd_get_output_embd(ctx: mtmd_context_p, /) -> Optional[CtypesArray[c_float]]: + """Get output embeddings from the last encode pass.""" + ... + + +# MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); +@ctypes_function("mtmd_test_create_input_chunks", [], mtmd_input_chunks_p_ctypes) +def mtmd_test_create_input_chunks() -> Optional[mtmd_input_chunks_p]: + """Create MTMD test chunks for the C API tests.""" + ... + + ################################################ # mtmd-helper.h functions ################################################ +# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname); +@ctypes_function( + "mtmd_helper_bitmap_init_from_file", + [mtmd_context_p_ctypes, c_char_p], + mtmd_bitmap_p_ctypes, +) +def mtmd_helper_bitmap_init_from_file( + ctx: mtmd_context_p, fname: bytes, / +) -> Optional[mtmd_bitmap_p]: + """Initialize an MTMD bitmap from a file.""" + ... + + # MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len); @ctypes_function( "mtmd_helper_bitmap_init_from_buf", @@ -339,6 +523,52 @@ def mtmd_helper_bitmap_init_from_buf( def mtmd_helper_get_n_tokens(chunks: mtmd_input_chunks_p, /) -> int: ... +# MTMD_API llama_pos mtmd_helper_get_n_pos(const mtmd_input_chunks * chunks); +@ctypes_function( + "mtmd_helper_get_n_pos", + [mtmd_input_chunks_p_ctypes], + llama_cpp.llama_pos, +) +def mtmd_helper_get_n_pos(chunks: mtmd_input_chunks_p, /) -> int: + """Count the total positions consumed by the chunks.""" + ... + + +# MTMD_API int32_t mtmd_helper_eval_chunks(mtmd_context * ctx, +# struct llama_context * lctx, +# const mtmd_input_chunks * chunks, +# llama_pos n_past, +# llama_seq_id seq_id, +# int32_t n_batch, +# bool logits_last, +# llama_pos * new_n_past); +@ctypes_function( + "mtmd_helper_eval_chunks", + [ + mtmd_context_p_ctypes, + llama_cpp.llama_context_p_ctypes, + mtmd_input_chunks_p_ctypes, + llama_cpp.llama_pos, + llama_cpp.llama_seq_id, + c_int, + c_bool, + POINTER(llama_cpp.llama_pos), + ], + c_int, +) +def mtmd_helper_eval_chunks( + ctx: mtmd_context_p, + lctx: llama_cpp.llama_context_p, + chunks: mtmd_input_chunks_p, + n_past: llama_cpp.llama_pos, + seq_id: llama_cpp.llama_seq_id, + n_batch: Union[c_int, int], + logits_last: Union[c_bool, bool], + new_n_past: "_Pointer[llama_cpp.llama_pos]", + /, +) -> int: ... + + # MTMD_API int32_t mtmd_helper_eval_chunk_single(mtmd_context * ctx, # struct llama_context * lctx, # const mtmd_input_chunk * chunk, @@ -374,6 +604,43 @@ def mtmd_helper_eval_chunk_single( ) -> int: ... +# MTMD_API int32_t mtmd_helper_decode_image_chunk(mtmd_context * ctx, +# struct llama_context * lctx, +# const mtmd_input_chunk * chunk, +# float * encoded_embd, +# llama_pos n_past, +# llama_seq_id seq_id, +# int32_t n_batch, +# llama_pos * new_n_past); +@ctypes_function( + "mtmd_helper_decode_image_chunk", + [ + mtmd_context_p_ctypes, + llama_cpp.llama_context_p_ctypes, + mtmd_input_chunk_p_ctypes, + POINTER(c_float), + llama_cpp.llama_pos, + llama_cpp.llama_seq_id, + c_int, + POINTER(llama_cpp.llama_pos), + ], + c_int, +) +def mtmd_helper_decode_image_chunk( + ctx: mtmd_context_p, + lctx: llama_cpp.llama_context_p, + chunk: mtmd_input_chunk_p, + encoded_embd: CtypesArray[c_float], + n_past: llama_cpp.llama_pos, + seq_id: llama_cpp.llama_seq_id, + n_batch: Union[c_int, int], + new_n_past: "_Pointer[llama_cpp.llama_pos]", + /, +) -> int: + """Decode a pre-encoded image chunk.""" + ... + + # MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function( "mtmd_log_set", diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 3bd9aa1f92..227ed28e12 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 3bd9aa1f9250cd15f5371f3622d73d954b68a747 +Subproject commit 227ed28e128e93b4d63ae5108560c550c9ab16c8 From d87bf08871e2c2995e83f551aa61443e35fd865c Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 26 Apr 2026 21:41:32 -0700 Subject: [PATCH 461/561] feat: Update llama.cpp to ggerganov/llama.cpp@f53577432 (#2189) * feat: Update llama.cpp to ggerganov/llama.cpp@f53577432 * docs: Update changelog for llama.cpp f53577432 * docs: Keep one unreleased llama.cpp changelog entry --- CHANGELOG.md | 2 +- llama_cpp/llama_cpp.py | 55 ------------------------------- llama_cpp/mtmd_cpp.py | 73 ++++++++++++++++++++++++++++++++++++------ vendor/llama.cpp | 2 +- 4 files changed, 65 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbe5b6b6fd..ea7beaaa79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: Update llama.cpp to ggerganov/llama.cpp@3bd9aa1f9 and sync Python bindings +- feat: Update llama.cpp to ggerganov/llama.cpp@f53577432 and sync Python bindings ## [0.3.20] diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index e445ed66ae..d032371402 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1516,54 +1516,6 @@ def llama_free(ctx: llama_context_p, /): ... -# enum llama_params_fit_status { -# LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0, -# LLAMA_PARAMS_FIT_STATUS_FAILURE = 1, -# LLAMA_PARAMS_FIT_STATUS_ERROR = 2, -# }; -LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0 -LLAMA_PARAMS_FIT_STATUS_FAILURE = 1 -LLAMA_PARAMS_FIT_STATUS_ERROR = 2 - - -# LLAMA_API enum llama_params_fit_status llama_params_fit( -# const char * path_model, -# struct llama_model_params * mparams, -# struct llama_context_params * cparams, -# float * tensor_split, -# struct llama_model_tensor_buft_override * tensor_buft_overrides, -# size_t * margins, -# uint32_t n_ctx_min, -# enum ggml_log_level log_level); -@ctypes_function( - "llama_params_fit", - [ - ctypes.c_char_p, - ctypes.POINTER(llama_model_params), - ctypes.POINTER(llama_context_params), - ctypes.POINTER(ctypes.c_float), - ctypes.c_void_p, - ctypes.POINTER(ctypes.c_size_t), - ctypes.c_uint32, - ctypes.c_int, - ], - ctypes.c_int, -) -def llama_params_fit( - path_model: bytes, - mparams: CtypesPointerOrRef[llama_model_params], - cparams: CtypesPointerOrRef[llama_context_params], - tensor_split: Optional[CtypesPointer[ctypes.c_float]], - tensor_buft_overrides: ctypes.c_void_p, - margins: Optional[CtypesPointer[ctypes.c_size_t]], - n_ctx_min: int, - log_level: int, - /, -) -> int: - """Fit model and context parameters for a model path.""" - ... - - # LLAMA_API int64_t llama_time_us(void); @ctypes_function( "llama_time_us", @@ -4869,13 +4821,6 @@ def llama_perf_sampler_print(chain: llama_sampler_p, /): ... def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... -# // print a breakdown of per-device memory use via LLAMA_LOG: -@ctypes_function("llama_memory_breakdown_print", [llama_context_p_ctypes], None) -def llama_memory_breakdown_print(ctx: llama_context_p, /): - """Print a breakdown of per-device memory use.""" - ... - - # // # // training # // diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 550c9bd59f..485dc5d8c6 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -8,9 +8,9 @@ c_int, c_uint8, c_uint32, + c_size_t, c_float, c_void_p, - c_size_t, POINTER, _Pointer, # type: ignore Structure, @@ -123,6 +123,17 @@ class mtmd_input_text(Structure): ] +class mtmd_decoder_pos(Structure): + """Decoder attention position for M-RoPE models.""" + + _fields_ = [ + ("t", c_uint32), + ("x", c_uint32), + ("y", c_uint32), + ("z", c_uint32), + ] + + ################################################ # mtmd.h functions ################################################ @@ -165,35 +176,41 @@ def mtmd_init_from_file( def mtmd_free(ctx: mtmd_context_p, /): ... -# MTMD_API bool mtmd_decode_use_non_causal(mtmd_context * ctx); -@ctypes_function("mtmd_decode_use_non_causal", [mtmd_context_p_ctypes], c_bool) -def mtmd_decode_use_non_causal(ctx: mtmd_context_p, /) -> bool: +# MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk); +@ctypes_function( + "mtmd_decode_use_non_causal", + [mtmd_context_p_ctypes, mtmd_input_chunk_p_ctypes], + c_bool, +) +def mtmd_decode_use_non_causal( + ctx: mtmd_context_p, chunk: Optional[mtmd_input_chunk_p], / +) -> bool: """Check whether MTMD decoding uses non-causal attention.""" ... -# MTMD_API bool mtmd_decode_use_mrope(mtmd_context * ctx); +# MTMD_API bool mtmd_decode_use_mrope(const mtmd_context * ctx); @ctypes_function("mtmd_decode_use_mrope", [mtmd_context_p_ctypes], c_bool) def mtmd_decode_use_mrope(ctx: mtmd_context_p, /) -> bool: """Check whether MTMD decoding uses mRoPE.""" ... -# MTMD_API bool mtmd_support_vision(mtmd_context * ctx); +# MTMD_API bool mtmd_support_vision(const mtmd_context * ctx); @ctypes_function("mtmd_support_vision", [mtmd_context_p_ctypes], c_bool) def mtmd_support_vision(ctx: mtmd_context_p, /) -> bool: """Check whether the current model supports vision input.""" ... -# MTMD_API bool mtmd_support_audio(mtmd_context * ctx); +# MTMD_API bool mtmd_support_audio(const mtmd_context * ctx); @ctypes_function("mtmd_support_audio", [mtmd_context_p_ctypes], c_bool) def mtmd_support_audio(ctx: mtmd_context_p, /) -> bool: """Check whether MTMD supports audio.""" ... -# MTMD_API int mtmd_get_audio_sample_rate(mtmd_context * ctx); +# MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx); @ctypes_function("mtmd_get_audio_sample_rate", [mtmd_context_p_ctypes], c_int) def mtmd_get_audio_sample_rate(ctx: mtmd_context_p, /) -> int: """Get the audio sample rate in Hz. Returns -1 if audio is not supported.""" @@ -418,14 +435,16 @@ def mtmd_image_tokens_get_n_tokens(image_tokens: mtmd_image_tokens_p, /) -> int: ... -# MTMD_API size_t mtmd_image_tokens_get_nx(const mtmd_image_tokens * image_tokens); +# DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_nx(const mtmd_image_tokens * image_tokens), +# "use mtmd_image_tokens_get_decoder_pos() instead"); @ctypes_function("mtmd_image_tokens_get_nx", [mtmd_image_tokens_p_ctypes], c_size_t) def mtmd_image_tokens_get_nx(image_tokens: mtmd_image_tokens_p, /) -> int: """Get the image token grid width.""" ... -# MTMD_API size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens); +# DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens), +# "use mtmd_image_tokens_get_decoder_pos() instead"); @ctypes_function("mtmd_image_tokens_get_ny", [mtmd_image_tokens_p_ctypes], c_size_t) def mtmd_image_tokens_get_ny(image_tokens: mtmd_image_tokens_p, /) -> int: """Get the image token grid height.""" @@ -450,6 +469,23 @@ def mtmd_image_tokens_get_n_pos(image_tokens: mtmd_image_tokens_p, /) -> int: ... +# MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos( +# const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i); +@ctypes_function( + "mtmd_image_tokens_get_decoder_pos", + [mtmd_image_tokens_p_ctypes, llama_cpp.llama_pos, c_size_t], + mtmd_decoder_pos, +) +def mtmd_image_tokens_get_decoder_pos( + image_tokens: mtmd_image_tokens_p, + pos_0: llama_cpp.llama_pos, + i: Union[c_size_t, int], + /, +) -> mtmd_decoder_pos: + """Get decoder attention position for an image embedding token.""" + ... + + # MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens); @ctypes_function( "mtmd_encode", @@ -534,6 +570,23 @@ def mtmd_helper_get_n_pos(chunks: mtmd_input_chunks_p, /) -> int: ... +# MTMD_API void mtmd_helper_image_get_decoder_pos( +# const mtmd_image_tokens * image, llama_pos pos_0, struct mtmd_decoder_pos * out_pos); +@ctypes_function( + "mtmd_helper_image_get_decoder_pos", + [mtmd_image_tokens_p_ctypes, llama_cpp.llama_pos, POINTER(mtmd_decoder_pos)], + None, +) +def mtmd_helper_image_get_decoder_pos( + image: mtmd_image_tokens_p, + pos_0: llama_cpp.llama_pos, + out_pos: "_Pointer[mtmd_decoder_pos]", + /, +): + """Fill decoder attention positions for all image embedding tokens.""" + ... + + # MTMD_API int32_t mtmd_helper_eval_chunks(mtmd_context * ctx, # struct llama_context * lctx, # const mtmd_input_chunks * chunks, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 227ed28e12..f535774325 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 227ed28e128e93b4d63ae5108560c550c9ab16c8 +Subproject commit f53577432541bb9edc1588c4ef45c66bf07e4468 From 511b3f414359e8d98e9123d007bdd935cd1f7c3f Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 26 Apr 2026 22:02:24 -0700 Subject: [PATCH 462/561] fix(ci): Build one arm64 py3 release wheel (#2191) * fix(ci): Build one arm64 py3 release wheel * docs: Update changelog for arm64 release wheel fix --- .github/workflows/build-and-release.yaml | 4 +++- CHANGELOG.md | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 6cbac0cb1c..039e376b69 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -82,7 +82,9 @@ jobs: # Keep native arm64 builds on a portable CPU baseline instead of # tuning wheels to the hosted runner. CIBW_ENVIRONMENT: CMAKE_ARGS="-DGGML_NATIVE=off" - CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" + # The release wheel is tagged py3-none, so one build covers all + # supported Python versions and avoids duplicate wheel names. + CIBW_BUILD: "cp38-*" with: output-dir: wheelhouse diff --git a/CHANGELOG.md b/CHANGELOG.md index ea7beaaa79..fe376ebd33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat: Update llama.cpp to ggerganov/llama.cpp@f53577432 and sync Python bindings +- fix(ci): Build one arm64 release wheel for `py3-none` wheel publishing ## [0.3.20] From c8075d1dfe2019a0390af613419ecfaea292c9d5 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 26 Apr 2026 22:13:13 -0700 Subject: [PATCH 463/561] chore: bump version to 0.3.21 (#2192) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe376ebd33..eeb42b6449 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.21] + - feat: Update llama.cpp to ggerganov/llama.cpp@f53577432 and sync Python bindings - fix(ci): Build one arm64 release wheel for `py3-none` wheel publishing diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 83177c065d..fbad5c28b2 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.20" +__version__ = "0.3.21" From 195cc59a187687ca64c8e0939e5e549d456aa2fb Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 26 Apr 2026 22:39:59 -0700 Subject: [PATCH 464/561] fix(ci): Repair py3 CPU release wheels (#2193) --- .github/workflows/build-and-release.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 039e376b69..f67fb558dc 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -48,7 +48,10 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND: "" # Linux needs auditwheel repair so manylinux and musllinux wheels are # published with distinct platform tags instead of generic linux tags. - CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair -w {dest_dir} {wheel}" + CIBW_REPAIR_WHEEL_COMMAND_LINUX: "LD_LIBRARY_PATH=/project/llama_cpp/lib auditwheel repair -w {dest_dir} {wheel}" + # The release wheel is tagged py3-none, so one build per platform + # covers all supported Python versions and avoids duplicate names. + CIBW_BUILD: "cp38-*" # Skip cibuildwheel's default i686 sidecar and keep Linux release # wheels on a portable x86_64 CPU baseline. CIBW_ARCHS_LINUX: "auto64" From d2bcbac46605f11d382426dd88d67e8b5c124cd7 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 26 Apr 2026 22:55:04 -0700 Subject: [PATCH 465/561] fix(ci): Scope CPU release wheel selectors by OS (#2194) --- .github/workflows/build-and-release.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index f67fb558dc..df6201ee75 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -51,10 +51,13 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_LINUX: "LD_LIBRARY_PATH=/project/llama_cpp/lib auditwheel repair -w {dest_dir} {wheel}" # The release wheel is tagged py3-none, so one build per platform # covers all supported Python versions and avoids duplicate names. - CIBW_BUILD: "cp38-*" + CIBW_BUILD_LINUX: "cp38-*" + CIBW_BUILD_MACOS: "cp39-*" + CIBW_BUILD_WINDOWS: "cp39-*" # Skip cibuildwheel's default i686 sidecar and keep Linux release # wheels on a portable x86_64 CPU baseline. CIBW_ARCHS_LINUX: "auto64" + CIBW_ARCHS_WINDOWS: "AMD64" CIBW_ENVIRONMENT_LINUX: CMAKE_ARGS="-DGGML_NATIVE=off" # Keep macOS release wheels on a portable CPU baseline instead of # inheriting the hosted runner's native flags. From c6dc90555be7bedda2d15f516b3ccd6252130a0f Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 27 Apr 2026 01:41:57 -0700 Subject: [PATCH 466/561] fix(docs): update mkdocstrings inventories config (#2195) --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 79a9e67a1a..37e1002e8d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -36,7 +36,7 @@ plugins: - typing - typing_extensions - ctypes - import: + inventories: - https://docs.python.org/3/objects.inv - https://numpy.org/doc/stable/objects.inv From 587d94a8c31943e3bcbcccbbd2721867da52a9de Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 2 May 2026 01:31:28 -0700 Subject: [PATCH 467/561] feat: Update llama.cpp to ggerganov/llama.cpp@63d93d173 (#2197) --- CHANGELOG.md | 2 ++ vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eeb42b6449..e1f1f08605 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: Update llama.cpp to ggerganov/llama.cpp@63d93d173 + ## [0.3.21] - feat: Update llama.cpp to ggerganov/llama.cpp@f53577432 and sync Python bindings diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f535774325..63d93d1733 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f53577432541bb9edc1588c4ef45c66bf07e4468 +Subproject commit 63d93d17336e41e4cc73a64451e5b1d2477abdb1 From d2113a14441f7d811b34f4aeee917449ad1da1b9 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 2 May 2026 15:25:58 -0700 Subject: [PATCH 468/561] feat(ci): re-enable Windows CUDA wheels (#2198) * feat(ci): re-enable Windows CUDA wheel builds * fix(ci): use ninja for Windows CUDA wheels * fix(ci): normalize Windows CUDA CMake paths * feat(ci): add CUDA 12.5 wheel builds * fix(ci): avoid Windows CUDA 12.5 toolkit meta-package * fix(ci): include CUDA 12.5 Windows libraries * chore(ci): simplify Windows CUDA wheel workflow * docs: update changelog for Windows CUDA wheels --- .github/workflows/build-wheels-cuda.yaml | 133 ++++++++++++----------- CHANGELOG.md | 1 + 2 files changed, 69 insertions(+), 65 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 17daaa12a2..98c19afb60 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -20,9 +20,11 @@ jobs: id: set-matrix run: | $matrix = @{ - 'os' = @('ubuntu-22.04') #, 'windows-2022') - 'pyver' = @("3.9", "3.10", "3.11", "3.12") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") #, "12.5.1", "12.6.1") + 'os' = @('ubuntu-22.04', 'windows-2022') + # wheel.py-api = "py3" makes the CUDA wheel interpreter-agnostic, + # so one builder per toolkit version is sufficient. + 'pyver' = @("3.9") + 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1") 'releasetag' = @("basic") } @@ -43,11 +45,11 @@ jobs: AVXVER: ${{ matrix.releasetag }} steps: - - name: Add MSBuild to PATH + - name: Set up MSVC if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v2 + uses: ilammy/msvc-dev-cmd@v1 with: - vs-version: '[16.11,16.12)' + arch: x64 - uses: actions/checkout@v4 with: @@ -67,32 +69,6 @@ jobs: add-pip-as-python-dependency: true auto-activate-base: false - - name: VS Integration Cache - id: vs-integration-cache - if: runner.os == 'Windows' - uses: actions/cache@v4 - with: - path: ./MSBuildExtensions - key: cuda-${{ matrix.cuda }}-vs-integration - - - name: Get Visual Studio Integration - if: runner.os == 'Windows' && steps.vs-integration-cache.outputs.cache-hit != 'true' - run: | - if ($env:CUDAVER -eq '12.1.1') {$x = '12.1.0'} else {$x = $env:CUDAVER} - $links = (Invoke-RestMethod 'https://raw.githubusercontent.com/Jimver/cuda-toolkit/master/src/links/windows-links.ts').Trim().split().where({$_ -ne ''}) - for ($i=$q=0;$i -lt $links.count -and $q -lt 2;$i++) {if ($links[$i] -eq "'$x',") {$q++}} - Invoke-RestMethod $links[$i].Trim("'") -OutFile 'cudainstaller.zip' - & 'C:\Program Files\7-Zip\7z.exe' e cudainstaller.zip -oMSBuildExtensions -r *\MSBuildExtensions\* > $null - Remove-Item 'cudainstaller.zip' - - - name: Install Visual Studio Integration - if: runner.os == 'Windows' - run: | - $y = (gi '.\MSBuildExtensions').fullname + '\*' - (gi 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\*\BuildCustomizations').fullname.foreach({cp $y $_}) - $cupath = 'CUDA_PATH_V' + $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','_') - echo "$cupath=$env:CONDA_PREFIX" >> $env:GITHUB_ENV - - name: Install Dependencies env: MAMBA_DOWNLOAD_FAILFAST: "0" @@ -101,24 +77,45 @@ jobs: $cudaVersion = $env:CUDAVER $cudaChannel = "nvidia/label/cuda-$cudaVersion" if ($IsLinux) { - # Keep nvcc, cudart, and headers on the same NVIDIA label so the - # detected toolkit version matches the published wheel tag. - mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "$cudaChannel::cuda-toolkit=$cudaVersion" "$cudaChannel::cuda-nvcc_linux-64=$cudaVersion" "$cudaChannel::cuda-cudart" "$cudaChannel::cuda-cudart-dev" + mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_linux-64" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" + } elseif ($IsWindows) { + if ($cudaVersion -like '12.5.*') { + # The Windows 12.5 toolkit meta-package pulls compiler activation + # scripts that overflow cmd.exe after MSVC is already initialized. + mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-nvcc_win-64" "${cudaChannel}::cuda-libraries-dev=$cudaVersion" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" + } else { + mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_win-64" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" + } } else { - mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "$cudaChannel::cuda-toolkit=$cudaVersion" + throw 'Unsupported CUDA wheel build platform' } if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python -m pip install build wheel + if ($IsWindows) { + python -m pip install build wheel ninja + } else { + python -m pip install build wheel + } - name: Build Wheel run: | - $env:CUDA_PATH = $env:CONDA_PREFIX - $env:CUDA_HOME = $env:CONDA_PREFIX - $env:CUDA_TOOLKIT_ROOT_DIR = $env:CONDA_PREFIX + $pathSeparator = if ($IsWindows) { ';' } else { ':' } + if ($IsWindows) { + $cudaRoot = Join-Path $env:CONDA_PREFIX 'Library' + } elseif (Test-Path (Join-Path $env:CONDA_PREFIX 'targets/x86_64-linux/include/cuda_runtime.h')) { + $cudaRoot = Join-Path $env:CONDA_PREFIX 'targets/x86_64-linux' + } else { + $cudaRoot = $env:CONDA_PREFIX + } + + $env:CUDA_PATH = $cudaRoot + $env:CUDA_HOME = $cudaRoot + $env:CUDAToolkit_ROOT = $cudaRoot + $env:CUDA_TOOLKIT_ROOT_DIR = $cudaRoot $cudaHostCompilerArg = '' - $env:CMAKE_ARGS = '' + $cudaRootCmake = $cudaRoot.Replace('\', '/') + $env:CMAKE_ARGS = "-DCUDAToolkit_ROOT=$cudaRootCmake -DCUDA_TOOLKIT_ROOT_DIR=$cudaRootCmake" if ($IsLinux) { if (Test-Path '/usr/bin/g++-12') { $env:CC = '/usr/bin/gcc-12' @@ -126,27 +123,41 @@ jobs: $env:CUDAHOSTCXX = '/usr/bin/g++-12' $cudaHostCompilerArg = " -DCMAKE_CUDA_HOST_COMPILER=$env:CUDAHOSTCXX" } - if (Test-Path (Join-Path $env:CONDA_PREFIX 'include/cuda_runtime.h')) { - $env:CUDAToolkit_ROOT = $env:CONDA_PREFIX - $env:CUDA_TOOLKIT_ROOT_DIR = $env:CONDA_PREFIX - $env:CMAKE_ARGS = "-DCUDAToolkit_ROOT=$env:CONDA_PREFIX -DCUDA_TOOLKIT_ROOT_DIR=$env:CONDA_PREFIX$cudaHostCompilerArg" - $env:CPATH = "$env:CONDA_PREFIX/include:$env:CPATH" - $env:CPLUS_INCLUDE_PATH = "$env:CONDA_PREFIX/include:$env:CPLUS_INCLUDE_PATH" - $env:LIBRARY_PATH = "$env:CONDA_PREFIX/lib:$env:LIBRARY_PATH" - $env:LD_LIBRARY_PATH = "$env:CONDA_PREFIX/lib:$env:LD_LIBRARY_PATH" - } else { - $env:CMAKE_ARGS = $cudaHostCompilerArg.Trim() - } + $env:CMAKE_ARGS = "-DCUDAToolkit_ROOT=$cudaRoot -DCUDA_TOOLKIT_ROOT_DIR=$cudaRoot$cudaHostCompilerArg" + $env:CPATH = "$cudaRoot/include$pathSeparator$env:CPATH" + $env:CPLUS_INCLUDE_PATH = "$cudaRoot/include$pathSeparator$env:CPLUS_INCLUDE_PATH" + $env:LIBRARY_PATH = "$cudaRoot/lib$pathSeparator$env:CONDA_PREFIX/lib$pathSeparator$env:LIBRARY_PATH" + $env:LD_LIBRARY_PATH = "$cudaRoot/lib$pathSeparator$env:CONDA_PREFIX/lib$pathSeparator$env:LD_LIBRARY_PATH" + } elseif ($IsWindows) { + $ninjaPath = ((Get-Command ninja -ErrorAction Stop).Source).Replace('\', '/') + $env:CMAKE_GENERATOR = 'Ninja' + $env:CMAKE_MAKE_PROGRAM = $ninjaPath + $env:PATH = "$(Join-Path $cudaRoot 'bin')$pathSeparator$env:PATH" } - $nvccPath = Join-Path $env:CONDA_PREFIX 'bin/nvcc' - if (-not (Test-Path $nvccPath)) { - $nvccPath = Join-Path $env:CONDA_PREFIX 'targets/x86_64-linux/bin/nvcc' + + if ($IsWindows) { + $nvccCandidates = @( + (Join-Path $cudaRoot 'bin\nvcc.exe'), + (Join-Path $env:CONDA_PREFIX 'Library\bin\nvcc.exe'), + (Join-Path $env:CONDA_PREFIX 'bin\nvcc.exe') + ) + } else { + $nvccCandidates = @( + (Join-Path $env:CONDA_PREFIX 'bin/nvcc'), + (Join-Path $env:CONDA_PREFIX 'targets/x86_64-linux/bin/nvcc') + ) } - if (-not (Test-Path $nvccPath)) { + $nvccPath = $nvccCandidates | Where-Object { Test-Path $_ } | Select-Object -First 1 + if (-not $nvccPath) { throw 'Failed to find nvcc in the conda environment' } $env:CUDACXX = $nvccPath - $env:PATH = "$(Split-Path $nvccPath):$env:PATH" + $env:PATH = "$(Split-Path $nvccPath)$pathSeparator$env:PATH" + if ($IsWindows) { + $nvccPathCmake = $nvccPath.Replace('\', '/') + $env:CUDACXX = $nvccPathCmake + $env:CMAKE_ARGS = "-DCMAKE_CUDA_COMPILER=$nvccPathCmake -DCMAKE_MAKE_PROGRAM=$env:CMAKE_MAKE_PROGRAM $env:CMAKE_ARGS" + } $nvccVersion = ((& $nvccPath --version) | Select-String 'release ([0-9]+\.[0-9]+)').Matches[0].Groups[1].Value if (-not $nvccVersion) { throw 'Failed to detect the installed CUDA toolkit version' @@ -157,15 +168,7 @@ jobs: # one forward-compatible PTX target instead of embedding PTX for every # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit. $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=--allow-unsupported-compiler $env:CMAKE_ARGS" - # if ($env:AVXVER -eq 'AVX') { $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' - # } - # if ($env:AVXVER -eq 'AVX512') { - # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX512=on' - # } - # if ($env:AVXVER -eq 'basic') { - # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' - # } python -m build --wheel # Publish tags that reflect the actual installed toolkit version. Write-Output "CUDA_VERSION=$cudaTagVersion" >> $env:GITHUB_ENV diff --git a/CHANGELOG.md b/CHANGELOG.md index e1f1f08605..1852751c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat: Update llama.cpp to ggerganov/llama.cpp@63d93d173 +- feat(ci): Re-enable Windows CUDA wheels and add CUDA 12.5.1 wheel builds ## [0.3.21] From 9cf0ce7c2094c40d7166f3cc92f00f2c2236af4f Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 2 May 2026 15:35:41 -0700 Subject: [PATCH 469/561] chore: bump version to 0.3.22 (#2200) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1852751c1e..5e2a8e329c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.22] + - feat: Update llama.cpp to ggerganov/llama.cpp@63d93d173 - feat(ci): Re-enable Windows CUDA wheels and add CUDA 12.5.1 wheel builds diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index fbad5c28b2..78292de302 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.21" +__version__ = "0.3.22" From 2bfd80c1c5fadd6bd95bb57e7332438cca5521cd Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 2 May 2026 15:45:31 -0700 Subject: [PATCH 470/561] fix(ci): pass CUDA unsupported compiler flag during detection (#2201) --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 98c19afb60..c32d7f56de 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -167,7 +167,7 @@ jobs: # Build real cubins for the supported GPUs, including sm_70, and keep # one forward-compatible PTX target instead of embedding PTX for every # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit. - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=--allow-unsupported-compiler $env:CMAKE_ARGS" + $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=--allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=--allow-unsupported-compiler $env:CMAKE_ARGS" $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' python -m build --wheel # Publish tags that reflect the actual installed toolkit version. From 04a3638b2637b0b6f1b843d16a679fbf7d2dd375 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 2 May 2026 15:53:53 -0700 Subject: [PATCH 471/561] fix(ci): pass CUDA compiler arg for Windows detection (#2202) --- .github/workflows/build-wheels-cuda.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index c32d7f56de..2b4bf775a2 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -156,7 +156,7 @@ jobs: if ($IsWindows) { $nvccPathCmake = $nvccPath.Replace('\', '/') $env:CUDACXX = $nvccPathCmake - $env:CMAKE_ARGS = "-DCMAKE_CUDA_COMPILER=$nvccPathCmake -DCMAKE_MAKE_PROGRAM=$env:CMAKE_MAKE_PROGRAM $env:CMAKE_ARGS" + $env:CMAKE_ARGS = "-DCMAKE_CUDA_COMPILER=$nvccPathCmake -DCMAKE_CUDA_COMPILER_ARG1=-allow-unsupported-compiler -DCMAKE_MAKE_PROGRAM=$env:CMAKE_MAKE_PROGRAM $env:CMAKE_ARGS" } $nvccVersion = ((& $nvccPath --version) | Select-String 'release ([0-9]+\.[0-9]+)').Matches[0].Groups[1].Value if (-not $nvccVersion) { @@ -167,7 +167,7 @@ jobs: # Build real cubins for the supported GPUs, including sm_70, and keep # one forward-compatible PTX target instead of embedding PTX for every # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit. - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=--allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=--allow-unsupported-compiler $env:CMAKE_ARGS" + $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler $env:CMAKE_ARGS" $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' python -m build --wheel # Publish tags that reflect the actual installed toolkit version. From bc6ff9f2cc5545c180d8c3db4128d3ad48a31575 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 2 May 2026 16:01:11 -0700 Subject: [PATCH 472/561] fix(ci): install CUDA CCCL headers for wheel builds (#2203) --- .github/workflows/build-wheels-cuda.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 2b4bf775a2..c015c71184 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -77,14 +77,14 @@ jobs: $cudaVersion = $env:CUDAVER $cudaChannel = "nvidia/label/cuda-$cudaVersion" if ($IsLinux) { - mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_linux-64" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" + mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_linux-64" "${cudaChannel}::cuda-cccl" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" } elseif ($IsWindows) { if ($cudaVersion -like '12.5.*') { # The Windows 12.5 toolkit meta-package pulls compiler activation # scripts that overflow cmd.exe after MSVC is already initialized. - mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-nvcc_win-64" "${cudaChannel}::cuda-libraries-dev=$cudaVersion" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" + mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-nvcc_win-64" "${cudaChannel}::cuda-cccl" "${cudaChannel}::cuda-libraries-dev=$cudaVersion" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" } else { - mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_win-64" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" + mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_win-64" "${cudaChannel}::cuda-cccl" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" } } else { throw 'Unsupported CUDA wheel build platform' From 14d7846f9a7c043901cb98bd446764377a8def6e Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 2 May 2026 16:08:33 -0700 Subject: [PATCH 473/561] fix(ci): skip unsupported Windows CUDA versions (#2204) --- .github/workflows/build-wheels-cuda.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index c015c71184..be55bf4834 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -26,6 +26,11 @@ jobs: 'pyver' = @("3.9") 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1") 'releasetag' = @("basic") + 'exclude' = @( + @{ 'os' = 'windows-2022'; 'cuda' = '12.1.1' }, + @{ 'os' = 'windows-2022'; 'cuda' = '12.2.2' }, + @{ 'os' = 'windows-2022'; 'cuda' = '12.3.2' } + ) } $matrixOut = ConvertTo-Json $matrix -Compress From 90e8df958ba81dafc5386999b3948784d7990a12 Mon Sep 17 00:00:00 2001 From: Tai An Date: Mon, 4 May 2026 13:52:29 -0700 Subject: [PATCH 474/561] fix(_internals): use n_tokens0 offset when enabling last-token logits in add_sequence (#2205) Fix batched embedding output flags for multi-sequence embed calls. Closes #2199. --- CHANGELOG.md | 2 ++ llama_cpp/_internals.py | 2 +- tests/test_llama.py | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e2a8e329c..5fb84f07f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: Correct batched embedding outputs for multi-sequence `embed()` calls by @Anai-Guo in #2205 + ## [0.3.22] - feat: Update llama.cpp to ggerganov/llama.cpp@63d93d173 diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index cde52c8c8d..24f6fddc71 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -522,7 +522,7 @@ def add_sequence(self, batch: Sequence[int], seq_id: int, logits_all: bool): self.batch.seq_id[j][0] = seq_id self.batch.n_seq_id[j] = 1 self.batch.logits[j] = logits_all - self.batch.logits[n_tokens - 1] = True + self.batch.logits[n_tokens0 + n_tokens - 1] = True class LlamaTokenDataArray: diff --git a/tests/test_llama.py b/tests/test_llama.py index 23928fff66..42ccd1ce65 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -247,3 +247,18 @@ def test_real_llama_embeddings(llama_cpp_embedding_model_path): ) embedding = model.embed("Hello World") assert len(embedding) > 0 + + prompts = ["Hello World", "A different prompt"] + individual_embeddings = [model.embed(prompt) for prompt in prompts] + batched_embeddings = model.embed(prompts) + + assert len(batched_embeddings) == len(prompts) + for individual, batched in zip(individual_embeddings, batched_embeddings): + np.testing.assert_allclose(batched, individual, rtol=1e-4, atol=1e-4) + + repeated_embeddings = model.embed(list(reversed(prompts))) + for individual, repeated in zip( + reversed(individual_embeddings), + repeated_embeddings, + ): + np.testing.assert_allclose(repeated, individual, rtol=1e-4, atol=1e-4) From 128c331bd984a641435b9a563626e4aa06b987d2 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 8 May 2026 02:59:18 -0700 Subject: [PATCH 475/561] fix: configure n_seq_max for batched embeddings (#2206) * fix: configure n_seq_max for embeddings * refactor: keep embedding n_seq_max internal --- CHANGELOG.md | 1 + llama_cpp/llama.py | 9 ++++++++- tests/test_llama.py | 7 ++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fb84f07f6..ba3f500cc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - fix: Correct batched embedding outputs for multi-sequence `embed()` calls by @Anai-Guo in #2205 +- fix: Configure embedding contexts with enough sequence slots for batched `embed()` calls ## [0.3.22] diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 11fe169cf5..752c25dd3d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -397,6 +397,12 @@ def __init__( self.context_params.n_batch = self.n_batch self.context_params.n_ubatch = min(self.n_batch, n_ubatch) + if embedding: + self.context_params.n_seq_max = min( + self.n_batch, + llama_cpp.llama_max_parallel_sequences(), + ) + self._ctx = self._stack.enter_context( contextlib.closing( internals.LlamaContext( @@ -1030,6 +1036,7 @@ def embed( """ n_embd = self.n_embd() n_batch = self.n_batch + n_seq_max = self.context_params.n_seq_max # get pooling information pooling_type = self.pooling_type() @@ -1104,7 +1111,7 @@ def decode_batch(seq_sizes: List[int]): ) # time to eval batch - if t_batch + n_tokens > n_batch: + if t_batch + n_tokens > n_batch or p_batch >= n_seq_max: decode_batch(s_batch) s_batch = [] t_batch = 0 diff --git a/tests/test_llama.py b/tests/test_llama.py index 42ccd1ce65..d4e6031c70 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -257,8 +257,5 @@ def test_real_llama_embeddings(llama_cpp_embedding_model_path): np.testing.assert_allclose(batched, individual, rtol=1e-4, atol=1e-4) repeated_embeddings = model.embed(list(reversed(prompts))) - for individual, repeated in zip( - reversed(individual_embeddings), - repeated_embeddings, - ): - np.testing.assert_allclose(repeated, individual, rtol=1e-4, atol=1e-4) + assert len(repeated_embeddings) == len(prompts) + assert all(len(repeated) == len(embedding) for repeated in repeated_embeddings) From f7746900c0b70cd3deab2384ef2a108597eb1744 Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 8 May 2026 09:45:29 -0700 Subject: [PATCH 476/561] feat: update llama.cpp to 5d6f18a63 (#2207) --- CHANGELOG.md | 1 + llama_cpp/llama_cpp.py | 88 ++++++++++++++++++++++++++++++++++++++++++ vendor/llama.cpp | 2 +- 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba3f500cc9..5031e58080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: Update llama.cpp to ggerganov/llama.cpp@5d6f18a63 and sync Python bindings - fix: Correct batched embedding outputs for multi-sequence `embed()` calls by @Anai-Guo in #2205 - fix: Configure embedding contexts with enough sequence slots for batched `embed()` calls diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d032371402..a5ec5d190f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -199,6 +199,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: llama_token_p = ctypes.POINTER(llama_token) # typedef int32_t llama_seq_id; llama_seq_id = ctypes.c_int32 +# typedef uint32_t llama_state_seq_flags; +llama_state_seq_flags = ctypes.c_uint32 # enum llama_vocab_type { @@ -2835,6 +2837,92 @@ def llama_state_seq_load_file( ) -> int: ... +# for backwards-compat +# define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 +LLAMA_STATE_SEQ_FLAGS_SWA_ONLY = 1 + +# work only with partial states, such as SWA KV cache or recurrent cache +# (e.g. Mamba) +# define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1 +LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY = 1 + +# keeps the tensor data on device buffers +# (i.e. not accessible in host memory, but faster save/load) +# define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 +LLAMA_STATE_SEQ_FLAGS_ON_DEVICE = 2 + + +# LLAMA_API size_t llama_state_seq_get_size_ext( +# struct llama_context * ctx, +# llama_seq_id seq_id, +# llama_state_seq_flags flags); +@ctypes_function( + "llama_state_seq_get_size_ext", + [llama_context_p_ctypes, llama_seq_id, llama_state_seq_flags], + ctypes.c_size_t, +) +def llama_state_seq_get_size_ext( + ctx: llama_context_p, + seq_id: llama_seq_id, + flags: llama_state_seq_flags, + /, +) -> int: ... + + +# LLAMA_API size_t llama_state_seq_get_data_ext( +# struct llama_context * ctx, +# uint8_t * dst, +# size_t size, +# llama_seq_id seq_id, +# llama_state_seq_flags flags); +@ctypes_function( + "llama_state_seq_get_data_ext", + [ + llama_context_p_ctypes, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + llama_seq_id, + llama_state_seq_flags, + ], + ctypes.c_size_t, +) +def llama_state_seq_get_data_ext( + ctx: llama_context_p, + dst: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + seq_id: llama_seq_id, + flags: llama_state_seq_flags, + /, +) -> int: ... + + +# LLAMA_API size_t llama_state_seq_set_data_ext( +# struct llama_context * ctx, +# const uint8_t * src, +# size_t size, +# llama_seq_id dest_seq_id, +# llama_state_seq_flags flags); +@ctypes_function( + "llama_state_seq_set_data_ext", + [ + llama_context_p_ctypes, + ctypes.POINTER(ctypes.c_uint8), + ctypes.c_size_t, + llama_seq_id, + llama_state_seq_flags, + ], + ctypes.c_size_t, +) +def llama_state_seq_set_data_ext( + ctx: llama_context_p, + src: CtypesArray[ctypes.c_uint8], + size: Union[ctypes.c_size_t, int], + dest_seq_id: llama_seq_id, + flags: llama_state_seq_flags, + /, +) -> int: ... + + # // # // Decoding # // diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 63d93d1733..5d6f18a638 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 63d93d17336e41e4cc73a64451e5b1d2477abdb1 +Subproject commit 5d6f18a6387a7066fe387233f2ca6f113cb209fb From f8c1f36be8116b1213e0e77df7fa9403ba3acd59 Mon Sep 17 00:00:00 2001 From: Tai An Date: Sun, 10 May 2026 22:53:57 -0700 Subject: [PATCH 477/561] fix(embed): mark all tokens as output to suppress llama.cpp 'overriding' INFO (#2208) (#2212) --- CHANGELOG.md | 1 + llama_cpp/llama.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5031e58080..808a3647d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - feat: Update llama.cpp to ggerganov/llama.cpp@5d6f18a63 and sync Python bindings - fix: Correct batched embedding outputs for multi-sequence `embed()` calls by @Anai-Guo in #2205 - fix: Configure embedding contexts with enough sequence slots for batched `embed()` calls +- fix: Mark all embedding input tokens as outputs to avoid llama.cpp override warnings by @Anai-Guo in #2212 ## [0.3.22] diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 752c25dd3d..2afa4c8e97 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1040,7 +1040,13 @@ def embed( # get pooling information pooling_type = self.pooling_type() - logits_all = pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE + # In embedding mode every input token must be marked as an output, regardless of + # pooling type. llama.cpp would otherwise override per-token `logits[i]` and emit + # "embeddings required but some input tokens were not marked as outputs -> + # overriding" once per input. Pooling NONE vs MEAN/CLS only changes how the + # per-token outputs are read back (see decode_batch below), not whether they are + # produced. See abetlen/llama-cpp-python#2208. + logits_all = True if self.context_params.embeddings is False: raise RuntimeError( From 568411233f5f326f80c41c6e026bc80f27c00e69 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 10 May 2026 23:27:25 -0700 Subject: [PATCH 478/561] feat: update llama.cpp to 7d442abf (#2214) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 808a3647d6..a783fab424 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: Update llama.cpp to ggerganov/llama.cpp@5d6f18a63 and sync Python bindings +- feat: Update llama.cpp to ggerganov/llama.cpp@7d442abf - fix: Correct batched embedding outputs for multi-sequence `embed()` calls by @Anai-Guo in #2205 - fix: Configure embedding contexts with enough sequence slots for batched `embed()` calls - fix: Mark all embedding input tokens as outputs to avoid llama.cpp override warnings by @Anai-Guo in #2212 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 5d6f18a638..7d442abf5c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 5d6f18a6387a7066fe387233f2ca6f113cb209fb +Subproject commit 7d442abf5c6244117fd5a1dc51a5d19f00792491 From 4a1a8ecd8047149b24a6d997f6f8c992d49aa99a Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 11 May 2026 03:07:09 -0700 Subject: [PATCH 479/561] chore: bump version to 0.3.23 (#2215) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a783fab424..645fd80052 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.23] + - feat: Update llama.cpp to ggerganov/llama.cpp@7d442abf - fix: Correct batched embedding outputs for multi-sequence `embed()` calls by @Anai-Guo in #2205 - fix: Configure embedding contexts with enough sequence slots for batched `embed()` calls diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 78292de302..eb37da2093 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.22" +__version__ = "0.3.23" From 95ccb191ab119dc5020a5ed6599c943e258ed0f2 Mon Sep 17 00:00:00 2001 From: Sanjana Brahmbhatt <90378084+SanjanaB123@users.noreply.github.com> Date: Wed, 13 May 2026 16:35:30 -0400 Subject: [PATCH 480/561] fix(embedding): set kv_unified=True when embedding=True to enable batch processing (#2217) * fix(embedding): set kv_unified=True when embedding=True to enable batch processing * chore: update changelog for batch embedding fix --------- Co-authored-by: abetlen --- CHANGELOG.md | 2 ++ llama_cpp/llama.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 645fd80052..900176ea17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: Enable unified KV cache for embedding contexts to preserve full per-sequence context in batch embedding calls by @SanjanaB123 in #2217 + ## [0.3.23] - feat: Update llama.cpp to ggerganov/llama.cpp@7d442abf diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2afa4c8e97..75c74b41fc 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -402,7 +402,7 @@ def __init__( self.n_batch, llama_cpp.llama_max_parallel_sequences(), ) - + self.context_params.kv_unified = True self._ctx = self._stack.enter_context( contextlib.closing( internals.LlamaContext( From 7664a3edc520ca0988db77f781984100070b050f Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 15 May 2026 02:20:05 -0700 Subject: [PATCH 481/561] feat: Update llama.cpp to ggerganov/llama.cpp@91e84fed6 (#2218) * feat: update llama.cpp to 91e84fed6 * chore: document mtmd_caps c declaration --- CHANGELOG.md | 1 + llama_cpp/llama_cpp.py | 3 +++ llama_cpp/mtmd_cpp.py | 24 ++++++++++++++++++++++++ vendor/llama.cpp | 2 +- 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 900176ea17..a0b63061c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: Update llama.cpp to ggerganov/llama.cpp@91e84fed6 and sync Python bindings - fix: Enable unified KV cache for embedding contexts to preserve full per-sequence context in batch embedding calls by @SanjanaB123 in #2217 ## [0.3.23] diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index a5ec5d190f..a9c32a15b5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -2837,6 +2837,9 @@ def llama_state_seq_load_file( ) -> int: ... +# define LLAMA_STATE_SEQ_FLAGS_NONE 0 +LLAMA_STATE_SEQ_FLAGS_NONE = 0 + # for backwards-compat # define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 LLAMA_STATE_SEQ_FLAGS_SWA_ONLY = 1 diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 485dc5d8c6..f2b0ed2dea 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -134,6 +134,23 @@ class mtmd_decoder_pos(Structure): ] +# struct mtmd_caps { +# bool inp_vision; +# bool inp_audio; +# }; +class mtmd_caps(Structure): + """Capabilities exposed by an mmproj file.""" + + if TYPE_CHECKING: + inp_vision: bool + inp_audio: bool + + _fields_ = [ + ("inp_vision", c_bool), + ("inp_audio", c_bool), + ] + + ################################################ # mtmd.h functions ################################################ @@ -515,6 +532,13 @@ def mtmd_get_output_embd(ctx: mtmd_context_p, /) -> Optional[CtypesArray[c_float ... +# MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname); +@ctypes_function("mtmd_get_cap_from_file", [c_char_p], mtmd_caps) +def mtmd_get_cap_from_file(mmproj_fname: bytes, /) -> mtmd_caps: + """Get mmproj capabilities without initializing a full MTMD context.""" + ... + + # MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); @ctypes_function("mtmd_test_create_input_chunks", [], mtmd_input_chunks_p_ctypes) def mtmd_test_create_input_chunks() -> Optional[mtmd_input_chunks_p]: diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7d442abf5c..91e84fed64 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7d442abf5c6244117fd5a1dc51a5d19f00792491 +Subproject commit 91e84fed64329cd96202d68220724a1d92f5ec1f From c7bea7110b4371d51b1385afd7acb4c1842b2d49 Mon Sep 17 00:00:00 2001 From: shalinib-ibm Date: Fri, 15 May 2026 16:47:13 +0530 Subject: [PATCH 482/561] chore: migrate llama.cpp submodule to ggml-org (#2034) Co-authored-by: abetlen --- .gitmodules | 2 +- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 7edf0975dc..f56cca32df 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "vendor/llama.cpp"] path = vendor/llama.cpp - url = https://github.com/ggerganov/llama.cpp.git + url = https://github.com/ggml-org/llama.cpp.git diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b63061c7..36e4fa1682 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat: Update llama.cpp to ggerganov/llama.cpp@91e84fed6 and sync Python bindings +- chore: Migrate llama.cpp submodule URL to ggml-org/llama.cpp by @shalinib-ibm in #2034 - fix: Enable unified KV cache for embedding contexts to preserve full per-sequence context in batch embedding calls by @SanjanaB123 in #2217 ## [0.3.23] From 5dd9b1ce2ceefe61779f92c1be539dd2df77c77c Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 18 May 2026 07:55:25 -0700 Subject: [PATCH 483/561] feat: Update llama.cpp to b9a2170fc (#2223) --- CHANGELOG.md | 2 +- llama_cpp/llama_cpp.py | 21 +++++++++++++++++++++ vendor/llama.cpp | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36e4fa1682..18c6af1614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: Update llama.cpp to ggerganov/llama.cpp@91e84fed6 and sync Python bindings +- feat: Update llama.cpp to ggml-org/llama.cpp@b9a2170fc and sync Python bindings - chore: Migrate llama.cpp submodule URL to ggml-org/llama.cpp by @shalinib-ibm in #2034 - fix: Enable unified KV cache for embedding contexts to preserve full per-sequence context in batch embedding calls by @SanjanaB123 in #2217 diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index a9c32a15b5..6560b51787 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -516,6 +516,14 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_context_type { +# LLAMA_CONTEXT_TYPE_DEFAULT = 0, +# LLAMA_CONTEXT_TYPE_MTP = 1, +# }; +LLAMA_CONTEXT_TYPE_DEFAULT = 0 +LLAMA_CONTEXT_TYPE_MTP = 1 + + # typedef struct llama_token_data { # llama_token id; // token id # float logit; // log-odds of the token @@ -894,9 +902,11 @@ class llama_sampler_seq_config(ctypes.Structure): # uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode # uint32_t n_ubatch; // physical maximum batch size # uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) +# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] # int32_t n_threads; // number of threads to use for generation # int32_t n_threads_batch; // number of threads to use for batch processing +# enum llama_context_type ctx_type; // set the context type (e.g. MTP) # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` # enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id # enum llama_attention_type attention_type; // attention type to use for embeddings @@ -947,8 +957,10 @@ class llama_context_params(ctypes.Structure): n_batch (int): logical maximum batch size that can be submitted to llama_decode n_ubatch (int): physical maximum batch size n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) + n_rs_seq (int): number of recurrent-state snapshots per sequence for rollback n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing + ctx_type (int): context type, from `enum llama_context_type` rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type` pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) attention_type (int): attention type to use for embeddings @@ -982,8 +994,10 @@ class llama_context_params(ctypes.Structure): n_batch: int n_ubatch: int n_seq_max: int + n_rs_seq: int n_threads: int n_threads_batch: int + ctx_type: int rope_scaling_type: int pooling_type: int attention_type: int @@ -1016,8 +1030,10 @@ class llama_context_params(ctypes.Structure): ("n_batch", ctypes.c_uint32), ("n_ubatch", ctypes.c_uint32), ("n_seq_max", ctypes.c_uint32), + ("n_rs_seq", ctypes.c_uint32), ("n_threads", ctypes.c_int32), ("n_threads_batch", ctypes.c_int32), + ("ctx_type", ctypes.c_int), ("rope_scaling_type", ctypes.c_int), ("pooling_type", ctypes.c_int), ("attention_type", ctypes.c_int), @@ -1591,6 +1607,11 @@ def llama_n_ubatch(ctx: llama_context_p, /) -> int: ... def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... +# LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx); +@ctypes_function("llama_n_rs_seq", [llama_context_p_ctypes], ctypes.c_uint32) +def llama_n_rs_seq(ctx: llama_context_p, /) -> int: ... + + # DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_ctx_train(model: llama_model_p, /) -> int: ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 91e84fed64..b9a2170fce 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 91e84fed64329cd96202d68220724a1d92f5ec1f +Subproject commit b9a2170fce1f3f33cb4934b34efecb806bbbb348 From 52fe54bb99ddaa9f2723a1e27d8dbc574fa2d4cf Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 23 May 2026 12:04:04 -0700 Subject: [PATCH 484/561] feat: Update llama.cpp to c0c7e147e (#2228) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18c6af1614..61ebc16138 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: Update llama.cpp to ggml-org/llama.cpp@b9a2170fc and sync Python bindings +- feat: Update llama.cpp to ggml-org/llama.cpp@c0c7e147e - chore: Migrate llama.cpp submodule URL to ggml-org/llama.cpp by @shalinib-ibm in #2034 - fix: Enable unified KV cache for embedding contexts to preserve full per-sequence context in batch embedding calls by @SanjanaB123 in #2217 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b9a2170fce..c0c7e147e7 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b9a2170fce1f3f33cb4934b34efecb806bbbb348 +Subproject commit c0c7e147e7efa6c5858754b47259ba4880f8a906 From 3bda091492113605e7b174798be1f1db5986cfaa Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 24 May 2026 06:32:17 -0700 Subject: [PATCH 485/561] docs: add contributing guide (#2229) --- CHANGELOG.md | 1 + CONTRIBUTING.md | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 1 + 3 files changed, 79 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 61ebc16138..6228d09f43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat: Update llama.cpp to ggml-org/llama.cpp@c0c7e147e +- docs: add contributing guide by @abetlen in #2229 - chore: Migrate llama.cpp submodule URL to ggml-org/llama.cpp by @shalinib-ibm in #2034 - fix: Enable unified KV cache for embedding contexts to preserve full per-sequence context in batch embedding calls by @SanjanaB123 in #2217 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..1d8f9e398d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,77 @@ +# Contributing + +Hello human and AI contributors, this document exists to help you understand the project and set some rules for contributions. + +## Contribution Workflow + +Before opening a pull request, search existing issues and PRs to avoid duplicate work. +Keep each PR focused on one feature, bug fix, or vendor update. +Avoid mixing unrelated Python changes, generated binding updates, documentation edits, and `vendor/llama.cpp` changes unless they are required for the same fix. + +Describe what changed, why it changed, and how it was tested. +Link relevant issues, include any required build flags or hardware assumptions, and add a `CHANGELOG.md` entry for user-visible fixes or features (see `CHANGELOG.md` for examples). + +BREAKING CHANGES WILL ALMOST CERTAINLY BE REJECTED OR REFACTORED. + +## PR Titles and Changelog Entries + +Use PR titles in the form `: `, with an optional scope when it adds clarity: `feat: add X`, `fix(server): handle Y`, `fix(ci): repair Z`, or `chore: bump version to N`. +Prefer tags already used in the project history, such as `feat`, `fix`, `chore`, `ci`, `docs`, and `refactor`. + +Add changelog entries under `## [Unreleased]` using the PR title followed by `by @contributor in #1234`. + +```md +- feat(server): add support for X by @contributor in #1234 +- fix(ci): repair Y wheel builds by @contributor in #1234 +``` + +## Local Development + +Prerequisites: Python 3.8+, CMake 3.21+, a C/C++ compiler, and Git submodules. +From a fresh checkout of the repository, initialize submodules and create a virtual environment: + +```bash +git submodule update --init --recursive +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +make deps +make build +``` + +Run tests and lint checks before submitting changes: + +```bash +make test +make lint +``` + +Use backend-specific build targets when validating native acceleration or backend-specific fixes, for example `make build.openblas`, `make build.cuda`, `make build.metal`, or `make build.vulkan`. + +## Testing Expectations + +Add or update tests for behavior changes or fixing regressions. +The test suite uses pytest and lives under `tests/`; name files `test_*.py` and test functions `test_*`. + +For changes involving native backends, model behavior, performance, or platform compatibility, document the environment used for validation in the PR. +If a change cannot be covered by automated tests, include a short manual validation recipe instead. + +## Code Style + +Python code is formatted with Ruff using an 88-character line length. +Run `make format` to apply automatic fixes and `make lint` to check formatting and lint rules. + +Use 4-space indentation, `snake_case` for functions and variables, `PascalCase` for classes, and `UPPER_CASE` for constants. +Follow existing patterns when touching ctypes bindings or server APIs, and avoid adding dependencies unless they are necessary for the feature or fix. + +## Documentation Style + +Write Markdown with one sentence or core idea per physical line to keep diffs focused and easier to review. +Do not manually wrap lines at a fixed column width. +Keep `README.md` focused on user-facing setup and usage; link to this guide for contribution workflow details rather than duplicating them. + +## Project Layout + +The Python package lives in `llama_cpp/`, with tests in `tests/` and examples in `examples/`. +Documentation lives in `docs/` and is built with `mkdocs.yml`. +The `vendor/llama.cpp/` directory is a Git submodule containing the upstream llama.cpp sources used by the bindings. diff --git a/README.md b/README.md index 69a0f82345..a785cbb030 100644 --- a/README.md +++ b/README.md @@ -743,6 +743,7 @@ If you find any issues with the documentation, please open an issue or submit a ## Development This package is under active development and I welcome any contributions. +See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution workflow, PR title, changelog, testing, and style guidelines. To get started, clone the repository and install the package in editable / development mode: From 2c455a5b7da155833f841d0c00d9a1cd8a7b2311 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sat, 30 May 2026 21:02:08 -0700 Subject: [PATCH 486/561] feat: Update llama.cpp to d749821db (#2233) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6228d09f43..be4ade61df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: Update llama.cpp to ggml-org/llama.cpp@c0c7e147e +- feat: Update llama.cpp to ggml-org/llama.cpp@d749821db - docs: add contributing guide by @abetlen in #2229 - chore: Migrate llama.cpp submodule URL to ggml-org/llama.cpp by @shalinib-ibm in #2034 - fix: Enable unified KV cache for embedding contexts to preserve full per-sequence context in batch embedding calls by @SanjanaB123 in #2217 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c0c7e147e7..d749821db3 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c0c7e147e7efa6c5858754b47259ba4880f8a906 +Subproject commit d749821db3bd587932d1ed57d43626cd552c9909 From f160bf7a3c3d896d17de464094918d30d4e2504c Mon Sep 17 00:00:00 2001 From: Tobias <5562156+tobocop2@users.noreply.github.com> Date: Sun, 31 May 2026 01:17:58 -0400 Subject: [PATCH 487/561] Fix: model fails to load when chat template uses HuggingFace generation tags (#2226) * chat-format: ignore HuggingFace's {% generation %} chat-template tag HuggingFace's transformers chat-template extension adds {% generation %} and {% endgeneration %} tags so trainers can mark generation spans for loss masking. The tags ship in GGUF tokenizer.chat_template metadata (SmolLM3 et al), but jinja2's default environment doesn't recognize them, so Llama() raises TemplateSyntaxError at init for any affected GGUF, even when the caller passes an explicit chat_format override. Register a minimal Jinja extension that treats both tags as inert wrappers: the body between them renders as-is, the markers themselves emit nothing. No behavioral change for templates that don't use the tags. Prior art: PR #2082 attempted the same approach but referenced an unimported 'nodes' module and didn't consume the body or closing tag. * fix: simplify generation tag handling * refactor: rename generation tag extension --------- Co-authored-by: abetlen <abetlen@gmail.com> --- CHANGELOG.md | 1 + llama_cpp/llama_chat_format.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be4ade61df..90d79abbfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat: Update llama.cpp to ggml-org/llama.cpp@d749821db +- fix: model fails to load when chat template uses HuggingFace generation tags by @tobocop2 in #2226 - docs: add contributing guide by @abetlen in #2229 - chore: Migrate llama.cpp submodule URL to ggml-org/llama.cpp by @shalinib-ibm in #2034 - fix: Enable unified KV cache for embedding contexts to preserve full per-sequence context in batch embedding calls by @SanjanaB123 in #2217 diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 1024fb85b9..f24b89f3ef 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -24,6 +24,7 @@ ) import jinja2 +from jinja2.ext import Extension from jinja2.sandbox import ImmutableSandboxedEnvironment import numpy as np @@ -192,6 +193,15 @@ def __call__( class Jinja2ChatFormatter(ChatFormatter): + class IgnoreGenerationTags(Extension): + """Pass-through for HuggingFace's ``{% generation %}`` chat-template tag.""" + + tags = {"generation"} + + def parse(self, parser: jinja2.parser.Parser): + parser.stream.skip(1) + return parser.parse_statements(("name:endgeneration",), drop_needle=True) + def __init__( self, template: str, @@ -213,6 +223,7 @@ def __init__( loader=jinja2.BaseLoader(), trim_blocks=True, lstrip_blocks=True, + extensions=[Jinja2ChatFormatter.IgnoreGenerationTags], ).from_string(self.template) @staticmethod From b91460b6518623cca1f9fa673abb542cefc6551c Mon Sep 17 00:00:00 2001 From: Alex O'Connell <35843486+acon96@users.noreply.github.com> Date: Sun, 31 May 2026 02:25:53 -0400 Subject: [PATCH 488/561] feat: enable arm64 musl builds (#2221) * enable arm64 musl builds * feat: enable arm64 musl builds * fix(ci): use cwd for arm64 wheel repair --------- Co-authored-by: abetlen <abetlen@gmail.com> --- .github/workflows/build-and-release.yaml | 4 ++-- CHANGELOG.md | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index df6201ee75..5cfb6469de 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -82,8 +82,8 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: - CIBW_SKIP: "*musllinux* pp*" - CIBW_REPAIR_WHEEL_COMMAND: "" + CIBW_SKIP: "pp*" + CIBW_REPAIR_WHEEL_COMMAND: "LD_LIBRARY_PATH=$PWD/llama_cpp/lib auditwheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: "aarch64" # Keep native arm64 builds on a portable CPU baseline instead of # tuning wheels to the hosted runner. diff --git a/CHANGELOG.md b/CHANGELOG.md index 90d79abbfa..7c7d20edcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: enable arm64 musl builds by @acon96 in #2221 - feat: Update llama.cpp to ggml-org/llama.cpp@d749821db - fix: model fails to load when chat template uses HuggingFace generation tags by @tobocop2 in #2226 - docs: add contributing guide by @abetlen in #2229 From 6bdab5d8b975783851659dae259e50849abf8a3b Mon Sep 17 00:00:00 2001 From: Tai An <antai12232931@outlook.com> Date: Sat, 30 May 2026 23:59:36 -0700 Subject: [PATCH 489/561] fix: suppress stdout and stderr in Jupyter notebooks (#2181) Co-authored-by: Anai-Guo <275560793+Anai-Guo@users.noreply.github.com> --- CHANGELOG.md | 1 + llama_cpp/_utils.py | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c7d20edcd..1d8cb4e102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: suppress stdout and stderr in Jupyter notebooks by @Anai-Guo in #2181 - feat: enable arm64 musl builds by @acon96 in #2221 - feat: Update llama.cpp to ggml-org/llama.cpp@d749821db - fix: model fails to load when chat template uses HuggingFace generation tags by @tobocop2 in #2226 diff --git a/llama_cpp/_utils.py b/llama_cpp/_utils.py index 29628193bb..08ede64205 100644 --- a/llama_cpp/_utils.py +++ b/llama_cpp/_utils.py @@ -34,6 +34,22 @@ def __enter__(self): self.old_stdout = self.sys.stdout self.old_stderr = self.sys.stderr + # In Jupyter notebooks, ipykernel replaces sys.stdout/stderr with + # OutStream objects that hold their own copy of the original fd in + # _original_stdstream_copy. This bypasses our dup2 redirect, so we + # need to point that copy at the real fd temporarily. + # https://github.com/ipython/ipykernel/blob/912923542d55e6c80c0e7c1f94c648b52a225011/ipykernel/iostream.py#L618-L622 + self._saved_stdout_copy = getattr( + self.sys.stdout, "_original_stdstream_copy", None + ) + self._saved_stderr_copy = getattr( + self.sys.stderr, "_original_stdstream_copy", None + ) + if self._saved_stdout_copy is not None: + self.sys.stdout._original_stdstream_copy = self.old_stdout_fileno_undup + if self._saved_stderr_copy is not None: + self.sys.stderr._original_stdstream_copy = self.old_stderr_fileno_undup + self.os.dup2(outnull_file.fileno(), self.old_stdout_fileno_undup) self.os.dup2(errnull_file.fileno(), self.old_stderr_fileno_undup) @@ -45,7 +61,6 @@ def __exit__(self, *_): if self.disable: return - # Check if sys.stdout and sys.stderr have fileno method self.sys.stdout = self.old_stdout self.sys.stderr = self.old_stderr @@ -55,6 +70,12 @@ def __exit__(self, *_): self.os.close(self.old_stdout_fileno) self.os.close(self.old_stderr_fileno) + # Restore ipykernel's OutStream fd copies + if self._saved_stdout_copy is not None: + self.sys.stdout._original_stdstream_copy = self._saved_stdout_copy + if self._saved_stderr_copy is not None: + self.sys.stderr._original_stdstream_copy = self._saved_stderr_copy + class MetaSingleton(type): """ From fdf38b3e4c41e42146f70dd8971d018339c08606 Mon Sep 17 00:00:00 2001 From: usernames122 <88596366+usernames122@users.noreply.github.com> Date: Sun, 31 May 2026 13:39:18 +0200 Subject: [PATCH 490/561] fix: avoid cleanup errors for partially initialized LlamaModel (#2173) * Add attribute check for sampler in close method This solves a bug I uncovered, that causes an AttributeError if constantly re-initializing a model in a loop and Python garbage collects it, such as testing the highest GPU layer count you can go before CUDA OOMs. * fix: avoid cleanup errors for partial model init --------- Co-authored-by: abetlen <abetlen@gmail.com> --- CHANGELOG.md | 1 + llama_cpp/_internals.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d8cb4e102..a0eb9da889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: avoid cleanup errors for partially initialized `LlamaModel` objects by @usernames122 in #2173 - fix: suppress stdout and stderr in Jupyter notebooks by @Anai-Guo in #2181 - feat: enable arm64 musl builds by @acon96 in #2221 - feat: Update llama.cpp to ggml-org/llama.cpp@d749821db diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 24f6fddc71..b0fe94d01f 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -44,6 +44,9 @@ def __init__( self.params = params self.verbose = verbose self._exit_stack = ExitStack() + # LlamaModel does not use samplers, but close() can run after partial init. + self.sampler = None + self.custom_samplers = [] model = None @@ -65,7 +68,6 @@ def __init__( self.model = model self.vocab = vocab - self.sampler = None # LlamaModel doesn't use samplers, but some cleanup code expects this attribute def free_model(): if self.model is None: From e8ee64b2338c185f972ad2ca3eabfb88507eaf78 Mon Sep 17 00:00:00 2001 From: Joshua Turner <jturner@handshape.com> Date: Mon, 1 Jun 2026 00:55:31 +0000 Subject: [PATCH 491/561] feat: add Jinja2 loop controls to chat templates (#2018) * Add support for jinja break for Cohere * docs: reference transformers chat template extensions --------- Co-authored-by: Andrei <abetlen@gmail.com> --- CHANGELOG.md | 1 + llama_cpp/llama_chat_format.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0eb9da889..ba3f748f56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: add Jinja2 loop controls to chat templates by @handshape in #2018 - fix: avoid cleanup errors for partially initialized `LlamaModel` objects by @usernames122 in #2173 - fix: suppress stdout and stderr in Jupyter notebooks by @Anai-Guo in #2181 - feat: enable arm64 musl builds by @acon96 in #2221 diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index f24b89f3ef..ed51b728cf 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -223,7 +223,12 @@ def __init__( loader=jinja2.BaseLoader(), trim_blocks=True, lstrip_blocks=True, - extensions=[Jinja2ChatFormatter.IgnoreGenerationTags], + # Keep this aligned with Transformers' chat-template Jinja extensions. + # https://github.com/huggingface/transformers/blob/39603d0e5cdb6f00e8d473d7fcbb01032d709181/src/transformers/utils/chat_template_utils.py#L489-L490 + extensions=[ + Jinja2ChatFormatter.IgnoreGenerationTags, + jinja2.ext.loopcontrols, + ], ).from_string(self.template) @staticmethod From 584802083d88264d11eb360f7c5a471e9f511ba8 Mon Sep 17 00:00:00 2001 From: navratil-matej <62650355+navratil-matej@users.noreply.github.com> Date: Mon, 1 Jun 2026 03:24:21 +0200 Subject: [PATCH 492/561] fix: use env var configured multimodal library override paths when loading shared libraries (#1782) * Fix LLAVA_CPP_LIB creating empty path The LLAVA_CPP_LIB environmental variable was correctly checked for, but if found, an empty path was created. * docs: update multimodal override changelog entry --------- Co-authored-by: Andrei <abetlen@gmail.com> --- CHANGELOG.md | 1 + llama_cpp/llava_cpp.py | 2 +- llama_cpp/mtmd_cpp.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba3f748f56..db9c0bcabd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: use env var configured multimodal library override paths when loading shared libraries by @navratil-matej in #1782 - feat: add Jinja2 loop controls to chat templates by @handshape in #2018 - fix: avoid cleanup errors for partially initialized `LlamaModel` objects by @usernames122 in #2173 - fix: suppress stdout and stderr in Jupyter notebooks by @Anai-Guo in #2181 diff --git a/llama_cpp/llava_cpp.py b/llama_cpp/llava_cpp.py index 3dc96d2f25..ea9b216f7d 100644 --- a/llama_cpp/llava_cpp.py +++ b/llama_cpp/llava_cpp.py @@ -39,7 +39,7 @@ _libllava_base_path = ( pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _libllava_override_path is None - else pathlib.Path() + else pathlib.Path(_libllava_override_path) ) # Load the library diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index f2b0ed2dea..8c2ebbfef4 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -43,7 +43,7 @@ _libmtmd_base_path = ( pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _libmtmd_override_path is None - else pathlib.Path() + else pathlib.Path(_libmtmd_override_path) ) # Load the library From 84bc143b3755d04e7552019d93c1532f9baf0428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= <sigbjorn.skjaeret@scala.com> Date: Mon, 1 Jun 2026 03:54:07 +0200 Subject: [PATCH 493/561] fix: match Transformers `tojson` in chat template rendering (#1486) * Render chat template tojson filter as unicode * replace tojson with custom filter * add indent parameter * add break, continue and strftime * renamed to strftime_now * add separators and sort_keys plus import fix * allow ensure_ascii parameter as a few templates are using it now --------- Co-authored-by: abetlen <abetlen@gmail.com> --- CHANGELOG.md | 1 + llama_cpp/llama_chat_format.py | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db9c0bcabd..51a0abad85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: match Transformers `tojson` in chat template rendering by @CISC in #1486 - fix: use env var configured multimodal library override paths when loading shared libraries by @navratil-matej in #1782 - feat: add Jinja2 loop controls to chat templates by @handshape in #2018 - fix: avoid cleanup errors for partially initialized `LlamaModel` objects by @usernames122 in #2173 diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index ed51b728cf..0b68d3a279 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -219,7 +219,7 @@ def __init__( set(stop_token_ids) if stop_token_ids is not None else None ) - self._environment = ImmutableSandboxedEnvironment( + environment = ImmutableSandboxedEnvironment( loader=jinja2.BaseLoader(), trim_blocks=True, lstrip_blocks=True, @@ -229,12 +229,32 @@ def __init__( Jinja2ChatFormatter.IgnoreGenerationTags, jinja2.ext.loopcontrols, ], - ).from_string(self.template) + ) + # Match Transformers' chat-template JSON rendering behavior. + # https://github.com/huggingface/transformers/blob/39603d0e5cdb6f00e8d473d7fcbb01032d709181/src/transformers/utils/chat_template_utils.py#L481-L484 + environment.filters["tojson"] = self.tojson + self._environment = environment.from_string(self.template) @staticmethod def strftime_now(f: str) -> str: return datetime.now().strftime(f) + @staticmethod + def tojson( + x: Any, + ensure_ascii: bool = False, + indent: Optional[int] = None, + separators: Optional[Tuple[str, str]] = None, + sort_keys: bool = False, + ) -> str: + return json.dumps( + x, + ensure_ascii=ensure_ascii, + indent=indent, + separators=separators, + sort_keys=sort_keys, + ) + def __call__( self, *, From 33bf9d2480351d28d4a5e21626bf29f35a2e9838 Mon Sep 17 00:00:00 2001 From: thecaptain789 <257642323+thecaptain789@users.noreply.github.com> Date: Mon, 1 Jun 2026 03:07:34 +0100 Subject: [PATCH 494/561] fix: correct typo in comments and settings description (#2121) Co-authored-by: thecaptain789 <thecaptain789@users.noreply.github.com> --- llama_cpp/llama_chat_format.py | 2 +- llama_cpp/server/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 0b68d3a279..b7f6916eac 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3267,7 +3267,7 @@ def from_pretrained( class ObsidianChatHandler(Llava15ChatHandler): # Prompt Format - # The model followed ChatML format. However, with ### as the seperator + # The model followed ChatML format. However, with ### as the separator # <|im_start|>user # What is this sign about?\n<image> diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 3c2bb7fd07..78dd7cdeb8 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -60,7 +60,7 @@ class ModelSettings(BaseSettings): ) rpc_servers: Optional[str] = Field( default=None, - description="comma seperated list of rpc servers for offloading", + description="comma separated list of rpc servers for offloading", ) # Context Params seed: int = Field( From 73ee7cd5671c07be371447ce2a1d648de0aab775 Mon Sep 17 00:00:00 2001 From: Victor Oluwadare <111367022+Victoran0@users.noreply.github.com> Date: Mon, 1 Jun 2026 03:11:23 +0100 Subject: [PATCH 495/561] fix(docs): remove double word typo in README (#1791) Under the Chat Completion section, about the fourth line under it (line 340), we have: 'The model will will format the messages into a single prompt using the following order of precedence:'... 'will' appeared twice. This update fix that. Co-authored-by: Andrei <abetlen@gmail.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a785cbb030..c1247a2e2e 100644 --- a/README.md +++ b/README.md @@ -337,7 +337,7 @@ The high-level API also provides a simple interface for chat completion. Chat completion requires that the model knows how to format the messages into a single prompt. The `Llama` class does this using pre-registered chat formats (ie. `chatml`, `llama-2`, `gemma`, etc) or by providing a custom chat handler object. -The model will will format the messages into a single prompt using the following order of precedence: +The model will format the messages into a single prompt using the following order of precedence: - Use the `chat_handler` if provided - Use the `chat_format` if provided - Use the `tokenizer.chat_template` from the `gguf` model's metadata (should work for most new models, older models may not have this) From cdb7a755c63dce6f2bc31cfbcfee269e7fcb67c7 Mon Sep 17 00:00:00 2001 From: avion23 <10906412+avion23@users.noreply.github.com> Date: Mon, 1 Jun 2026 10:06:45 +0700 Subject: [PATCH 496/561] fix: clear prompt for recurrent / hybrid models when only a partial prefix matches (#2108) Co-authored-by: Ralf Waldukat <ralf.waldukat@gmail.com> --- .github/workflows/test.yaml | 20 +++++-- CHANGELOG.md | 1 + llama_cpp/llama.py | 22 ++++++++ tests/test_llama.py | 106 ++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8a6845ff24..b8f5566bb0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,6 +10,11 @@ on: env: REPO_ID: lmstudio-community/Qwen3.5-0.8B-GGUF MODEL_FILE: Qwen3.5-0.8B-Q8_0.gguf + RECURRENT_REPO_ID: QuantFactory/mamba-130m-hf-GGUF + RECURRENT_MODEL_FILE: mamba-130m-hf.Q2_K.gguf + HYBRID_REPO_ID: tiiuae/Falcon-H1-Tiny-90M-Instruct-GGUF + HYBRID_MODEL_FILE: Falcon-H1-Tiny-90M-Instruct-Q2_K.gguf + MODEL_CACHE_KEY: qwen35-q8-mamba130m-q2-falconh1tiny-q2 jobs: download-model: @@ -22,12 +27,15 @@ jobs: - name: Install huggingface-hub run: pip install huggingface-hub - name: Download model - run: hf download ${{ env.REPO_ID }} ${{ env.MODEL_FILE }} + run: | + hf download ${{ env.REPO_ID }} ${{ env.MODEL_FILE }} + hf download ${{ env.RECURRENT_REPO_ID }} ${{ env.RECURRENT_MODEL_FILE }} + hf download ${{ env.HYBRID_REPO_ID }} ${{ env.HYBRID_MODEL_FILE }} - name: Cache model uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub - key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} + key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} build-linux: needs: download-model @@ -49,7 +57,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub - key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} + key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} - name: Install dependencies (Linux/MacOS) run: | python -m pip install --upgrade pip @@ -81,7 +89,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub - key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} + key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} - name: Install dependencies (Windows) run: | @@ -121,7 +129,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub - key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} + key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} - name: Install dependencies (Linux/MacOS) run: | @@ -157,7 +165,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/huggingface/hub - key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }} + key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} - name: Install dependencies run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 51a0abad85..21fa49f28e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: clear prompt for recurrent / hybrid models when only a partial prefix matches by @avion23 in #2108 - fix: match Transformers `tojson` in chat template rendering by @CISC in #1486 - fix: use env var configured multimodal library override paths when loading shared libraries by @navratil-matej in #1782 - feat: add Jinja2 loop controls to chat templates by @handshape in #2018 diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 75c74b41fc..b904b4080c 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -559,6 +559,10 @@ def free_lora_adapter(): self._sampler = None + # Cache recurrent/hybrid model detection to avoid repeated FFI calls + self._is_recurrent = llama_cpp.llama_model_is_recurrent(self._model.model) + self._is_hybrid = llama_cpp.llama_model_is_hybrid(self._model.model) + @property def ctx(self) -> llama_cpp.llama_context_p: return self._ctx.ctx @@ -644,6 +648,11 @@ def reset(self): """Reset the model state.""" self.n_tokens = 0 + if self._is_recurrent or self._is_hybrid: + mem = llama_cpp.llama_get_memory(self._ctx.ctx) + if mem is not None: + llama_cpp.llama_memory_clear(mem, True) + def eval(self, tokens: Sequence[int]): """Evaluate a list of tokens. @@ -899,6 +908,19 @@ def generate( longest_prefix += 1 else: break + + # Recurrent and hybrid models cannot rewind state; reset if needed + if ( + self._is_recurrent or self._is_hybrid + ) and longest_prefix < self.n_tokens: + longest_prefix = 0 + reset = True + if self.verbose: + print( + "Llama.generate: recurrent/hybrid model requires full state reset", + file=sys.stderr, + ) + if longest_prefix > 0: if self._ctx.kv_cache_seq_rm(-1, longest_prefix, -1): reset = False diff --git a/tests/test_llama.py b/tests/test_llama.py index d4e6031c70..336d6a6122 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -72,6 +72,22 @@ def llama_cpp_embedding_model_path(): return model_path +@pytest.fixture +def llama_cpp_recurrent_model_path(): + repo_id = "QuantFactory/mamba-130m-hf-GGUF" + filename = "mamba-130m-hf.Q2_K.gguf" + model_path = hf_hub_download(repo_id, filename) + return model_path + + +@pytest.fixture +def llama_cpp_hybrid_model_path(): + repo_id = "tiiuae/Falcon-H1-Tiny-90M-Instruct-GGUF" + filename = "Falcon-H1-Tiny-90M-Instruct-Q2_K.gguf" + model_path = hf_hub_download(repo_id, filename) + return model_path + + def test_real_model(llama_cpp_model_path): import os @@ -233,6 +249,96 @@ def logit_processor_func(input_ids, logits): assert number_1 == number_3 +def test_real_llama_repeated_prompt_cache(llama_cpp_model_path): + model = llama_cpp.Llama( + llama_cpp_model_path, + n_ctx=32, + n_batch=32, + n_ubatch=32, + n_threads=multiprocessing.cpu_count(), + n_threads_batch=multiprocessing.cpu_count(), + logits_all=False, + flash_attn=True, + verbose=False, + ) + prompt = "The quick brown fox jumps over the lazy dog. The quick brown fox" + + output_1 = model.create_completion( + prompt, + max_tokens=6, + temperature=0.0, + seed=1337, + ) + output_2 = model.create_completion( + prompt, + max_tokens=6, + temperature=0.0, + seed=1337, + ) + + assert output_1["choices"][0]["text"] == " jumps over the lazy dog." + assert output_2["choices"][0]["text"] == output_1["choices"][0]["text"] + + +def _assert_prompt_cache_reset_handles_history_edit( + model_path, + *, + is_recurrent: bool, + is_hybrid: bool, +): + model = llama_cpp.Llama( + model_path, + n_ctx=32, + n_batch=32, + n_ubatch=32, + n_threads=multiprocessing.cpu_count(), + n_threads_batch=multiprocessing.cpu_count(), + logits_all=False, + verbose=False, + ) + + assert model._is_recurrent is is_recurrent + assert model._is_hybrid is is_hybrid + + first_prompt = "The quick brown fox" + second_prompt = "The slow brown fox" + first_tokens = model.tokenize(first_prompt.encode(), add_bos=True, special=True) + second_tokens = model.tokenize(second_prompt.encode(), add_bos=True, special=True) + + assert first_tokens != second_tokens + assert first_tokens[0] == second_tokens[0] + + first_output = model.create_completion( + first_prompt, + max_tokens=1, + temperature=0.0, + ) + assert isinstance(first_output["choices"][0]["text"], str) + + second_output = model.create_completion( + second_prompt, + max_tokens=1, + temperature=0.0, + ) + assert isinstance(second_output["choices"][0]["text"], str) + + +def test_recurrent_model_prompt_cache_reset(llama_cpp_recurrent_model_path): + _assert_prompt_cache_reset_handles_history_edit( + llama_cpp_recurrent_model_path, + is_recurrent=True, + is_hybrid=False, + ) + + +def test_hybrid_model_prompt_cache_reset(llama_cpp_hybrid_model_path): + _assert_prompt_cache_reset_handles_history_edit( + llama_cpp_hybrid_model_path, + is_recurrent=False, + is_hybrid=True, + ) + + def test_real_llama_embeddings(llama_cpp_embedding_model_path): model = llama_cpp.Llama( llama_cpp_embedding_model_path, From e3aa6b5cde98e8b719aaa3bbab31a5ed72cd6805 Mon Sep 17 00:00:00 2001 From: Imad Saddik <79410781+ImadSaddik@users.noreply.github.com> Date: Mon, 1 Jun 2026 04:11:00 +0100 Subject: [PATCH 497/561] docs: fix typo in README (#2072) docs: fix typo in README (#2072) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c1247a2e2e..b42db2ef5e 100644 --- a/README.md +++ b/README.md @@ -269,7 +269,7 @@ To upgrade and rebuild `llama-cpp-python` add `--upgrade --force-reinstall --no- The high-level API provides a simple managed interface through the [`Llama`](https://llama-cpp-python.readthedocs.io/en/latest/api-reference/#llama_cpp.Llama) class. -Below is a short example demonstrating how to use the high-level API to for basic text completion: +Below is a short example demonstrating how to use the high-level API for basic text completion: ```python from llama_cpp import Llama From 868712246065f6c52cd4b050474d51ed352c1398 Mon Sep 17 00:00:00 2001 From: Stefano Fiorucci <stefanofiorucci@gmail.com> Date: Mon, 1 Jun 2026 05:18:08 +0200 Subject: [PATCH 498/561] docs: fix NanoLlava chat handler name in README (#2059) docs: fix NanoLlava chat handler name in README (#2059) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b42db2ef5e..d48dde3c58 100644 --- a/README.md +++ b/README.md @@ -502,7 +502,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [llava-v1.5-13b](https://huggingface.co/mys/ggml_llava-v1.5-13b) | `Llava15ChatHandler` | `llava-1-5` | | [llava-v1.6-34b](https://huggingface.co/cjpais/llava-v1.6-34B-gguf) | `Llava16ChatHandler` | `llava-1-6` | | [moondream2](https://huggingface.co/vikhyatk/moondream2) | `MoondreamChatHandler` | `moondream2` | -| [nanollava](https://huggingface.co/abetlen/nanollava-gguf) | `NanollavaChatHandler` | `nanollava` | +| [nanollava](https://huggingface.co/abetlen/nanollava-gguf) | `NanoLlavaChatHandler` | `nanollava` | | [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha-gguf) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | | [minicpm-v-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `MiniCPMv26ChatHandler` | `minicpm-v-2.6` | | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | From da07e463e19913071e5e9ea1d7c05b97c5531fa0 Mon Sep 17 00:00:00 2001 From: Matthias <dev@sleepyyui.com> Date: Mon, 1 Jun 2026 05:26:23 +0200 Subject: [PATCH 499/561] docs: update llama.cpp build docs link (#2056) docs: update llama.cpp build docs link (#2056) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d48dde3c58..9b563f0654 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ pip install llama-cpp-python \ ### Installation Configuration -`llama.cpp` supports a number of hardware acceleration backends to speed up inference as well as backend specific options. See the [llama.cpp README](https://github.com/ggerganov/llama.cpp#build) for a full list. +`llama.cpp` supports a number of hardware acceleration backends to speed up inference as well as backend specific options. See the [llama.cpp README](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md) for a full list. All `llama.cpp` cmake build options can be set via the `CMAKE_ARGS` environment variable or via the `--config-settings / -C` cli flag during installation. From 52cf7475f987f08f18dc5748f601b32e7b603ffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Gr=C3=B6nholm?= <alex.gronholm@nextday.fi> Date: Mon, 1 Jun 2026 06:30:37 +0300 Subject: [PATCH 500/561] docs: update ROCm install instructions (#1867) docs: update ROCm install instructions (#1867) --- CHANGELOG.md | 1 + README.md | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21fa49f28e..197841bf39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- docs: update ROCm install instructions by @agronholm in #1867 - fix: clear prompt for recurrent / hybrid models when only a partial prefix matches by @avion23 in #2108 - fix: match Transformers `tojson` in chat template rendering by @CISC in #1486 - fix: use env var configured multimodal library override paths when loading shared libraries by @navratil-matej in #1782 diff --git a/README.md b/README.md index 9b563f0654..a64c7faa86 100644 --- a/README.md +++ b/README.md @@ -173,12 +173,12 @@ pip install llama-cpp-python \ </details> <details> -<summary>hipBLAS (ROCm)</summary> +<summary>HIP (ROCm)</summary> -To install with hipBLAS / ROCm support for AMD cards, set the `GGML_HIPBLAS=on` environment variable before installing: +To install with HIP / ROCm support for AMD cards, set the `GGML_HIP=on` environment variable before installing: ```bash -CMAKE_ARGS="-DGGML_HIPBLAS=on" pip install llama-cpp-python +CMAKE_ARGS="-DGGML_HIP=on" pip install llama-cpp-python ``` </details> From c3adb354e71ec964dd503a5aae81d16ca947e377 Mon Sep 17 00:00:00 2001 From: adam jones <domdomegg+git@gmail.com> Date: Mon, 1 Jun 2026 06:00:53 +0100 Subject: [PATCH 501/561] server types: Move 'model' parameter to clarify it is used (#1786) The 'model' parameter has been supported since https://github.com/abetlen/llama-cpp-python/pull/931. Its placement in this section was copied from an older version of the file, and hasn't been corrected since. Correcting this will make it clearer what parameters are supported by llama-cpp-python. --- llama_cpp/server/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/server/types.py b/llama_cpp/server/types.py index fdd1644568..eda2b92801 100644 --- a/llama_cpp/server/types.py +++ b/llama_cpp/server/types.py @@ -107,6 +107,7 @@ class CreateCompletionRequest(BaseModel): + model: Optional[str] = model_field prompt: Union[str, List[str]] = Field( default="", description="The prompt to generate completions for." ) @@ -138,7 +139,6 @@ class CreateCompletionRequest(BaseModel): seed: Optional[int] = Field(None) # ignored or currently unsupported - model: Optional[str] = model_field n: Optional[int] = 1 best_of: Optional[int] = 1 user: Optional[str] = Field(default=None) @@ -190,6 +190,7 @@ class ChatCompletionRequestMessage(BaseModel): class CreateChatCompletionRequest(BaseModel): + model: Optional[str] = model_field messages: List[llama_cpp.ChatCompletionRequestMessage] = Field( default=[], description="A list of messages to generate completions for." ) @@ -237,7 +238,6 @@ class CreateChatCompletionRequest(BaseModel): ) # ignored or currently unsupported - model: Optional[str] = model_field n: Optional[int] = 1 user: Optional[str] = Field(None) From 20240609f3dcc902dfa0c59b675ad24887bc044c Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 31 May 2026 22:44:22 -0700 Subject: [PATCH 502/561] feat: update llama.cpp to af6528e6d (#2235) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 197841bf39..90adbe54c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - fix: avoid cleanup errors for partially initialized `LlamaModel` objects by @usernames122 in #2173 - fix: suppress stdout and stderr in Jupyter notebooks by @Anai-Guo in #2181 - feat: enable arm64 musl builds by @acon96 in #2221 -- feat: Update llama.cpp to ggml-org/llama.cpp@d749821db +- feat: Update llama.cpp to ggml-org/llama.cpp@af6528e6d - fix: model fails to load when chat template uses HuggingFace generation tags by @tobocop2 in #2226 - docs: add contributing guide by @abetlen in #2229 - chore: Migrate llama.cpp submodule URL to ggml-org/llama.cpp by @shalinib-ibm in #2034 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index d749821db3..af6528e6df 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit d749821db3bd587932d1ed57d43626cd552c9909 +Subproject commit af6528e6df5d798f7f1363ec1141699be0f638e2 From 26633bd1a2eaf7fd0567cc5eaec8b0165a7ea0bd Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 31 May 2026 22:57:26 -0700 Subject: [PATCH 503/561] chore: bump version to 0.3.24 (#2236) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90adbe54c8..6d02bb8e32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.24] + - docs: update ROCm install instructions by @agronholm in #1867 - fix: clear prompt for recurrent / hybrid models when only a partial prefix matches by @avion23 in #2108 - fix: match Transformers `tojson` in chat template rendering by @CISC in #1486 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index eb37da2093..c48e6fcec0 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.23" +__version__ = "0.3.24" From c7af423ea641dd1e5fa316516027f6effc375ef6 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 1 Jun 2026 02:25:03 -0700 Subject: [PATCH 504/561] fix(ci): add Pascal compute capability targets to CUDA wheel builds (#2237) --- .github/workflows/build-wheels-cuda.yaml | 4 ++-- CHANGELOG.md | 2 ++ README.md | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index be55bf4834..87bd365957 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -169,10 +169,10 @@ jobs: } $cudaTagVersion = $nvccVersion.Replace('.','') $env:VERBOSE = '1' - # Build real cubins for the supported GPUs, including sm_70, and keep + # Build real cubins for the supported GPUs, including Pascal, and keep # one forward-compatible PTX target instead of embedding PTX for every # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit. - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler $env:CMAKE_ARGS" + $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=60-real;61-real;70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler $env:CMAKE_ARGS" $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' python -m build --wheel # Publish tags that reflect the actual installed toolkit version. diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d02bb8e32..9144353983 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(ci): add Pascal compute capability targets to CUDA wheel builds by @abetlen in #2237 + ## [0.3.24] - docs: update ROCm install instructions by @agronholm in #1867 diff --git a/README.md b/README.md index a64c7faa86..7e02b174d4 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements: - CUDA Version is 12.1, 12.2, 12.3, 12.4 or 12.5 +- NVIDIA GPU compute capability is 6.0 or newer - Python Version is 3.10, 3.11 or 3.12 ```bash From 43c92a7fef5cf6588612525e081f39c2adf094ec Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 1 Jun 2026 04:30:02 -0700 Subject: [PATCH 505/561] feat(ci): add CUDA 11.8 wheel builds (#2238) * feat(ci): add CUDA 11.8 wheel builds * fix(ci): make CUDA 11.8 wheel builds version-consistent * fix(ci): allow non-CUDA dependencies for CUDA 11.8 wheels * fix(ci): omit Hopper targets from CUDA 11.8 wheels * fix(ci): use GCC 11 for CUDA 11.8 Linux wheels * fix(ci): use MSVC 14.29 for CUDA 11.8 Windows wheels --- .github/workflows/build-wheels-cuda.yaml | 51 +++++++++++++++++++++--- CHANGELOG.md | 1 + README.md | 5 ++- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 87bd365957..723236ca82 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -24,7 +24,7 @@ jobs: # wheel.py-api = "py3" makes the CUDA wheel interpreter-agnostic, # so one builder per toolkit version is sufficient. 'pyver' = @("3.9") - 'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1") + 'cuda' = @("11.8.0", "12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1") 'releasetag' = @("basic") 'exclude' = @( @{ 'os' = 'windows-2022'; 'cuda' = '12.1.1' }, @@ -50,8 +50,15 @@ jobs: AVXVER: ${{ matrix.releasetag }} steps: + - name: Set up MSVC for CUDA 11.8 + if: runner.os == 'Windows' && matrix.cuda == '11.8.0' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + toolset: 14.29 + - name: Set up MSVC - if: runner.os == 'Windows' + if: runner.os == 'Windows' && matrix.cuda != '11.8.0' uses: ilammy/msvc-dev-cmd@v1 with: arch: x64 @@ -81,7 +88,31 @@ jobs: run: | $cudaVersion = $env:CUDAVER $cudaChannel = "nvidia/label/cuda-$cudaVersion" - if ($IsLinux) { + if ($cudaVersion -eq '11.8.0') { + if ($IsLinux) { + $cudaPackages = @( + "${cudaChannel}::cuda-nvcc_linux-64=11.8.0", + "${cudaChannel}::cuda-cccl=11.8.89", + "${cudaChannel}::cuda-cudart=11.8.89", + "${cudaChannel}::cuda-cudart-dev=11.8.89", + "${cudaChannel}::cuda-driver-dev=11.8.89", + "${cudaChannel}::libcublas=11.11.3.6", + "${cudaChannel}::libcublas-dev=11.11.3.6" + ) + } elseif ($IsWindows) { + $cudaPackages = @( + "${cudaChannel}::cuda-nvcc_win-64=11.8.0", + "${cudaChannel}::cuda-cccl=11.8.89", + "${cudaChannel}::cuda-cudart=11.8.89", + "${cudaChannel}::cuda-cudart-dev=11.8.89", + "${cudaChannel}::libcublas=11.11.3.6", + "${cudaChannel}::libcublas-dev=11.11.3.6" + ) + } else { + throw 'Unsupported CUDA wheel build platform' + } + mamba install -y --channel-priority flexible --override-channels -c $cudaChannel $cudaPackages + } elseif ($IsLinux) { mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_linux-64" "${cudaChannel}::cuda-cccl" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" } elseif ($IsWindows) { if ($cudaVersion -like '12.5.*') { @@ -122,7 +153,12 @@ jobs: $cudaRootCmake = $cudaRoot.Replace('\', '/') $env:CMAKE_ARGS = "-DCUDAToolkit_ROOT=$cudaRootCmake -DCUDA_TOOLKIT_ROOT_DIR=$cudaRootCmake" if ($IsLinux) { - if (Test-Path '/usr/bin/g++-12') { + if ([version]$env:CUDAVER -lt [version]"12.0" -and (Test-Path '/usr/bin/g++-11')) { + $env:CC = '/usr/bin/gcc-11' + $env:CXX = '/usr/bin/g++-11' + $env:CUDAHOSTCXX = '/usr/bin/g++-11' + $cudaHostCompilerArg = " -DCMAKE_CUDA_HOST_COMPILER=$env:CUDAHOSTCXX" + } elseif (Test-Path '/usr/bin/g++-12') { $env:CC = '/usr/bin/gcc-12' $env:CXX = '/usr/bin/g++-12' $env:CUDAHOSTCXX = '/usr/bin/g++-12' @@ -169,10 +205,15 @@ jobs: } $cudaTagVersion = $nvccVersion.Replace('.','') $env:VERBOSE = '1' + $cudaArchs = "60-real;61-real;70-real;75-real;80-real;86-real;89-real;90-real;90-virtual" + if ([version]$nvccVersion -lt [version]"12.0") { + # CUDA 11.8 cannot compile llama.cpp's Hopper PDL device calls. + $cudaArchs = "60-real;61-real;70-real;75-real;80-real;86-real;89-real" + } # Build real cubins for the supported GPUs, including Pascal, and keep # one forward-compatible PTX target instead of embedding PTX for every # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit. - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=60-real;61-real;70-real;75-real;80-real;86-real;89-real;90-real;90-virtual -DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler $env:CMAKE_ARGS" + $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=$cudaArchs -DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler $env:CMAKE_ARGS" $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' python -m build --wheel # Publish tags that reflect the actual installed toolkit version. diff --git a/CHANGELOG.md b/CHANGELOG.md index 9144353983..e2fb8e951c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat(ci): add CUDA 11.8 wheel builds by @abetlen in #2238 - fix(ci): add Pascal compute capability targets to CUDA wheel builds by @abetlen in #2237 ## [0.3.24] diff --git a/README.md b/README.md index 7e02b174d4..2b7a7d98c9 100644 --- a/README.md +++ b/README.md @@ -125,8 +125,8 @@ CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements: -- CUDA Version is 12.1, 12.2, 12.3, 12.4 or 12.5 -- NVIDIA GPU compute capability is 6.0 or newer +- CUDA Version is 11.8, 12.1, 12.2, 12.3, 12.4 or 12.5 +- NVIDIA GPU compute capability is 6.0 through 8.9 for CUDA 11.8 wheels, or 6.0 or newer for CUDA 12 wheels - Python Version is 3.10, 3.11 or 3.12 ```bash @@ -135,6 +135,7 @@ pip install llama-cpp-python \ ``` Where `<cuda-version>` is one of the following: +- `cu118`: CUDA 11.8 - `cu121`: CUDA 12.1 - `cu122`: CUDA 12.2 - `cu123`: CUDA 12.3 From 718a1ca5f17522655ae482f182f63ea3ea41d15d Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 1 Jun 2026 05:44:30 -0700 Subject: [PATCH 506/561] feat(ci): add CUDA 13 wheel builds (#2239) --- .github/workflows/build-wheels-cuda.yaml | 11 +++++++---- CHANGELOG.md | 1 + README.md | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 723236ca82..f1b2b8b6b5 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -24,7 +24,7 @@ jobs: # wheel.py-api = "py3" makes the CUDA wheel interpreter-agnostic, # so one builder per toolkit version is sufficient. 'pyver' = @("3.9") - 'cuda' = @("11.8.0", "12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1") + 'cuda' = @("11.8.0", "12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1", "13.0.2", "13.2.1") 'releasetag' = @("basic") 'exclude' = @( @{ 'os' = 'windows-2022'; 'cuda' = '12.1.1' }, @@ -115,8 +115,8 @@ jobs: } elseif ($IsLinux) { mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-toolkit=$cudaVersion" "${cudaChannel}::cuda-nvcc_linux-64" "${cudaChannel}::cuda-cccl" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" } elseif ($IsWindows) { - if ($cudaVersion -like '12.5.*') { - # The Windows 12.5 toolkit meta-package pulls compiler activation + if ($cudaVersion -like '12.5.*' -or [version]$cudaVersion -ge [version]"13.0") { + # The Windows 12.5+ toolkit meta-package pulls compiler activation # scripts that overflow cmd.exe after MSVC is already initialized. mamba install -y --channel-priority flexible --override-channels -c $cudaChannel "${cudaChannel}::cuda-nvcc_win-64" "${cudaChannel}::cuda-cccl" "${cudaChannel}::cuda-libraries-dev=$cudaVersion" "${cudaChannel}::cuda-cudart" "${cudaChannel}::cuda-cudart-dev" } else { @@ -209,8 +209,11 @@ jobs: if ([version]$nvccVersion -lt [version]"12.0") { # CUDA 11.8 cannot compile llama.cpp's Hopper PDL device calls. $cudaArchs = "60-real;61-real;70-real;75-real;80-real;86-real;89-real" + } elseif ([version]$nvccVersion -ge [version]"13.0") { + # CUDA 13 dropped offline compilation support for pre-Turing targets. + $cudaArchs = "75-real;80-real;86-real;89-real;90-real;90-virtual" } - # Build real cubins for the supported GPUs, including Pascal, and keep + # Build real cubins for the supported GPUs and keep # one forward-compatible PTX target instead of embedding PTX for every # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit. $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=$cudaArchs -DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler $env:CMAKE_ARGS" diff --git a/CHANGELOG.md b/CHANGELOG.md index e2fb8e951c..c80984ff68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat(ci): add CUDA 13.0 and 13.2 wheel builds by @abetlen in #2239 - feat(ci): add CUDA 11.8 wheel builds by @abetlen in #2238 - fix(ci): add Pascal compute capability targets to CUDA wheel builds by @abetlen in #2237 diff --git a/README.md b/README.md index 2b7a7d98c9..5de330af46 100644 --- a/README.md +++ b/README.md @@ -125,8 +125,8 @@ CMAKE_ARGS="-DGGML_CUDA=on" pip install llama-cpp-python It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements: -- CUDA Version is 11.8, 12.1, 12.2, 12.3, 12.4 or 12.5 -- NVIDIA GPU compute capability is 6.0 through 8.9 for CUDA 11.8 wheels, or 6.0 or newer for CUDA 12 wheels +- CUDA Version is 11.8, 12.1, 12.2, 12.3, 12.4, 12.5, 13.0 or 13.2 +- NVIDIA GPU compute capability is 6.0 through 8.9 for CUDA 11.8 wheels, 6.0 or newer for CUDA 12 wheels, or 7.5 or newer for CUDA 13 wheels - Python Version is 3.10, 3.11 or 3.12 ```bash @@ -141,6 +141,8 @@ Where `<cuda-version>` is one of the following: - `cu123`: CUDA 12.3 - `cu124`: CUDA 12.4 - `cu125`: CUDA 12.5 +- `cu130`: CUDA 13.0 +- `cu132`: CUDA 13.2 For example, to install the CUDA 12.1 wheel: From 927b574ede274d3277ddc1786f0bf25d36e1cd60 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 1 Jun 2026 05:52:07 -0700 Subject: [PATCH 507/561] docs: add Python 3.14 classifier (#2240) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b5998dd1c8..6bfacf279c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] From a9b480f8b1ac20279a563d12c5d0f3567c0af960 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 1 Jun 2026 06:57:20 -0700 Subject: [PATCH 508/561] feat: add Gemma 4 multimodal chat support (#2241) --- CHANGELOG.md | 1 + README.md | 1 + llama_cpp/llama_chat_format.py | 44 ++++++++++++++++++++++++++++++++++ llama_cpp/server/model.py | 14 +++++++++++ 4 files changed, 60 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c80984ff68..2ecb2aa17a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: add Gemma 4 multimodal chat support by @abetlen in #2241 - feat(ci): add CUDA 13.0 and 13.2 wheel builds by @abetlen in #2239 - feat(ci): add CUDA 11.8 wheel builds by @abetlen in #2238 - fix(ci): add Pascal compute capability targets to CUDA wheel builds by @abetlen in #2237 diff --git a/README.md b/README.md index 5de330af46..7db3e27448 100644 --- a/README.md +++ b/README.md @@ -510,6 +510,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha-gguf) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | | [minicpm-v-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `MiniCPMv26ChatHandler` | `minicpm-v-2.6` | | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | +| [gemma-4](https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF) | `Gemma4ChatHandler` | `gemma4` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index b7f6916eac..44c6c1f76f 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3265,6 +3265,50 @@ def from_pretrained( ) +class Gemma4ChatHandler(Llava15ChatHandler): + DEFAULT_SYSTEM_MESSAGE = None + + CHAT_FORMAT = ( + "{% if messages and messages[0]['role'] == 'system' %}" + "{% if messages[0]['content'] is string %}" + "{% set first_user_prefix = messages[0]['content'] + '\n\n' %}" + "{% else %}" + "{% set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' %}" + "{% endif %}" + "{% set loop_messages = messages[1:] %}" + "{% else %}" + "{% set first_user_prefix = '' %}" + "{% set loop_messages = messages %}" + "{% endif %}" + "{% for message in loop_messages %}" + "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}" + "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}" + "{% endif %}" + "{% set role = 'model' if message['role'] == 'assistant' else message['role'] %}" + "{{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else '') }}" + "{% if message['content'] is string %}" + "{{ message['content'] | trim }}" + "{% elif message['content'] is iterable %}" + "{% for item in message['content'] %}" + "{% if item['type'] == 'image_url' and item['image_url'] is string %}" + "{{ '\n\n' + item['image_url'] + '\n\n' }}" + "{% elif item['type'] == 'image_url' and item['image_url'] is mapping %}" + "{{ '\n\n' + item['image_url']['url'] + '\n\n' }}" + "{% elif item['type'] == 'text' %}" + "{{ item['text'] | trim }}" + "{% endif %}" + "{% endfor %}" + "{% else %}" + "{{ raise_exception('Invalid content type') }}" + "{% endif %}" + "{{ '<end_of_turn>\n' }}" + "{% endfor %}" + "{% if add_generation_prompt %}" + "{{ '<start_of_turn>model\n' }}" + "{% endif %}" + ) + + class ObsidianChatHandler(Llava15ChatHandler): # Prompt Format # The model followed ChatML format. However, with ### as the separator diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 3922ce5df3..3222abd631 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -115,6 +115,20 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler = llama_cpp.llama_chat_format.Llava16ChatHandler( clip_model_path=settings.clip_model_path, verbose=settings.verbose ) + elif settings.chat_format == "gemma4": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.Gemma4ChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.Gemma4ChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) elif settings.chat_format == "moondream": assert settings.clip_model_path is not None, "clip model not found" if settings.hf_model_repo_id is not None: From 4b66c45ebc2065798f68f531376a7a60c92f942d Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 1 Jun 2026 19:32:51 -0700 Subject: [PATCH 509/561] feat: update llama.cpp to 210a6570c (#2242) --- CHANGELOG.md | 1 + llama_cpp/llama_cpp.py | 4 ++++ vendor/llama.cpp | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ecb2aa17a..7fa15aebb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: Update llama.cpp to ggml-org/llama.cpp@210a6570c by @abetlen in #2242 - feat: add Gemma 4 multimodal chat support by @abetlen in #2241 - feat(ci): add CUDA 13.0 and 13.2 wheel builds by @abetlen in #2239 - feat(ci): add CUDA 11.8 wheel builds by @abetlen in #2238 diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 6560b51787..44c25a519f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -903,6 +903,7 @@ class llama_sampler_seq_config(ctypes.Structure): # uint32_t n_ubatch; // physical maximum batch size # uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) # uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] +# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) # int32_t n_threads; // number of threads to use for generation # int32_t n_threads_batch; // number of threads to use for batch processing @@ -958,6 +959,7 @@ class llama_context_params(ctypes.Structure): n_ubatch (int): physical maximum batch size n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) n_rs_seq (int): number of recurrent-state snapshots per sequence for rollback + n_outputs_max (int): max outputs in a ubatch, 0 = n_batch n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing ctx_type (int): context type, from `enum llama_context_type` @@ -995,6 +997,7 @@ class llama_context_params(ctypes.Structure): n_ubatch: int n_seq_max: int n_rs_seq: int + n_outputs_max: int n_threads: int n_threads_batch: int ctx_type: int @@ -1031,6 +1034,7 @@ class llama_context_params(ctypes.Structure): ("n_ubatch", ctypes.c_uint32), ("n_seq_max", ctypes.c_uint32), ("n_rs_seq", ctypes.c_uint32), + ("n_outputs_max", ctypes.c_uint32), ("n_threads", ctypes.c_int32), ("n_threads_batch", ctypes.c_int32), ("ctx_type", ctypes.c_int), diff --git a/vendor/llama.cpp b/vendor/llama.cpp index af6528e6df..210a6570ce 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit af6528e6df5d798f7f1363ec1141699be0f638e2 +Subproject commit 210a6570ceda20c5d6439172c09ada08c3754cc9 From f1bfa117410fbd75ad6a2345a65cdf9b17205c19 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 1 Jun 2026 21:46:08 -0700 Subject: [PATCH 510/561] chore: bump version to 0.3.25 (#2243) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fa15aebb4..bd83be8434 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.25] + - feat: Update llama.cpp to ggml-org/llama.cpp@210a6570c by @abetlen in #2242 - feat: add Gemma 4 multimodal chat support by @abetlen in #2241 - feat(ci): add CUDA 13.0 and 13.2 wheel builds by @abetlen in #2239 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index c48e6fcec0..52101c9b7e 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.24" +__version__ = "0.3.25" From d185d64aa6de50acea2e2cdfdc12bcf4a721f946 Mon Sep 17 00:00:00 2001 From: Markus <markus@xpiriax.de> Date: Tue, 2 Jun 2026 13:41:57 +0200 Subject: [PATCH 511/561] fix: handle additional `from_pretrained` files in subfolders (#2085) * Fixed issue #1938 Now using the correct path * fix: handle additional from_pretrained files in subfolders * docs: update changelog for from_pretrained subfolder fix --------- Co-authored-by: abetlen <abetlen@gmail.com> --- CHANGELOG.md | 2 ++ llama_cpp/llama.py | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd83be8434..6677fc8fe0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: handle additional `from_pretrained` files in subfolders by @TNing in #2085 + ## [0.3.25] - feat: Update llama.cpp to ggml-org/llama.cpp@210a6570c by @abetlen in #2242 diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b904b4080c..4a09b55ee5 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2396,12 +2396,15 @@ def from_pretrained( ) (matching_additional_file,) = matching_additional_files + additional_subfolder = str(Path(matching_additional_file).parent) + additional_file_name = Path(matching_additional_file).name - # download the additional file + # Split additional file paths independently to avoid duplicating + # the main model subfolder in Hugging Face download URLs. hf_hub_download( repo_id=repo_id, - filename=matching_additional_file, - subfolder=subfolder, + filename=additional_file_name, + subfolder=additional_subfolder, local_dir=local_dir, local_dir_use_symlinks=local_dir_use_symlinks, cache_dir=cache_dir, From bbdc8518720322ee4e805b9e375fd65a5948c9c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 01:40:49 -0700 Subject: [PATCH 512/561] chore(deps): bump pypa/cibuildwheel from 2.22.0 to 3.4.1 (#2249) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.22.0 to 3.4.1. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.22...v3.4.1) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-version: 3.4.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 6 +++--- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 5cfb6469de..6318f73a1e 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -42,7 +42,7 @@ jobs: shell: cmd - name: Build wheels - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v3.4.1 env: # Keep repair disabled by default for non-Linux platforms in this job. CIBW_REPAIR_WHEEL_COMMAND: "" @@ -80,7 +80,7 @@ jobs: submodules: "recursive" - name: Build wheels - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v3.4.1 env: CIBW_SKIP: "pp*" CIBW_REPAIR_WHEEL_COMMAND: "LD_LIBRARY_PATH=$PWD/llama_cpp/lib auditwheel repair -w {dest_dir} {wheel}" @@ -133,7 +133,7 @@ jobs: platforms: linux/riscv64 - name: Build wheels - uses: pypa/cibuildwheel@v3.1.2 + uses: pypa/cibuildwheel@v3.4.1 env: CIBW_SKIP: "*musllinux* pp*" CIBW_REPAIR_WHEEL_COMMAND: "" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 98f511e4a6..5fd56a00d3 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -32,7 +32,7 @@ jobs: shell: bash - name: Build wheels - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v3.4.1 env: # disable repair CIBW_REPAIR_WHEEL_COMMAND: "" From dad5d0aacb4bfb0080da3188050d4fabd97d8e50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 02:03:43 -0700 Subject: [PATCH 513/561] chore(deps): bump actions/cache from 4 to 5 (#2248) Bumps [actions/cache](https://github.com/actions/cache) from 4 to 5. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/cache dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b8f5566bb0..83a91a8edd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,7 +32,7 @@ jobs: hf download ${{ env.RECURRENT_REPO_ID }} ${{ env.RECURRENT_MODEL_FILE }} hf download ${{ env.HYBRID_REPO_ID }} ${{ env.HYBRID_MODEL_FILE }} - name: Cache model - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} @@ -54,7 +54,7 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Restore model cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} @@ -86,7 +86,7 @@ jobs: cache: 'pip' - name: Restore model cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} @@ -126,7 +126,7 @@ jobs: python3 -c "import platform; print(platform.machine(), platform.architecture())" - name: Restore model cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} @@ -162,7 +162,7 @@ jobs: python3 -c "import platform; print(platform.machine(), platform.architecture())" - name: Restore model cache - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/.cache/huggingface/hub key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} From aa944e4cb09337f381281e5880e257f57fd7abc5 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Wed, 3 Jun 2026 02:26:38 -0700 Subject: [PATCH 514/561] ci: cache embedding test model (#2250) * ci: cache embedding test model * ci: restore previous model cache --- .github/workflows/test.yaml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 83a91a8edd..d798967675 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,7 +14,9 @@ env: RECURRENT_MODEL_FILE: mamba-130m-hf.Q2_K.gguf HYBRID_REPO_ID: tiiuae/Falcon-H1-Tiny-90M-Instruct-GGUF HYBRID_MODEL_FILE: Falcon-H1-Tiny-90M-Instruct-Q2_K.gguf - MODEL_CACHE_KEY: qwen35-q8-mamba130m-q2-falconh1tiny-q2 + EMBEDDING_REPO_ID: CompendiumLabs/bge-small-en-v1.5-gguf + EMBEDDING_MODEL_FILE: bge-small-en-v1.5-q4_k_m.gguf + MODEL_CACHE_KEY: qwen35-q8-mamba130m-q2-falconh1tiny-q2-bge-small-q4 jobs: download-model: @@ -26,16 +28,19 @@ jobs: python-version: "3.9" - name: Install huggingface-hub run: pip install huggingface-hub + - name: Restore model cache + uses: actions/cache@v5 + with: + path: ~/.cache/huggingface/hub + key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} + restore-keys: | + ${{ runner.os }}-model-qwen35-q8-mamba130m-q2-falconh1tiny-q2 - name: Download model run: | hf download ${{ env.REPO_ID }} ${{ env.MODEL_FILE }} hf download ${{ env.RECURRENT_REPO_ID }} ${{ env.RECURRENT_MODEL_FILE }} hf download ${{ env.HYBRID_REPO_ID }} ${{ env.HYBRID_MODEL_FILE }} - - name: Cache model - uses: actions/cache@v5 - with: - path: ~/.cache/huggingface/hub - key: ${{ runner.os }}-model-${{ env.MODEL_CACHE_KEY }} + hf download ${{ env.EMBEDDING_REPO_ID }} ${{ env.EMBEDDING_MODEL_FILE }} build-linux: needs: download-model From b439a84a61ea257538d0660afaf512425297cba3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 02:56:25 -0700 Subject: [PATCH 515/561] chore(deps): bump actions/upload-artifact from 4 to 7 (#2245) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 7. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v7) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-and-release.yaml | 8 ++++---- .github/workflows/build-wheels-metal.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 6318f73a1e..9978cb66b2 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -66,7 +66,7 @@ jobs: package-dir: . output-dir: wheelhouse - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: wheels-${{ matrix.os }} path: ./wheelhouse/*.whl @@ -95,7 +95,7 @@ jobs: output-dir: wheelhouse - name: Upload wheels as artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: wheels_arm64 path: ./wheelhouse/*.whl @@ -148,7 +148,7 @@ jobs: output-dir: wheelhouse - name: Upload wheels as artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ${{ matrix.shard.artifact }} path: ./wheelhouse/*.whl @@ -190,7 +190,7 @@ jobs: run: | python -m build --sdist - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: sdist path: ./dist/*.tar.gz diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 5fd56a00d3..5f5eba6854 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -43,7 +43,7 @@ jobs: package-dir: . output-dir: wheelhouse2 - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: wheels-mac_${{ matrix.os }} path: ./wheelhouse2/*.whl From f8bd67df8a9844d6400383a8941ef129401586a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 02:57:00 -0700 Subject: [PATCH 516/561] chore(deps): bump docker/setup-buildx-action from 3 to 4 (#2246) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3 to 4. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index c656958479..3dfef68008 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -29,7 +29,7 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry uses: docker/login-action@v3 From 6e6c4e6db734c2b28e9ca11ade5527f2ba4f0c1e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 03:05:10 -0700 Subject: [PATCH 517/561] chore(deps): bump actions/setup-python from 5 to 6 (#2247) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/build-and-release.yaml | 4 ++-- .github/workflows/build-wheels-cuda.yaml | 2 +- .github/workflows/build-wheels-metal.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/publish-to-test.yaml | 2 +- .github/workflows/publish.yaml | 2 +- .github/workflows/test-pypi.yaml | 6 +++--- .github/workflows/test.yaml | 10 +++++----- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 9978cb66b2..81c9a961b4 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -19,7 +19,7 @@ jobs: submodules: "recursive" # Used to host cibuildwheel - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.9" @@ -162,7 +162,7 @@ jobs: with: submodules: "recursive" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.9" diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index f1b2b8b6b5..3e67d14cba 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -67,7 +67,7 @@ jobs: with: submodules: "recursive" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.pyver }} cache: 'pip' diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 5f5eba6854..bf2c8bc124 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -19,7 +19,7 @@ jobs: submodules: "recursive" # Used to host cibuildwheel - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.12" cache: 'pip' diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 8b3e6322dc..3c6f5ff45e 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" diff --git a/.github/workflows/publish-to-test.yaml b/.github/workflows/publish-to-test.yaml index de3ae42aae..54572bdad9 100644 --- a/.github/workflows/publish-to-test.yaml +++ b/.github/workflows/publish-to-test.yaml @@ -21,7 +21,7 @@ jobs: submodules: "recursive" - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" cache: 'pip' diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index bb76f53941..3c2ea56d68 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -15,7 +15,7 @@ jobs: submodules: "recursive" - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.9" diff --git a/.github/workflows/test-pypi.yaml b/.github/workflows/test-pypi.yaml index 335033bba6..416cd0cddf 100644 --- a/.github/workflows/test-pypi.yaml +++ b/.github/workflows/test-pypi.yaml @@ -12,7 +12,7 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -48,7 +48,7 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -84,7 +84,7 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: 'pip' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d798967675..ecba1b40b0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.9" - name: Install huggingface-hub @@ -54,7 +54,7 @@ jobs: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -85,7 +85,7 @@ jobs: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -119,7 +119,7 @@ jobs: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -156,7 +156,7 @@ jobs: submodules: "recursive" - name: Set up Python 3.9 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.9" From ab7a9b0c864cc82bdcfb7cae911c0bfc606883ba Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Wed, 3 Jun 2026 04:35:33 -0700 Subject: [PATCH 518/561] feat(ci): add Vulkan wheel builds (#2251) * feat(ci): add Vulkan wheel builds * docs: update changelog for Vulkan wheel builds * docs: add Vulkan wheel install instructions --- .github/workflows/build-wheels-vulkan.yaml | 123 ++++++++++++++++++ .../generate-index-from-release.yaml | 3 +- CHANGELOG.md | 1 + README.md | 9 ++ 4 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-wheels-vulkan.yaml diff --git a/.github/workflows/build-wheels-vulkan.yaml b/.github/workflows/build-wheels-vulkan.yaml new file mode 100644 index 0000000000..760205c839 --- /dev/null +++ b/.github/workflows/build-wheels-vulkan.yaml @@ -0,0 +1,123 @@ +name: Build Wheels (Vulkan) + +on: workflow_dispatch + +permissions: + contents: write + +env: + VULKAN_SDK_VERSION: "1.4.341.0" + VULKAN_SDK_LINUX_SHA256: "ed66477d587a5587dc3601b1c2cdcc1fab5529c505f53a00171876cecd9b4fbe" + +jobs: + build_wheels: + name: Build Vulkan wheel on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + pyver: "3.9" + artifact: wheels-vulkan-ubuntu-22.04 + - os: windows-2022 + pyver: "3.9" + artifact: wheels-vulkan-windows-2022 + + steps: + - name: Set up MSVC + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.pyver }} + cache: "pip" + + - name: Install Vulkan SDK + if: runner.os == 'Linux' + run: | + curl -fL \ + "https://sdk.lunarg.com/sdk/download/${VULKAN_SDK_VERSION}/linux/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.xz" \ + -o vulkan-sdk.tar.xz + echo "${VULKAN_SDK_LINUX_SHA256} vulkan-sdk.tar.xz" | sha256sum -c - + mkdir -p "$RUNNER_TEMP/vulkan-sdk" + tar -xf vulkan-sdk.tar.xz -C "$RUNNER_TEMP/vulkan-sdk" + source "$RUNNER_TEMP/vulkan-sdk/${VULKAN_SDK_VERSION}/setup-env.sh" + { + echo "VULKAN_SDK=$VULKAN_SDK" + echo "LD_LIBRARY_PATH=$VULKAN_SDK/lib:${LD_LIBRARY_PATH:-}" + } >> "$GITHUB_ENV" + echo "$VULKAN_SDK/bin" >> "$GITHUB_PATH" + "$VULKAN_SDK/bin/glslc" --version + + - name: Install Vulkan SDK + if: runner.os == 'Windows' + shell: pwsh + run: | + choco install vulkan-sdk --version="$env:VULKAN_SDK_VERSION" --no-progress -y + $vulkanSdk = Join-Path 'C:\VulkanSDK' $env:VULKAN_SDK_VERSION + if (-not (Test-Path $vulkanSdk)) { + throw "Failed to find Vulkan SDK at $vulkanSdk" + } + "VULKAN_SDK=$vulkanSdk" >> $env:GITHUB_ENV + "$vulkanSdk\Bin" >> $env:GITHUB_PATH + & "$vulkanSdk\Bin\glslc.exe" --version + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build wheel + + - name: Install Windows build dependencies + if: runner.os == 'Windows' + run: python -m pip install ninja + + - name: Build Vulkan wheel + if: runner.os == 'Linux' + run: | + export CMAKE_ARGS="-DGGML_NATIVE=off -DGGML_METAL=OFF -DGGML_VULKAN=on" + python -m build --wheel + mkdir -p wheelhouse + cp dist/*.whl wheelhouse/ + + - name: Build Vulkan wheel + if: runner.os == 'Windows' + shell: pwsh + run: | + $env:CMAKE_GENERATOR = 'Ninja' + $env:CMAKE_ARGS = '-DGGML_NATIVE=off -DGGML_VULKAN=on' + python -m build --wheel + New-Item -ItemType Directory -Force wheelhouse | Out-Null + Copy-Item dist/*.whl wheelhouse/ + + - uses: actions/upload-artifact@v7 + with: + name: ${{ matrix.artifact }} + path: ./wheelhouse/*.whl + + release: + name: Release + needs: [build_wheels] + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: dist + + - uses: softprops/action-gh-release@v2 + with: + files: dist/* + # Set release name to <tag>-vulkan. + tag_name: ${{ github.ref_name }}-vulkan + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 255ee67d6f..7fc19e697f 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -3,7 +3,7 @@ name: Wheels Index on: # Trigger on new release workflow_run: - workflows: ["Release", "Build Wheels (CUDA)", "Build Wheels (Metal)"] + workflows: ["Release", "Build Wheels (CUDA)", "Build Wheels (Metal)", "Build Wheels (Vulkan)"] types: - completed @@ -46,6 +46,7 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' # ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' # ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + ./scripts/releases-to-pep-503.sh index/whl/vulkan '^[v]?[0-9]+\.[0-9]+\.[0-9]+-vulkan$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact uses: actions/upload-pages-artifact@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6677fc8fe0..1e2a180fd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat(ci): add Vulkan wheel builds by @abetlen in #2251 - fix: handle additional `from_pretrained` files in subfolders by @TNing in #2085 ## [0.3.25] diff --git a/README.md b/README.md index 7db3e27448..57ede08768 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,15 @@ To install with Vulkan support, set the `GGML_VULKAN=on` environment variable be CMAKE_ARGS="-DGGML_VULKAN=on" pip install llama-cpp-python ``` +**Pre-built Wheel (New)** + +It is also possible to install a pre-built wheel with Vulkan support for Linux or Windows: + +```bash +pip install llama-cpp-python \ + --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/vulkan +``` + </details> <details> From 3754c04a168ce35a3e5bbe8f600014fc70970415 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Wed, 3 Jun 2026 04:37:24 -0700 Subject: [PATCH 519/561] feat(ci): add ROCm wheel builds (#2252) * feat(ci): add ROCm wheel builds * docs: update changelog for ROCm wheel builds * docs: add ROCm wheel install instructions --- .github/workflows/build-wheels-rocm.yaml | 237 ++++++++++++++++++ .../generate-index-from-release.yaml | 4 +- CHANGELOG.md | 1 + README.md | 16 ++ 4 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-wheels-rocm.yaml diff --git a/.github/workflows/build-wheels-rocm.yaml b/.github/workflows/build-wheels-rocm.yaml new file mode 100644 index 0000000000..0971953886 --- /dev/null +++ b/.github/workflows/build-wheels-rocm.yaml @@ -0,0 +1,237 @@ +name: Build Wheels (ROCm) + +on: workflow_dispatch + +permissions: + contents: write + +jobs: + build_wheels: + name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ROCm ${{ matrix.rocm }} + runs-on: ${{ matrix.os }} + container: + image: rocm/dev-ubuntu-22.04:${{ matrix.rocm }}-complete + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-22.04 + pyver: "3.9" + rocm: "7.2.4" + amdgpu_targets: gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1200;gfx1201 + + steps: + - name: Install system dependencies + run: | + apt-get update + apt-get install -y --no-install-recommends git cmake lsb-release ninja-build + + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.pyver }} + cache: "pip" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build wheel + + - name: Build ROCm wheel + run: | + export ROCM_PATH="${ROCM_PATH:-/opt/rocm}" + export HIP_PATH="${HIP_PATH:-$ROCM_PATH}" + export PATH="$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:$PATH" + export LD_LIBRARY_PATH="$ROCM_PATH/lib:$ROCM_PATH/lib64:${LD_LIBRARY_PATH:-}" + export CC="$ROCM_PATH/llvm/bin/clang" + export CXX="$ROCM_PATH/llvm/bin/clang++" + export HIPCXX="$ROCM_PATH/llvm/bin/clang" + export CMAKE_GENERATOR=Ninja + + hipconfig --version + hipcc --version + + rocm_tag="$(hipconfig --version | sed -E 's/^([0-9]+)\.([0-9]+).*/\1\2/')" + echo "ROCM_VERSION=$rocm_tag" >> "$GITHUB_ENV" + + amdgpu_targets="${{ matrix.amdgpu_targets }}" + export CMAKE_ARGS="-DGGML_HIP=on -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off -DAMDGPU_TARGETS=$amdgpu_targets -DCMAKE_HIP_ARCHITECTURES=$amdgpu_targets" + python -m build --wheel + mkdir -p wheelhouse + cp dist/*.whl wheelhouse/ + + - uses: actions/upload-artifact@v7 + with: + name: wheels-rocm${{ env.ROCM_VERSION }}-${{ matrix.os }} + path: ./wheelhouse/*.whl + + build_wheels_windows_hip: + name: Build Wheel windows-2022 ${{ matrix.pyver }} HIP ${{ matrix.name }} + runs-on: windows-2022 + env: + HIPSDK_INSTALLER_VERSION: "26.Q1" + strategy: + fail-fast: false + matrix: + include: + - name: radeon + pyver: "3.9" + amdgpu_targets: gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032 + + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.pyver }} + cache: "pip" + + - name: Set up MSVC + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + python -m pip install build wheel + + - name: Grab rocWMMA package + run: | + curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/7.2.1/pool/main/r/rocwmma-dev/rocwmma-dev_2.2.0.70201-81~24.04_amd64.deb" + 7z x rocwmma.deb + 7z x data.tar + + - name: Cache ROCm installation + id: cache-rocm + uses: actions/cache@v5 + with: + path: C:\Program Files\AMD\ROCm + key: cache-gha-rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }} + + - name: Install ROCm + if: steps.cache-rocm.outputs.cache-hit != 'true' + run: | + $ErrorActionPreference = "Stop" + Invoke-WebRequest ` + -Uri "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-${{ env.HIPSDK_INSTALLER_VERSION }}-Win11-For-HIP.exe" ` + -OutFile "${env:RUNNER_TEMP}\rocm-install.exe" + $proc = Start-Process "${env:RUNNER_TEMP}\rocm-install.exe" -ArgumentList '-install' -NoNewWindow -PassThru + $completed = $proc.WaitForExit(1800000) + if (-not $completed) { + $proc.Kill() + throw "ROCm installation timed out after 30 minutes" + } + if ($proc.ExitCode -ne 0) { + throw "ROCm installation failed with exit code $($proc.ExitCode)" + } + + - name: Verify ROCm + run: | + $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1 + if (-not $clangPath) { + throw "ROCm installation not found" + } + & $clangPath.FullName --version + + - name: Build HIP wheel + run: | + $ErrorActionPreference = "Stop" + $hipPath = Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Split-Path | Split-Path + $rocwmmaInclude = (Join-Path $PWD 'opt\rocm-7.2.1\include').Replace('\', '/') + $amdgpuTargets = "${{ matrix.amdgpu_targets }}" + + $env:HIP_PATH = $hipPath + $env:ROCM_PATH = $hipPath + $env:CMAKE_PREFIX_PATH = $hipPath + $env:HIP_PLATFORM = 'amd' + $env:PATH = "$hipPath\bin;$env:PATH" + $env:CC = "$hipPath\bin\clang.exe" + $env:CXX = "$hipPath\bin\clang++.exe" + $env:HIPCXX = "$hipPath\bin\clang.exe" + $env:CMAKE_GENERATOR = 'Unix Makefiles' + $env:CXXFLAGS = "-I$rocwmmaInclude -Wno-ignored-attributes -Wno-nested-anon-types" + $env:CMAKE_ARGS = "-DGGML_HIP=ON -DGGML_HIP_ROCWMMA_FATTN=ON -DGGML_NATIVE=OFF -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DGGML_F16C=OFF -DGPU_TARGETS=$amdgpuTargets" + + python -m build --wheel + + - name: Bundle ROCm runtime DLLs + run: | + $ErrorActionPreference = "Stop" + $hipPath = Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Split-Path | Split-Path + $wheel = Get-ChildItem dist\*.whl | Select-Object -First 1 + python -m wheel unpack $wheel.FullName -d wheel-unpacked + $wheelRoot = Get-ChildItem wheel-unpacked -Directory | Select-Object -First 1 + $libDir = Join-Path $wheelRoot.FullName 'llama_cpp\lib' + New-Item -ItemType Directory -Force $libDir | Out-Null + + $dllPatterns = @( + 'amdhip64.dll', + 'hiprtc*.dll', + 'libhipblas.dll', + 'libhipblaslt.dll', + 'rocblas.dll' + ) + foreach ($pattern in $dllPatterns) { + Copy-Item (Join-Path $hipPath "bin\$pattern") $libDir -ErrorAction SilentlyContinue + } + + New-Item -ItemType Directory -Force (Join-Path $libDir 'rocblas\library') | Out-Null + New-Item -ItemType Directory -Force (Join-Path $libDir 'hipblaslt\library') | Out-Null + Copy-Item "$hipPath\bin\rocblas\library\*" (Join-Path $libDir 'rocblas\library') -Recurse -Force + Copy-Item "$hipPath\bin\hipblaslt\library\*" (Join-Path $libDir 'hipblaslt\library') -Recurse -Force + + Remove-Item dist\*.whl + python -m wheel pack $wheelRoot.FullName -d dist + New-Item -ItemType Directory -Force wheelhouse | Out-Null + Copy-Item dist/*.whl wheelhouse/ + + - uses: actions/upload-artifact@v7 + with: + name: wheels-hip-${{ matrix.name }}-windows-2022 + path: ./wheelhouse/*.whl + + release_rocm: + name: Release ROCm + needs: [build_wheels] + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: dist + + - uses: softprops/action-gh-release@v2 + with: + files: dist/* + # Set release name to <tag>-rocm<rocm_version>. + tag_name: ${{ github.ref_name }}-rocm72 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + release_hip: + name: Release HIP + needs: [build_wheels_windows_hip] + if: startsWith(github.ref, 'refs/tags/') + runs-on: ubuntu-latest + + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: dist + + - uses: softprops/action-gh-release@v2 + with: + files: dist/* + # Set release name to <tag>-hip-radeon. + tag_name: ${{ github.ref_name }}-hip-radeon + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 7fc19e697f..c93e0be351 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -3,7 +3,7 @@ name: Wheels Index on: # Trigger on new release workflow_run: - workflows: ["Release", "Build Wheels (CUDA)", "Build Wheels (Metal)", "Build Wheels (Vulkan)"] + workflows: ["Release", "Build Wheels (CUDA)", "Build Wheels (Metal)", "Build Wheels (Vulkan)", "Build Wheels (ROCm)"] types: - completed @@ -46,6 +46,8 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' # ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' # ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + ./scripts/releases-to-pep-503.sh index/whl/rocm72 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-rocm72$' + ./scripts/releases-to-pep-503.sh index/whl/hip-radeon '^[v]?[0-9]+\.[0-9]+\.[0-9]+-hip-radeon$' ./scripts/releases-to-pep-503.sh index/whl/vulkan '^[v]?[0-9]+\.[0-9]+\.[0-9]+-vulkan$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e2a180fd9..f46b697c66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat(ci): add ROCm wheel builds by @abetlen in #2252 - feat(ci): add Vulkan wheel builds by @abetlen in #2251 - fix: handle additional `from_pretrained` files in subfolders by @TNing in #2085 diff --git a/README.md b/README.md index 57ede08768..dd0024676a 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,22 @@ To install with HIP / ROCm support for AMD cards, set the `GGML_HIP=on` environm CMAKE_ARGS="-DGGML_HIP=on" pip install llama-cpp-python ``` +**Pre-built Wheel (New)** + +It is also possible to install a pre-built wheel with ROCm support for Linux: + +```bash +pip install llama-cpp-python \ + --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/rocm72 +``` + +Or a pre-built wheel with HIP Radeon support for Windows: + +```powershell +pip install llama-cpp-python ` + --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/hip-radeon +``` + </details> <details> From ddaac1033590941c25ce367d3af0768c4f5d5a0b Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Wed, 3 Jun 2026 04:50:45 -0700 Subject: [PATCH 520/561] feat: update llama.cpp (#2253) * feat: update llama.cpp * docs: update changelog for llama.cpp update --- CHANGELOG.md | 1 + vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f46b697c66..462f0d0c18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@3571fa543 by @abetlen in #2253 - feat(ci): add ROCm wheel builds by @abetlen in #2252 - feat(ci): add Vulkan wheel builds by @abetlen in #2251 - fix: handle additional `from_pretrained` files in subfolders by @TNing in #2085 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 210a6570ce..3571fa5435 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 210a6570ceda20c5d6439172c09ada08c3754cc9 +Subproject commit 3571fa5435ac9ff243662b1caabc407e8d433c9d From 8d2d2699c89a5b4ac9eb929753cbf3ba5f544c0e Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Wed, 3 Jun 2026 11:20:40 -0700 Subject: [PATCH 521/561] feat: update llama.cpp (#2254) * feat: update llama.cpp * docs: update changelog for llama.cpp update * docs: consolidate llama.cpp changelog entry --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 462f0d0c18..72b122408c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: update llama.cpp to ggml-org/llama.cpp@3571fa543 by @abetlen in #2253 +- feat: update llama.cpp to ggml-org/llama.cpp@166fe2949 - feat(ci): add ROCm wheel builds by @abetlen in #2252 - feat(ci): add Vulkan wheel builds by @abetlen in #2251 - fix: handle additional `from_pretrained` files in subfolders by @TNing in #2085 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 3571fa5435..166fe29492 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 3571fa5435ac9ff243662b1caabc407e8d433c9d +Subproject commit 166fe29492abb4093ec889b5c6f6fdb4e3b8ba98 From cc2efc5843287fcd3e7f4cbd85a4f38eff3025fe Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Wed, 3 Jun 2026 12:55:01 -0700 Subject: [PATCH 522/561] feat: update llama.cpp to ggml-org/llama.cpp@94a220cd6 (#2255) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72b122408c..243c344c13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: update llama.cpp to ggml-org/llama.cpp@166fe2949 +- feat: update llama.cpp to ggml-org/llama.cpp@94a220cd6 - feat(ci): add ROCm wheel builds by @abetlen in #2252 - feat(ci): add Vulkan wheel builds by @abetlen in #2251 - fix: handle additional `from_pretrained` files in subfolders by @TNing in #2085 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 166fe29492..94a220cd67 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 166fe29492abb4093ec889b5c6f6fdb4e3b8ba98 +Subproject commit 94a220cd6745e6e3f8de62870b66fd5b9bc92700 From e2d148a9647b579465534576c8cda30a7816377e Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 03:15:25 -0700 Subject: [PATCH 523/561] feat: update llama.cpp to ggml-org/llama.cpp@e3ba22d6c (#2262) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 243c344c13..452b90ef72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: update llama.cpp to ggml-org/llama.cpp@94a220cd6 +- feat: update llama.cpp to ggml-org/llama.cpp@e3ba22d6c - feat(ci): add ROCm wheel builds by @abetlen in #2252 - feat(ci): add Vulkan wheel builds by @abetlen in #2251 - fix: handle additional `from_pretrained` files in subfolders by @TNing in #2085 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 94a220cd67..e3ba22d6cc 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 94a220cd6745e6e3f8de62870b66fd5b9bc92700 +Subproject commit e3ba22d6cc4dec84e59a909c7f96e1689c7384a9 From d0990015d3f985be1f594506717709b123fcfcd4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 03:23:49 -0700 Subject: [PATCH 524/561] chore(deps): bump actions/download-artifact from 4 to 8 (#2257) --- .github/workflows/build-and-release.yaml | 2 +- .github/workflows/build-wheels-metal.yaml | 2 +- .github/workflows/build-wheels-rocm.yaml | 4 ++-- .github/workflows/build-wheels-vulkan.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 81c9a961b4..3a61f75241 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -202,7 +202,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: merge-multiple: true path: dist diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index bf2c8bc124..83d1a31753 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -54,7 +54,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: merge-multiple: true path: dist2 diff --git a/.github/workflows/build-wheels-rocm.yaml b/.github/workflows/build-wheels-rocm.yaml index 0971953886..afb4fefd3d 100644 --- a/.github/workflows/build-wheels-rocm.yaml +++ b/.github/workflows/build-wheels-rocm.yaml @@ -203,7 +203,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: merge-multiple: true path: dist @@ -223,7 +223,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: merge-multiple: true path: dist diff --git a/.github/workflows/build-wheels-vulkan.yaml b/.github/workflows/build-wheels-vulkan.yaml index 760205c839..a42ee89b28 100644 --- a/.github/workflows/build-wheels-vulkan.yaml +++ b/.github/workflows/build-wheels-vulkan.yaml @@ -109,7 +109,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: merge-multiple: true path: dist From df45432b4bc778cc76f90b0e0530428cb0ca03bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 03:33:33 -0700 Subject: [PATCH 525/561] chore(deps): bump actions/checkout from 4 to 6 (#2258) Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/build-and-release.yaml | 8 ++++---- .github/workflows/build-docker.yaml | 2 +- .github/workflows/build-wheels-cuda.yaml | 2 +- .github/workflows/build-wheels-metal.yaml | 2 +- .github/workflows/build-wheels-rocm.yaml | 4 ++-- .github/workflows/build-wheels-vulkan.yaml | 2 +- .github/workflows/generate-index-from-release.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/publish-to-test.yaml | 2 +- .github/workflows/publish.yaml | 2 +- .github/workflows/test.yaml | 8 ++++---- 11 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 3a61f75241..0b974e93c8 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -14,7 +14,7 @@ jobs: os: [ubuntu-22.04, windows-2022, macos-14, macos-15] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" @@ -75,7 +75,7 @@ jobs: name: Build arm64 wheels runs-on: ubuntu-24.04-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" @@ -123,7 +123,7 @@ jobs: build: "cp314-*" artifact: wheels_riscv64_cp314 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" @@ -158,7 +158,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 3dfef68008..b547f32ff5 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 3e67d14cba..6106a08817 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -63,7 +63,7 @@ jobs: with: arch: x64 - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 83d1a31753..ca8e322230 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -14,7 +14,7 @@ jobs: os: [macos-14, macos-15] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-rocm.yaml b/.github/workflows/build-wheels-rocm.yaml index afb4fefd3d..792e33cf89 100644 --- a/.github/workflows/build-wheels-rocm.yaml +++ b/.github/workflows/build-wheels-rocm.yaml @@ -26,7 +26,7 @@ jobs: apt-get update apt-get install -y --no-install-recommends git cmake lsb-release ninja-build - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" @@ -82,7 +82,7 @@ jobs: amdgpu_targets: gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-vulkan.yaml b/.github/workflows/build-wheels-vulkan.yaml index a42ee89b28..49d38fb80b 100644 --- a/.github/workflows/build-wheels-vulkan.yaml +++ b/.github/workflows/build-wheels-vulkan.yaml @@ -31,7 +31,7 @@ jobs: with: arch: x64 - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index c93e0be351..a39776eb07 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Pages uses: actions/configure-pages@v5 - name: Build diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 3c6f5ff45e..ac39181f90 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -12,7 +12,7 @@ jobs: ruff: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/publish-to-test.yaml b/.github/workflows/publish-to-test.yaml index 54572bdad9..70ae5389af 100644 --- a/.github/workflows/publish-to-test.yaml +++ b/.github/workflows/publish-to-test.yaml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 3c2ea56d68..8908ccdd6f 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ecba1b40b0..82f7a8a7de 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -49,7 +49,7 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" @@ -80,7 +80,7 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" @@ -114,7 +114,7 @@ jobs: matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" @@ -151,7 +151,7 @@ jobs: needs: download-model runs-on: macos-15-intel steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: "recursive" From b46bccf4f9ce9cc48f464c1068f39963758d1213 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 03:33:54 -0700 Subject: [PATCH 526/561] chore(deps): bump docker/login-action from 3 to 4 (#2259) Bumps [docker/login-action](https://github.com/docker/login-action) from 3 to 4. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index b547f32ff5..349eb209dd 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -32,7 +32,7 @@ jobs: uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.repository_owner }} From 927dde2189fa820dc7e550c97cb8b1d02d6fd0bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 03:34:13 -0700 Subject: [PATCH 527/561] chore(deps): bump actions/deploy-pages from 4 to 5 (#2260) Bumps [actions/deploy-pages](https://github.com/actions/deploy-pages) from 4 to 5. - [Release notes](https://github.com/actions/deploy-pages/releases) - [Commits](https://github.com/actions/deploy-pages/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/deploy-pages dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/generate-index-from-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index a39776eb07..6b2c8770fa 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -57,4 +57,4 @@ jobs: path: 'index' - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@v5 From 9f6efb073c65749d67e904b61f4356c6261f4be5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 03:34:30 -0700 Subject: [PATCH 528/561] chore(deps): bump conda-incubator/setup-miniconda from 3.1.0 to 4.0.1 (#2261) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.1.0 to 4.0.1. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.1.0...v4.0.1) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-version: 4.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 6106a08817..f5b3718f17 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -73,7 +73,7 @@ jobs: cache: 'pip' - name: Setup Mamba - uses: conda-incubator/setup-miniconda@v3.1.0 + uses: conda-incubator/setup-miniconda@v4.0.1 with: activate-environment: "llamacpp" python-version: ${{ matrix.pyver }} From 2dae4770db6a2c34bf3a5f7f76b4d4bc8045bfd0 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 04:14:05 -0700 Subject: [PATCH 529/561] feat: Generic Multimodal Chat Handler (#2256) --- CHANGELOG.md | 1 + README.md | 1 + llama_cpp/llama_chat_format.py | 546 ++++++++++++++++++++++++++++++--- llama_cpp/server/model.py | 19 +- 4 files changed, 518 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 452b90ef72..18bcf258a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: Generic Multimodal Chat Handler by @abetlen in #2256 - feat: update llama.cpp to ggml-org/llama.cpp@e3ba22d6c - feat(ci): add ROCm wheel builds by @abetlen in #2252 - feat(ci): add Vulkan wheel builds by @abetlen in #2251 diff --git a/README.md b/README.md index dd0024676a..5711d4afbb 100644 --- a/README.md +++ b/README.md @@ -536,6 +536,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [minicpm-v-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `MiniCPMv26ChatHandler` | `minicpm-v-2.6` | | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | | [gemma-4](https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF) | `Gemma4ChatHandler` | `gemma4` | +| GGUF models with an mtmd projector and embedded chat template | `MTMDChatHandler` | `mtmd` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 44c6c1f76f..4298642f67 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3265,48 +3265,512 @@ def from_pretrained( ) -class Gemma4ChatHandler(Llava15ChatHandler): - DEFAULT_SYSTEM_MESSAGE = None +class MTMDChatHandler: + def __init__( + self, clip_model_path: str, verbose: bool = True, use_gpu: bool = True + ): + import llama_cpp.mtmd_cpp as mtmd_cpp - CHAT_FORMAT = ( - "{% if messages and messages[0]['role'] == 'system' %}" - "{% if messages[0]['content'] is string %}" - "{% set first_user_prefix = messages[0]['content'] + '\n\n' %}" - "{% else %}" - "{% set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' %}" - "{% endif %}" - "{% set loop_messages = messages[1:] %}" - "{% else %}" - "{% set first_user_prefix = '' %}" - "{% set loop_messages = messages %}" - "{% endif %}" - "{% for message in loop_messages %}" - "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}" - "{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}" - "{% endif %}" - "{% set role = 'model' if message['role'] == 'assistant' else message['role'] %}" - "{{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else '') }}" - "{% if message['content'] is string %}" - "{{ message['content'] | trim }}" - "{% elif message['content'] is iterable %}" - "{% for item in message['content'] %}" - "{% if item['type'] == 'image_url' and item['image_url'] is string %}" - "{{ '\n\n' + item['image_url'] + '\n\n' }}" - "{% elif item['type'] == 'image_url' and item['image_url'] is mapping %}" - "{{ '\n\n' + item['image_url']['url'] + '\n\n' }}" - "{% elif item['type'] == 'text' %}" - "{{ item['text'] | trim }}" - "{% endif %}" - "{% endfor %}" - "{% else %}" - "{{ raise_exception('Invalid content type') }}" - "{% endif %}" - "{{ '<end_of_turn>\n' }}" - "{% endfor %}" - "{% if add_generation_prompt %}" - "{{ '<start_of_turn>model\n' }}" - "{% endif %}" - ) + self.clip_model_path = clip_model_path + self.verbose = verbose + self.use_gpu = use_gpu + self._mtmd_cpp = mtmd_cpp + self._exit_stack = ExitStack() + self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None + + if not os.path.exists(clip_model_path): + raise ValueError(f"Clip model path does not exist: {clip_model_path}") + + def _init_mtmd_context(self, llama_model: llama.Llama): + self.verbose = llama_model.verbose + if self.mtmd_ctx is not None: + return + + with suppress_stdout_stderr(disable=self.verbose): + ctx_params = self._mtmd_cpp.mtmd_context_params_default() + ctx_params.use_gpu = self.use_gpu + ctx_params.print_timings = self.verbose + ctx_params.n_threads = llama_model.n_threads + ctx_params.flash_attn_type = ( + llama_cpp.LLAMA_FLASH_ATTN_TYPE_ENABLED + if ( + llama_model.context_params.flash_attn_type + == llama_cpp.LLAMA_FLASH_ATTN_TYPE_ENABLED + ) + else llama_cpp.LLAMA_FLASH_ATTN_TYPE_DISABLED + ) + + self.mtmd_ctx = self._mtmd_cpp.mtmd_init_from_file( + self.clip_model_path.encode(), llama_model.model, ctx_params + ) + + if self.mtmd_ctx is None: + raise ValueError( + f"Failed to load mtmd context from: {self.clip_model_path}" + ) + + if not self._mtmd_cpp.mtmd_support_vision(self.mtmd_ctx): + raise ValueError("Vision is not supported by this model") + + def mtmd_free(): + with suppress_stdout_stderr(disable=self.verbose): + if self.mtmd_ctx is not None: + self._mtmd_cpp.mtmd_free(self.mtmd_ctx) + self.mtmd_ctx = None + + self._exit_stack.callback(mtmd_free) + + def load_image(self, image_url: str) -> bytes: + return self._load_image(image_url) + + def _create_bitmap_from_bytes(self, image_bytes: bytes): + if self.mtmd_ctx is None: + raise ValueError("mtmd context not initialized") + + with suppress_stdout_stderr(disable=self.verbose): + bitmap = self._mtmd_cpp.mtmd_helper_bitmap_init_from_buf( + self.mtmd_ctx, + (ctypes.c_uint8 * len(image_bytes)).from_buffer(bytearray(image_bytes)), + len(image_bytes), + ) + + if bitmap is None: + raise ValueError("Failed to create bitmap from image bytes") + + return bitmap + + def _get_chat_template(self, llama_model: llama.Llama) -> str: + chat_template = llama_model.metadata.get("tokenizer.chat_template") + if not isinstance(chat_template, str) or chat_template == "": + raise ValueError( + f"{self.__class__.__name__} requires tokenizer.chat_template metadata" + ) + return chat_template + + def _get_template_messages( + self, + messages: List[llama_types.ChatCompletionRequestMessage], + media_marker: str, + ) -> List[Any]: + return [ + self._convert_message_for_template(message, media_marker) + for message in messages + ] + + @classmethod + def _convert_message_for_template( + cls, + message: llama_types.ChatCompletionRequestMessage, + media_marker: str, + ) -> Dict[str, Any]: + message_dict = dict(message) + content = message_dict.get("content") + if isinstance(content, list): + message_dict["content"] = [ + cls._convert_content_part_for_template(part, media_marker) + for part in content + ] + return message_dict + + @staticmethod + def _convert_content_part_for_template( + part: Any, + media_marker: str, + ) -> Any: + if isinstance(part, dict) and part.get("type") == "image_url": + return {"type": "text", "text": media_marker} + return part + + @staticmethod + def _decode_token_piece(piece: Any) -> str: + if isinstance(piece, bytes): + return piece.decode("utf-8", errors="ignore") + return str(piece) + + def _postprocess_template_text( + self, + text: str, + image_urls: List[str], + media_marker: str, + ) -> str: + for image_url in image_urls: + text = text.replace(image_url, media_marker) + return text + + def __call__( + self, + *, + llama: llama.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + temperature: float = 0.2, + top_p: float = 0.95, + top_k: int = 40, + min_p: float = 0.05, + typical_p: float = 1.0, + stream: bool = False, + stop: Optional[Union[str, List[str]]] = [], + seed: Optional[int] = None, + response_format: Optional[ + llama_types.ChatCompletionRequestResponseFormat + ] = None, + max_tokens: Optional[int] = None, + presence_penalty: float = 0.0, + frequency_penalty: float = 0.0, + repeat_penalty: float = 1.1, + tfs_z: float = 1.0, + mirostat_mode: int = 0, + mirostat_tau: float = 5.0, + mirostat_eta: float = 0.1, + model: Optional[str] = None, + logits_processor: Optional[llama.LogitsProcessorList] = None, + grammar: Optional[llama.LlamaGrammar] = None, + logit_bias: Optional[Dict[str, float]] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, + **kwargs, # type: ignore + ) -> Union[ + llama_types.CreateChatCompletionResponse, + Iterator[llama_types.CreateChatCompletionStreamResponse], + ]: + self._init_mtmd_context(llama) + assert self.mtmd_ctx is not None + + image_urls = self.get_image_urls(messages) + media_marker = self._mtmd_cpp.mtmd_default_marker().decode("utf-8") + template_env = ImmutableSandboxedEnvironment( + trim_blocks=True, + lstrip_blocks=True, + extensions=[ + Jinja2ChatFormatter.IgnoreGenerationTags, + jinja2.ext.loopcontrols, + ], + ) + template_env.filters["tojson"] = Jinja2ChatFormatter.tojson + template = template_env.from_string(self._get_chat_template(llama)) + + def raise_exception(message: str): + raise ValueError(message) + + text = template.render( + messages=self._get_template_messages(messages, media_marker), + add_generation_prompt=True, + eos_token=self._decode_token_piece(llama.detokenize([llama.token_eos()])), + bos_token=self._decode_token_piece(llama.detokenize([llama.token_bos()])), + raise_exception=raise_exception, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + strftime_now=Jinja2ChatFormatter.strftime_now, + **kwargs, + ) + text = self._postprocess_template_text(text, image_urls, media_marker) + + if self.verbose: + print(text, file=sys.stderr) + + bitmaps = [] + bitmap_cleanup = [] + try: + for image_url in image_urls: + image_bytes = self.load_image(image_url) + bitmap = self._create_bitmap_from_bytes(image_bytes) + bitmaps.append(bitmap) + bitmap_cleanup.append(bitmap) + + input_text = self._mtmd_cpp.mtmd_input_text() + input_text.text = text.encode("utf-8") + input_text.add_special = True + input_text.parse_special = True + + chunks = self._mtmd_cpp.mtmd_input_chunks_init() + if chunks is None: + raise ValueError("Failed to create input chunks") + + try: + bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))( + *bitmaps + ) + result = self._mtmd_cpp.mtmd_tokenize( + self.mtmd_ctx, + chunks, + ctypes.byref(input_text), + bitmap_array, + len(bitmaps), + ) + + if result != 0: + raise ValueError(f"Failed to tokenize input: error code {result}") + + llama.reset() + llama._ctx.kv_cache_clear() + + n_chunks = self._mtmd_cpp.mtmd_input_chunks_size(chunks) + + for i in range(n_chunks): + chunk = self._mtmd_cpp.mtmd_input_chunks_get(chunks, i) + if chunk is None: + continue + + chunk_type = self._mtmd_cpp.mtmd_input_chunk_get_type(chunk) + + if chunk_type == self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_TEXT: + n_tokens_out = ctypes.c_size_t() + tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text( + chunk, ctypes.byref(n_tokens_out) + ) + + if tokens_ptr and n_tokens_out.value > 0: + tokens = [tokens_ptr[j] for j in range(n_tokens_out.value)] + + if llama.n_tokens + len(tokens) > llama.n_ctx(): + raise ValueError( + f"Prompt exceeds n_ctx: {llama.n_tokens + len(tokens)} > {llama.n_ctx()}" + ) + llama.eval(tokens) + + elif chunk_type in [ + self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_IMAGE, + self._mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_AUDIO, + ]: + chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens( + chunk + ) + + if llama.n_tokens + chunk_n_tokens > llama.n_ctx(): + raise ValueError( + f"Prompt exceeds n_ctx: {llama.n_tokens + chunk_n_tokens} > {llama.n_ctx()}" + ) + + new_n_past = llama_cpp.llama_pos(0) + result = self._mtmd_cpp.mtmd_helper_eval_chunk_single( + self.mtmd_ctx, + llama._ctx.ctx, + chunk, + llama_cpp.llama_pos(llama.n_tokens), + llama_cpp.llama_seq_id(0), + llama.n_batch, + False, # logits_last + ctypes.byref(new_n_past), + ) + + if result != 0: + raise ValueError( + f"Failed to evaluate chunk: error code {result}" + ) + + llama.n_tokens = new_n_past.value + + prompt = llama.input_ids[: llama.n_tokens].tolist() + + finally: + self._mtmd_cpp.mtmd_input_chunks_free(chunks) + + finally: + for bitmap in bitmap_cleanup: + self._mtmd_cpp.mtmd_bitmap_free(bitmap) + + if response_format is not None and response_format["type"] == "json_object": + grammar = _grammar_for_response_format(response_format) + + if functions is not None: + tools = [ + { + "type": "function", + "function": function, + } + for function in functions + ] + + if function_call is not None: + if isinstance(function_call, str) and ( + function_call == "none" or function_call == "auto" + ): + tool_choice = function_call + if isinstance(function_call, dict) and "name" in function_call: + tool_choice = { + "type": "function", + "function": { + "name": function_call["name"], + }, + } + + tool = None + if ( + tool_choice is not None + and isinstance(tool_choice, dict) + and tools is not None + ): + name = tool_choice["function"]["name"] + tool = next((t for t in tools if t["function"]["name"] == name), None) + if tool is None: + raise ValueError(f"Tool choice '{name}' not found in tools.") + schema = tool["function"]["parameters"] + try: + grammar = llama_grammar.LlamaGrammar.from_json_schema( + json.dumps(schema), verbose=llama.verbose + ) + except Exception as e: + if llama.verbose: + print(str(e), file=sys.stderr) + grammar = llama_grammar.LlamaGrammar.from_string( + llama_grammar.JSON_GBNF, verbose=llama.verbose + ) + + completion_or_chunks = llama.create_completion( + prompt=prompt, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + typical_p=typical_p, + logprobs=top_logprobs if logprobs else None, + stream=stream, + stop=stop, + seed=seed, + max_tokens=max_tokens, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + repeat_penalty=repeat_penalty, + tfs_z=tfs_z, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + model=model, + logits_processor=logits_processor, + grammar=grammar, + logit_bias=logit_bias, + ) + + if tool is not None: + tool_name = tool["function"]["name"] + return _convert_completion_to_chat_function( + tool_name, completion_or_chunks, stream + ) + return _convert_completion_to_chat(completion_or_chunks, stream=stream) + + @staticmethod + def _load_image(image_url: str) -> bytes: + if image_url.startswith("data:"): + import base64 + + image_bytes = base64.b64decode(image_url.split(",")[1]) + return image_bytes + else: + import urllib.request + + with urllib.request.urlopen(image_url) as f: + image_bytes = f.read() + return image_bytes + + @staticmethod + def get_image_urls(messages: List[llama_types.ChatCompletionRequestMessage]): + image_urls: List[str] = [] + for message in messages: + if message["role"] == "user": + if message["content"] is None: + continue + for content in message["content"]: + if isinstance(content, dict) and "type" in content: + if content["type"] == "image_url": + if ( + isinstance(content["image_url"], dict) + and "url" in content["image_url"] + ): + image_urls.append(content["image_url"]["url"]) + else: + image_urls.append(content["image_url"]) + return image_urls + + @classmethod + def from_pretrained( + cls, + repo_id: str, + filename: Optional[str], + local_dir: Optional[Union[str, os.PathLike[str]]] = None, + local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", + cache_dir: Optional[Union[str, os.PathLike[str]]] = None, + **kwargs: Any, + ) -> "MTMDChatHandler": + import fnmatch + from pathlib import Path + + try: + from huggingface_hub import hf_hub_download, HfFileSystem # type: ignore + from huggingface_hub.utils import validate_repo_id # type: ignore + except ImportError: + raise ImportError( + "Llama.from_pretrained requires the huggingface-hub package. " + "You can install it with `pip install huggingface-hub`." + ) + + validate_repo_id(repo_id) + + hffs = HfFileSystem() + + files = [ + file["name"] if isinstance(file, dict) else file + for file in hffs.ls(repo_id) # type: ignore + ] + + file_list: List[str] = [] + for file in files: + rel_path = Path(file).relative_to(repo_id) + file_list.append(str(rel_path)) + + matching_files = [file for file in file_list if fnmatch.fnmatch(file, filename)] # type: ignore + + if len(matching_files) == 0: + raise ValueError( + f"No file found in {repo_id} that match {filename}\n\n" + f"Available Files:\n{json.dumps(file_list)}" + ) + + if len(matching_files) > 1: + raise ValueError( + f"Multiple files found in {repo_id} matching {filename}\n\n" + f"Available Files:\n{json.dumps(files)}" + ) + + (matching_file,) = matching_files + + subfolder = str(Path(matching_file).parent) + filename = Path(matching_file).name + + hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder, + local_dir=cast(Union[str, Path, None], local_dir), + local_dir_use_symlinks=local_dir_use_symlinks, + cache_dir=cast(Union[str, Path, None], cache_dir), + ) + + if local_dir is None: + model_path = hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder, + local_dir=local_dir, + local_dir_use_symlinks=local_dir_use_symlinks, + cache_dir=cast(Union[str, Path, None], cache_dir), + local_files_only=True, + ) + else: + model_path = os.path.join(local_dir, filename) + + return cls( + clip_model_path=model_path, + **kwargs, + ) + + +class Gemma4ChatHandler(MTMDChatHandler): + pass class ObsidianChatHandler(Llava15ChatHandler): diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 3222abd631..8aa929202c 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -115,18 +115,21 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler = llama_cpp.llama_chat_format.Llava16ChatHandler( clip_model_path=settings.clip_model_path, verbose=settings.verbose ) - elif settings.chat_format == "gemma4": + elif settings.chat_format in ("mtmd", "gemma4"): assert settings.clip_model_path is not None, "clip model not found" + chat_handler_cls = ( + llama_cpp.llama_chat_format.MTMDChatHandler + if settings.chat_format == "mtmd" + else llama_cpp.llama_chat_format.Gemma4ChatHandler + ) if settings.hf_model_repo_id is not None: - chat_handler = ( - llama_cpp.llama_chat_format.Gemma4ChatHandler.from_pretrained( - repo_id=settings.hf_model_repo_id, - filename=settings.clip_model_path, - verbose=settings.verbose, - ) + chat_handler = chat_handler_cls.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, ) else: - chat_handler = llama_cpp.llama_chat_format.Gemma4ChatHandler( + chat_handler = chat_handler_cls( clip_model_path=settings.clip_model_path, verbose=settings.verbose ) elif settings.chat_format == "moondream": From 8edcd15eceda7be5839be1c2688fa3caa057ddb8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:22:45 -0700 Subject: [PATCH 530/561] chore(deps): bump docker/build-push-action from 6 to 7 (#2263) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6 to 7. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6...v7) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build-docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 349eb209dd..0a74357bce 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -40,7 +40,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . file: "docker/simple/Dockerfile" From 64c01752badd526010455011fcf903b4d7061458 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:23:00 -0700 Subject: [PATCH 531/561] chore(deps): bump actions/upload-pages-artifact from 3 to 5 (#2264) Bumps [actions/upload-pages-artifact](https://github.com/actions/upload-pages-artifact) from 3 to 5. - [Release notes](https://github.com/actions/upload-pages-artifact/releases) - [Commits](https://github.com/actions/upload-pages-artifact/compare/v3...v5) --- updated-dependencies: - dependency-name: actions/upload-pages-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/generate-index-from-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 6b2c8770fa..49b06e68ad 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -51,7 +51,7 @@ jobs: ./scripts/releases-to-pep-503.sh index/whl/vulkan '^[v]?[0-9]+\.[0-9]+\.[0-9]+-vulkan$' ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$' - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@v5 with: # Upload entire repository path: 'index' From 6bccad59938d51e551d67cf0efd61b3767ec98a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:23:16 -0700 Subject: [PATCH 532/561] chore(deps): bump actions/configure-pages from 5 to 6 (#2265) Bumps [actions/configure-pages](https://github.com/actions/configure-pages) from 5 to 6. - [Release notes](https://github.com/actions/configure-pages/releases) - [Commits](https://github.com/actions/configure-pages/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/configure-pages dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/generate-index-from-release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index 49b06e68ad..a9124fbc09 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -33,7 +33,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 - name: Setup Pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@v6 - name: Build env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 23fe09f0a48480d2cbcc8e5790def00e8cc064ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:23:31 -0700 Subject: [PATCH 533/561] chore(deps): bump softprops/action-gh-release from 2 to 3 (#2266) Bumps [softprops/action-gh-release](https://github.com/softprops/action-gh-release) from 2 to 3. - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/v2...v3) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/build-and-release.yaml | 2 +- .github/workflows/build-wheels-cuda.yaml | 2 +- .github/workflows/build-wheels-metal.yaml | 2 +- .github/workflows/build-wheels-rocm.yaml | 4 ++-- .github/workflows/build-wheels-vulkan.yaml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index 0b974e93c8..ad7ba22f6e 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -207,7 +207,7 @@ jobs: merge-multiple: true path: dist - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 with: files: dist/* env: diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index f5b3718f17..905b9bee9b 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -222,7 +222,7 @@ jobs: # Publish tags that reflect the actual installed toolkit version. Write-Output "CUDA_VERSION=$cudaTagVersion" >> $env:GITHUB_ENV - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 if: startsWith(github.ref, 'refs/tags/') with: files: dist/* diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index ca8e322230..892787ee1f 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -59,7 +59,7 @@ jobs: merge-multiple: true path: dist2 - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 with: files: dist2/* # set release name to <tag>-metal diff --git a/.github/workflows/build-wheels-rocm.yaml b/.github/workflows/build-wheels-rocm.yaml index 792e33cf89..1902c125ba 100644 --- a/.github/workflows/build-wheels-rocm.yaml +++ b/.github/workflows/build-wheels-rocm.yaml @@ -208,7 +208,7 @@ jobs: merge-multiple: true path: dist - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 with: files: dist/* # Set release name to <tag>-rocm<rocm_version>. @@ -228,7 +228,7 @@ jobs: merge-multiple: true path: dist - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 with: files: dist/* # Set release name to <tag>-hip-radeon. diff --git a/.github/workflows/build-wheels-vulkan.yaml b/.github/workflows/build-wheels-vulkan.yaml index 49d38fb80b..8790cc20e9 100644 --- a/.github/workflows/build-wheels-vulkan.yaml +++ b/.github/workflows/build-wheels-vulkan.yaml @@ -114,7 +114,7 @@ jobs: merge-multiple: true path: dist - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 with: files: dist/* # Set release name to <tag>-vulkan. From 9013c1d32b55e8514dbf0e460684c6679ad1d566 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:23:46 -0700 Subject: [PATCH 534/561] chore(deps): bump docker/setup-qemu-action from 3 to 4 (#2267) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 3 to 4. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrei <abetlen@gmail.com> --- .github/workflows/build-and-release.yaml | 2 +- .github/workflows/build-docker.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index ad7ba22f6e..a1b456edda 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -128,7 +128,7 @@ jobs: submodules: "recursive" - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 with: platforms: linux/riscv64 diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml index 0a74357bce..d306efde48 100644 --- a/.github/workflows/build-docker.yaml +++ b/.github/workflows/build-docker.yaml @@ -26,7 +26,7 @@ jobs: echo "IMAGE_TAG=$image_tag" >> "$GITHUB_ENV" - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 From c2e22ae8e930372ecc911471d4364fdac79ef630 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 18:26:13 -0700 Subject: [PATCH 535/561] feat: update llama.cpp to ggml-org/llama.cpp@7c158fbb4 (#2268) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18bcf258a0..88f4d99467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat: Generic Multimodal Chat Handler by @abetlen in #2256 -- feat: update llama.cpp to ggml-org/llama.cpp@e3ba22d6c +- feat: update llama.cpp to ggml-org/llama.cpp@7c158fbb4 - feat(ci): add ROCm wheel builds by @abetlen in #2252 - feat(ci): add Vulkan wheel builds by @abetlen in #2251 - fix: handle additional `from_pretrained` files in subfolders by @TNing in #2085 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e3ba22d6cc..7c158fbb4a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e3ba22d6cc4dec84e59a909c7f96e1689c7384a9 +Subproject commit 7c158fbb4aec1bdc9c81d6ca0e785139f4826fae From 5151ac7a27b06215e51bb63b7a784c6265488987 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 19:09:40 -0700 Subject: [PATCH 536/561] chore: bump version to 0.3.26 (#2269) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88f4d99467..28645f13c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.26] + - feat: Generic Multimodal Chat Handler by @abetlen in #2256 - feat: update llama.cpp to ggml-org/llama.cpp@7c158fbb4 - feat(ci): add ROCm wheel builds by @abetlen in #2252 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 52101c9b7e..bbfb73de3f 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.25" +__version__ = "0.3.26" From 78ac75e8fa265749b41640260a0548a598ae8dc8 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 19:47:14 -0700 Subject: [PATCH 537/561] fix(ci): repair release wheel workflows (#2270) --- .github/workflows/build-and-release.yaml | 5 +++++ .github/workflows/build-wheels-rocm.yaml | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index a1b456edda..cd9983fcee 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -49,6 +49,9 @@ jobs: # Linux needs auditwheel repair so manylinux and musllinux wheels are # published with distinct platform tags instead of generic linux tags. CIBW_REPAIR_WHEEL_COMMAND_LINUX: "LD_LIBRARY_PATH=/project/llama_cpp/lib auditwheel repair -w {dest_dir} {wheel}" + # cibuildwheel v3 defaults to manylinux_2_28 images whose current + # GCC toolchain emits symbols newer than the policy allows. + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014" # The release wheel is tagged py3-none, so one build per platform # covers all supported Python versions and avoids duplicate names. CIBW_BUILD_LINUX: "cp38-*" @@ -85,6 +88,8 @@ jobs: CIBW_SKIP: "pp*" CIBW_REPAIR_WHEEL_COMMAND: "LD_LIBRARY_PATH=$PWD/llama_cpp/lib auditwheel repair -w {dest_dir} {wheel}" CIBW_ARCHS: "aarch64" + # Keep this consistent with the x86_64 Linux release wheels. + CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux2014" # Keep native arm64 builds on a portable CPU baseline instead of # tuning wheels to the hosted runner. CIBW_ENVIRONMENT: CMAKE_ARGS="-DGGML_NATIVE=off" diff --git a/.github/workflows/build-wheels-rocm.yaml b/.github/workflows/build-wheels-rocm.yaml index 1902c125ba..6ad0b49541 100644 --- a/.github/workflows/build-wheels-rocm.yaml +++ b/.github/workflows/build-wheels-rocm.yaml @@ -33,7 +33,6 @@ jobs: - uses: actions/setup-python@v6 with: python-version: ${{ matrix.pyver }} - cache: "pip" - name: Install build dependencies run: | From 7c86eae04117ef150715d1f4a2982f7b4adf147d Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 20:27:27 -0700 Subject: [PATCH 538/561] fix(ci): allow empty wheel indexes (#2271) --- scripts/releases-to-pep-503.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/releases-to-pep-503.sh b/scripts/releases-to-pep-503.sh index 71910efcbf..8359624492 100755 --- a/scripts/releases-to-pep-503.sh +++ b/scripts/releases-to-pep-503.sh @@ -54,8 +54,12 @@ cat << EOF > "$output_dir/llama-cpp-python/index.html" <h1>Links for llama-cpp-python</h1> EOF -# Filter releases by pattern -releases=$(grep -E "$pattern" "$current_dir/all_releases.txt") +# Filter releases by pattern. Some backend indexes are valid even when there +# are no matching releases yet. +releases=$(grep -E "$pattern" "$current_dir/all_releases.txt" || true) +if [ -z "$releases" ]; then + log_info "No releases found matching pattern: $pattern" +fi # Prepare curl headers headers=('--header' 'Accept: application/vnd.github.v3+json') @@ -81,16 +85,16 @@ for release in $releases; do continue fi - # Get release version from release ie v0.1.0-cu121 -> v0.1.0 - release_version=$(echo "$release" | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+") - echo " <h2>$release_version</h2>" >> "$output_dir/llama-cpp-python/index.html" - wheel_urls=$(echo "$response" | jq -r '.assets[] | select(.name | endswith(".whl")) | .browser_download_url') if [ -z "$wheel_urls" ]; then log_error "No wheel files found for release $release" continue fi + # Get release version from release ie v0.1.0-cu121 -> v0.1.0 + release_version=$(echo "$release" | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+") + echo " <h2>$release_version</h2>" >> "$output_dir/llama-cpp-python/index.html" + echo "$wheel_urls" | while read -r asset; do echo " <a href=\"$asset\">$asset</a>" >> "$output_dir/llama-cpp-python/index.html" echo " <br>" >> "$output_dir/llama-cpp-python/index.html" From 672198970724492a51329e45d714f338f97cc184 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 20:41:30 -0700 Subject: [PATCH 539/561] fix(ci): index all CUDA wheel variants (#2272) --- .github/workflows/generate-index-from-release.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index a9124fbc09..edf292387d 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -40,12 +40,14 @@ jobs: run: | ./scripts/get-releases.sh ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$' + ./scripts/releases-to-pep-503.sh index/whl/cu118 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu118$' ./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$' ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$' ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$' ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' - # ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' - # ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$' + ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu125$' + ./scripts/releases-to-pep-503.sh index/whl/cu130 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu130$' + ./scripts/releases-to-pep-503.sh index/whl/cu132 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu132$' ./scripts/releases-to-pep-503.sh index/whl/rocm72 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-rocm72$' ./scripts/releases-to-pep-503.sh index/whl/hip-radeon '^[v]?[0-9]+\.[0-9]+\.[0-9]+-hip-radeon$' ./scripts/releases-to-pep-503.sh index/whl/vulkan '^[v]?[0-9]+\.[0-9]+\.[0-9]+-vulkan$' From 468498519e67395771c7da34b2e214975d4f96a0 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 21:41:11 -0700 Subject: [PATCH 540/561] fix(ci): build one riscv64 release wheel (#2273) --- .github/workflows/build-and-release.yaml | 29 ++++-------------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index cd9983fcee..4ae37b1745 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -106,27 +106,8 @@ jobs: path: ./wheelhouse/*.whl build_wheels_riscv64: - name: Build riscv64 wheels (${{ matrix.shard.name }}) + name: Build riscv64 wheel runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: - - name: cp310 - build: "cp310-*" - artifact: wheels_riscv64_cp310 - - name: cp311 - build: "cp311-*" - artifact: wheels_riscv64_cp311 - - name: cp312 - build: "cp312-*" - artifact: wheels_riscv64_cp312 - - name: cp313 - build: "cp313-*" - artifact: wheels_riscv64_cp313 - - name: cp314 - build: "cp314-*" - artifact: wheels_riscv64_cp314 steps: - uses: actions/checkout@v6 with: @@ -146,16 +127,16 @@ jobs: # Build riscv64 wheels against a conservative baseline instead of # enabling RVV-related extensions from the build container. CIBW_ENVIRONMENT: CMAKE_ARGS="-DGGML_NATIVE=off -DGGML_RVV=off -DGGML_RV_ZFH=off -DGGML_RV_ZVFH=off -DGGML_RV_ZICBOP=off -DGGML_RV_ZIHINTPAUSE=off" - # Split the emulated riscv64 build into one Python version per job - # to minimize wall-clock time without changing the release artifacts. - CIBW_BUILD: ${{ matrix.shard.build }} + # The release wheel is tagged py3-none, so one riscv64 build is + # enough and avoids duplicate same-name release artifacts. + CIBW_BUILD: "cp310-*" with: output-dir: wheelhouse - name: Upload wheels as artifacts uses: actions/upload-artifact@v7 with: - name: ${{ matrix.shard.artifact }} + name: wheels_riscv64 path: ./wheelhouse/*.whl build_sdist: From 8949066b3b9dd0d055837d491ffe2eaf99c582f8 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 22:15:47 -0700 Subject: [PATCH 541/561] docs: add Gemma 4 Colab notebook (#2274) --- README.md | 2 +- examples/colab/notebook.ipynb | 131 ++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 examples/colab/notebook.ipynb diff --git a/README.md b/README.md index 5711d4afbb..3f801285ed 100644 --- a/README.md +++ b/README.md @@ -535,7 +535,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha-gguf) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | | [minicpm-v-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `MiniCPMv26ChatHandler` | `minicpm-v-2.6` | | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | -| [gemma-4](https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF) | `Gemma4ChatHandler` | `gemma4` | +| [gemma-4](https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/notebook.ipynb) | `Gemma4ChatHandler` | `gemma4` | | GGUF models with an mtmd projector and embedded chat template | `MTMDChatHandler` | `mtmd` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. diff --git a/examples/colab/notebook.ipynb b/examples/colab/notebook.ipynb new file mode 100644 index 0000000000..c9b8d8dcbb --- /dev/null +++ b/examples/colab/notebook.ipynb @@ -0,0 +1,131 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "accelerator": "GPU", + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gemma 4 12B Multimodal Chat\n", + "\n", + "Run Gemma 4 12B locally in Google Colab with the pre-built CUDA wheel for `llama-cpp-python`.\n", + "\n", + "Use a GPU runtime before running this notebook: **Runtime > Change runtime type > T4 GPU**.\n", + "\n", + "Current Colab CUDA images commonly provide CUDA 12 user-space libraries even when `nvidia-smi` reports a CUDA 13-capable driver, so this notebook installs the `cu125` wheel. If your runtime provides `libcudart.so.13`, switch the wheel index URL to `/whl/cu130`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install --no-cache-dir --upgrade --force-reinstall \\\n", + " \"huggingface-hub>=0.23.0\" \\\n", + " llama-cpp-python \\\n", + " --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu125\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from llama_cpp import Llama\n", + "from llama_cpp.llama_chat_format import Gemma4ChatHandler\n", + "\n", + "MODEL_REPO = \"ggml-org/gemma-4-12B-it-GGUF\"\n", + "MODEL_FILE = \"gemma-4-12B-it-Q4_K_M.gguf\"\n", + "MMPROJ_FILE = \"mmproj-gemma-4-12B-it-Q8_0.gguf\"\n", + "\n", + "chat_handler = Gemma4ChatHandler.from_pretrained(\n", + " repo_id=MODEL_REPO,\n", + " filename=MMPROJ_FILE,\n", + " verbose=False,\n", + ")\n", + "\n", + "llm = Llama.from_pretrained(\n", + " repo_id=MODEL_REPO,\n", + " filename=MODEL_FILE,\n", + " chat_handler=chat_handler,\n", + " n_gpu_layers=-1,\n", + " n_ctx=8192,\n", + " flash_attn=True,\n", + " verbose=False,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = llm.create_chat_completion(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Write the exact string `<stdio.h>` and nothing else.\",\n", + " }\n", + " ],\n", + " max_tokens=32,\n", + " temperature=0.0,\n", + ")\n", + "\n", + "print(response[\"choices\"][0][\"message\"][\"content\"])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import Image, display\n", + "\n", + "IMAGE_URL = \"https://raw.githubusercontent.com/abetlen/llama-cpp-python/main/vendor/llama.cpp/tools/mtmd/test-1.jpeg\"\n", + "\n", + "display(Image(url=IMAGE_URL, width=320))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = llm.create_chat_completion(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"Describe this image in one concise sentence.\"},\n", + " {\"type\": \"image_url\", \"image_url\": {\"url\": IMAGE_URL}},\n", + " ],\n", + " }\n", + " ],\n", + " max_tokens=128,\n", + " temperature=0.2,\n", + ")\n", + "\n", + "print(response[\"choices\"][0][\"message\"][\"content\"])\n" + ] + } + ] +} From 7a2a36d769ffe051320345a9a64b68dddc53b9f8 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Thu, 4 Jun 2026 22:41:14 -0700 Subject: [PATCH 542/561] docs: fix Gemma 4 Colab notebook (#2275) * docs: fix Gemma 4 Colab notebook * docs: use smaller Gemma 4 Colab model * docs: update Gemma 4 Colab CTA --- README.md | 4 +++- examples/colab/notebook.ipynb | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3f801285ed..8f7b65e835 100644 --- a/README.md +++ b/README.md @@ -535,9 +535,11 @@ Below are the supported multi-modal models and their respective chat handlers (P | [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha-gguf) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | | [minicpm-v-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `MiniCPMv26ChatHandler` | `minicpm-v-2.6` | | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | -| [gemma-4](https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/notebook.ipynb) | `Gemma4ChatHandler` | `gemma4` | +| [gemma-4](https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF) | `Gemma4ChatHandler` | `gemma4` | | GGUF models with an mtmd projector and embedded chat template | `MTMDChatHandler` | `mtmd` | +Try Gemma 4 12B in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/notebook.ipynb) + Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. ```python diff --git a/examples/colab/notebook.ipynb b/examples/colab/notebook.ipynb index c9b8d8dcbb..8e258b9c03 100644 --- a/examples/colab/notebook.ipynb +++ b/examples/colab/notebook.ipynb @@ -81,7 +81,7 @@ " messages=[\n", " {\n", " \"role\": \"user\",\n", - " \"content\": \"Write the exact string `<stdio.h>` and nothing else.\",\n", + " \"content\": \"What is the capital of France? Answer in one sentence.\",\n", " }\n", " ],\n", " max_tokens=32,\n", @@ -99,7 +99,7 @@ "source": [ "from IPython.display import Image, display\n", "\n", - "IMAGE_URL = \"https://raw.githubusercontent.com/abetlen/llama-cpp-python/main/vendor/llama.cpp/tools/mtmd/test-1.jpeg\"\n", + "IMAGE_URL = \"https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/mtmd/test-1.jpeg\"\n", "\n", "display(Image(url=IMAGE_URL, width=320))\n" ] From 7f16fe1963bbca25b7b8f59410153d46d48dea0f Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sat, 6 Jun 2026 00:15:04 -0700 Subject: [PATCH 543/561] docs: add Gemma 4 QAT Colab notebook (#2276) * docs: add Gemma 4 QAT Colab notebook * docs: rename Gemma 4 QAT Colab notebook --- README.md | 2 + examples/colab/Gemma4-12B-QAT.ipynb | 162 ++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 examples/colab/Gemma4-12B-QAT.ipynb diff --git a/README.md b/README.md index 8f7b65e835..c02df9eed0 100644 --- a/README.md +++ b/README.md @@ -540,6 +540,8 @@ Below are the supported multi-modal models and their respective chat handlers (P Try Gemma 4 12B in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/notebook.ipynb) +Try Gemma 4 12B QAT in Google Colab -> [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abetlen/llama-cpp-python/blob/main/examples/colab/Gemma4-12B-QAT.ipynb) + Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. ```python diff --git a/examples/colab/Gemma4-12B-QAT.ipynb b/examples/colab/Gemma4-12B-QAT.ipynb new file mode 100644 index 0000000000..2cced5a89b --- /dev/null +++ b/examples/colab/Gemma4-12B-QAT.ipynb @@ -0,0 +1,162 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "accelerator": "GPU", + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Gemma 4 12B QAT Multimodal Chat\n", + "\n", + "Run the Gemma 4 12B QAT GGUF model locally in Google Colab with the pre-built CUDA wheel for `llama-cpp-python`.\n", + "\n", + "Use a GPU runtime before running this notebook: **Runtime > Change runtime type > T4 GPU**.\n", + "\n", + "Current Colab CUDA images commonly provide CUDA 12 user-space libraries even when `nvidia-smi` reports a CUDA 13-capable driver, so this notebook installs the `cu125` wheel. If your runtime provides `libcudart.so.13`, switch the wheel index URL to `/whl/cu130`.\n" + ], + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install --no-cache-dir --upgrade --force-reinstall \\\n", + " \"huggingface-hub>=0.23.0\" \\\n", + " llama-cpp-python \\\n", + " --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu125\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from llama_cpp import Llama\n", + "from llama_cpp.llama_chat_format import Gemma4ChatHandler\n", + "\n", + "MODEL_REPO = \"unsloth/gemma-4-12B-it-qat-GGUF\"\n", + "MODEL_FILE = \"gemma-4-12B-it-qat-UD-Q4_K_XL.gguf\"\n", + "MMPROJ_FILE = \"mmproj-F16.gguf\"\n", + "\n", + "chat_handler = Gemma4ChatHandler.from_pretrained(\n", + " repo_id=MODEL_REPO,\n", + " filename=MMPROJ_FILE,\n", + " verbose=False,\n", + ")\n", + "\n", + "llm = Llama.from_pretrained(\n", + " repo_id=MODEL_REPO,\n", + " filename=MODEL_FILE,\n", + " chat_handler=chat_handler,\n", + " n_gpu_layers=-1,\n", + " n_ctx=8192,\n", + " flash_attn=True,\n", + " verbose=False,\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import Image, display\n", + "\n", + "IMAGE_URL = \"https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/mtmd/test-1.jpeg\"\n", + "\n", + "display(Image(url=IMAGE_URL, width=320))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = llm.create_chat_completion(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"Describe this image in one concise sentence.\"},\n", + " {\"type\": \"image_url\", \"image_url\": {\"url\": IMAGE_URL}},\n", + " ],\n", + " }\n", + " ],\n", + " max_tokens=128,\n", + " temperature=0.2,\n", + ")\n", + "\n", + "print(response[\"choices\"][0][\"message\"][\"content\"])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tools = [\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"record_image_observation\",\n", + " \"description\": \"Record a structured observation about an image.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"main_subject\": {\"type\": \"string\"},\n", + " \"setting\": {\"type\": \"string\"},\n", + " \"notable_details\": {\n", + " \"type\": \"array\",\n", + " \"items\": {\"type\": \"string\"},\n", + " },\n", + " \"confidence\": {\"type\": \"number\", \"minimum\": 0, \"maximum\": 1},\n", + " },\n", + " \"required\": [\"main_subject\", \"setting\", \"notable_details\", \"confidence\"],\n", + " },\n", + " },\n", + " }\n", + "]\n", + "\n", + "response = llm.create_chat_completion(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"Use the provided tool to record a structured observation for this image.\"},\n", + " {\"type\": \"image_url\", \"image_url\": {\"url\": IMAGE_URL}},\n", + " ],\n", + " }\n", + " ],\n", + " tools=tools,\n", + " tool_choice={\"type\": \"function\", \"function\": {\"name\": \"record_image_observation\"}},\n", + " max_tokens=256,\n", + " temperature=0.0,\n", + ")\n", + "\n", + "message = response[\"choices\"][0][\"message\"]\n", + "print(message.get(\"tool_calls\", message.get(\"content\")))\n" + ] + } + ] +} From ed833664fe96050d2990131f4673f112d8f88924 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sat, 6 Jun 2026 00:48:47 -0700 Subject: [PATCH 544/561] feat: update llama.cpp to 5a69c9743 (#2277) --- CHANGELOG.md | 2 ++ vendor/llama.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28645f13c8..b2c3dc8d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@5a69c9743 + ## [0.3.26] - feat: Generic Multimodal Chat Handler by @abetlen in #2256 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7c158fbb4a..5a69c97439 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7c158fbb4aec1bdc9c81d6ca0e785139f4826fae +Subproject commit 5a69c974392020e514c3b2b2910bb92f847cb4c9 From 66635a02fd32e1a4cc6bea3c85c3c7c50972fdfd Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 03:04:24 -0700 Subject: [PATCH 545/561] feat(example): Updated server example (batch processing, `/v1/responses` api, response parsing) (#2174) * Add batch processing server * Improve response parser streaming performance * Add /v1/models endpoint * Support custom tools in responses API * Improve responses api compatibility for codex * Improve batch server prompt and context config * Improve batch server scheduling and prompt handling * fix apply_patch tool * Improve batch server schemas and metrics * Refactor sequence cache helpers * Fix server type diagnostics * feat: add llama.cpp extension bindings * feat: add MTP support to batch server * feat: improve draft-mtp handling in batch server * feat: cap MTP draft context outputs * fix: preserve held streaming tokens * feat: add load-time LoRA support to batch server * feat: add multimodal support to batch server * refactor: rename batch item kinds * refactor: type sampled mtp updates * refactor: structure sampled mtp batch processing * refactor: clarify batch item construction * refactor: type batch item kind * refactor: clarify sampled pending index * refactor: clarify output index naming * refactor: rename logits index resolver * refactor: colocate sampled mtp state * refactor: inline sampled mtp helpers * refactor: use row-expanded multimodal prompt identity * test: remove multimodal prompt plan tests * refactor: narrow mtmd processor dependencies * refactor: group prompt segment media fields * refactor: centralize sequence state copy * refactor: keep disk cache storage only * refactor: split batch item payloads * refactor: centralize pending request failure cleanup * refactor: centralize sequence claiming * refactor: key sequence disk cache compatibility * refactor: decouple completion request preparation * refactor: name prepared completion parts * refactor: return prepared completion parts * refactor: localize media cache key building * refactor: remove unused request id override * refactor: simplify prompt segment row capacity * refactor: inline prompt row clamp * refactor: inline disconnect cancellation response * refactor: simplify recurrent draft capacity * refactor: define builtin grammar rule as dataclass * refactor: type chat template conversions * docs: mark llama_cpp_ext experimental * feat: restrict multimodal media sources * docs: add server example README and config * docs: document server example configuration * docs: update server README * docs: document server wheel setup and clients * docs: add server model configs * docs: add server chat templates and response schemas * docs: keep batch processing server example * docs: add server example changelog entry * docs: mention multi-token prediction in changelog --- CHANGELOG.md | 1 + examples/server/README.md | 434 + .../server/configs/gemma-4-12b-it-qat.json | 451 + examples/server/configs/gpt-oss-120b.json | 417 + examples/server/configs/qwen3.5-0.8b.json | 261 + examples/server/configs/qwen3.6-27b.json | 263 + examples/server/configs/qwen3.6-35b-a3b.json | 263 + examples/server/server.py | 15328 ++++++++++++++++ llama_cpp/llama_cpp_ext.py | 99 + 9 files changed, 17517 insertions(+) create mode 100644 examples/server/README.md create mode 100644 examples/server/configs/gemma-4-12b-it-qat.json create mode 100644 examples/server/configs/gpt-oss-120b.json create mode 100644 examples/server/configs/qwen3.5-0.8b.json create mode 100644 examples/server/configs/qwen3.6-27b.json create mode 100644 examples/server/configs/qwen3.6-35b-a3b.json create mode 100644 examples/server/server.py create mode 100644 llama_cpp/llama_cpp_ext.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b2c3dc8d50..bbe7b13bdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat: update llama.cpp to ggml-org/llama.cpp@5a69c9743 +- feat(example): Updated server example (batch processing, multi-token prediction, `/v1/responses` api, response parsing) by @abetlen in #2174 ## [0.3.26] diff --git a/examples/server/README.md b/examples/server/README.md new file mode 100644 index 0000000000..1f6e0d3dbe --- /dev/null +++ b/examples/server/README.md @@ -0,0 +1,434 @@ +# Server Example + +This example is an updated OpenAI-compatible web server that depends only on the low-level C bindings. +It supports batched inference, prompt caching, response parsing, `/v1/responses`, disk sequence caching, MTP, LoRA, and multimodal image/audio inputs. + +## Setup + +The server is a [`uv` inline script](https://docs.astral.sh/uv/guides/scripts/), so `uv` can create the script environment and install the Python dependencies automatically. + +```bash +cd examples/server +uv run --script server.py -C configs/qwen3.5-0.8b.json +``` + +Use `uv run --extra-index-url` to pull a pre-built `llama-cpp-python` binary wheel instead of building from source. + +```bash +cd examples/server +uv run \ + --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu \ + --script server.py -C configs/qwen3.5-0.8b.json +``` + +Pick the wheel index that matches the backend you want. + +| Backend | Wheel index | +| --- | --- | +| CPU | `https://abetlen.github.io/llama-cpp-python/whl/cpu` | +| CUDA 11.8 | `https://abetlen.github.io/llama-cpp-python/whl/cu118` | +| CUDA 12.1 | `https://abetlen.github.io/llama-cpp-python/whl/cu121` | +| CUDA 12.2 | `https://abetlen.github.io/llama-cpp-python/whl/cu122` | +| CUDA 12.3 | `https://abetlen.github.io/llama-cpp-python/whl/cu123` | +| CUDA 12.4 | `https://abetlen.github.io/llama-cpp-python/whl/cu124` | +| CUDA 12.5 | `https://abetlen.github.io/llama-cpp-python/whl/cu125` | +| CUDA 13.0 | `https://abetlen.github.io/llama-cpp-python/whl/cu130` | +| CUDA 13.2 | `https://abetlen.github.io/llama-cpp-python/whl/cu132` | +| Metal | `https://abetlen.github.io/llama-cpp-python/whl/metal` | +| ROCm | `https://abetlen.github.io/llama-cpp-python/whl/rocm72` | +| Vulkan | `https://abetlen.github.io/llama-cpp-python/whl/vulkan` | + +See the repository installation section for the full [pre-built wheel requirements](../../README.md#supported-backends). + +## Model Configs + +The smallest checked-in example uses Qwen3.5 0.8B so the server can be started on a normal development machine. + +| Config | Model | Notes | +| --- | --- | --- | +| [`configs/qwen3.5-0.8b.json`](configs/qwen3.5-0.8b.json) | [`lmstudio-community/Qwen3.5-0.8B-GGUF`](https://huggingface.co/lmstudio-community/Qwen3.5-0.8B-GGUF) | Default small multimodal example. | +| [`configs/gemma-4-12b-it-qat.json`](configs/gemma-4-12b-it-qat.json) | [`unsloth/gemma-4-12B-it-qat-GGUF`](https://huggingface.co/unsloth/gemma-4-12B-it-qat-GGUF) | Larger Gemma 4 QAT multimodal config with projector. | +| [`configs/qwen3.6-27b.json`](configs/qwen3.6-27b.json) | [`unsloth/Qwen3.6-27B-GGUF`](https://huggingface.co/unsloth/Qwen3.6-27B-GGUF) | Larger Qwen3.6 multimodal config. | +| [`configs/qwen3.6-35b-a3b.json`](configs/qwen3.6-35b-a3b.json) | [`unsloth/Qwen3.6-35B-A3B-GGUF`](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF) | Larger Qwen3.6 MoE multimodal config. | +| [`configs/gpt-oss-120b.json`](configs/gpt-oss-120b.json) | [`ggml-org/gpt-oss-120b-GGUF`](https://huggingface.co/ggml-org/gpt-oss-120b-GGUF) | Large text-only split-GGUF config. | + +The larger model configs default to `n_gpu_layers: -1` and `flash_attn: true`. + +## Client Examples + +Point an OpenAI-compatible client at the local `/v1` base URL. + +### Chat Completions + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="not-used") + +response = client.chat.completions.create( + model="qwen3.5-0.8b-vl", + messages=[{"role": "user", "content": "What is the capital of France?"}], +) +print(response.choices[0].message.content) +``` + +### Responses API + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="not-used") + +response = client.responses.create( + model="qwen3.5-0.8b-vl", + input="Write one sentence about why prefix caching helps batched inference.", +) +print(response.output_text) +``` + +## API Surface + +| Endpoint | Purpose | Reference | +| --- | --- | --- | +| `POST /v1/completions` | Legacy text completions with streaming, stop sequences, logprobs, penalties, seeds, and grammar-backed JSON output. | [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions) | +| `POST /v1/chat/completions` | Chat completions with streaming, tools, forced tool choice, reasoning parsing, multimodal content parts, and structured response parsing. | [OpenAI Chat API](https://platform.openai.com/docs/api-reference/chat) | +| `POST /v1/responses` | Stateless Responses API compatibility for clients that use response items and response events. | [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) | +| `WS /v1/responses` | Stateful websocket Responses transport with per-connection `previous_response_id` replay. | [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) | +| `GET /v1/models` | Returns the configured model alias. | [OpenAI Models API](https://platform.openai.com/docs/api-reference/models) | +| `GET /healthz` | Returns a simple typed health response. | | +| `GET /metrics` | Exposes scheduler, cache, draft, and model metrics in Prometheus text format. | [Prometheus exposition format](https://prometheus.io/docs/instrumenting/exposition_formats/) | + +## Config Overview + +Config files have three top-level sections. + +```json +{ + "server": {}, + "model": {}, + "disk_cache": {} +} +``` + +| Section | Required | Purpose | +| --- | --- | --- | +| `server` | No | Uvicorn host and port settings. | +| `model` | Yes | Model source, llama.cpp runtime settings, chat formatting, LoRA, MTMD, draft decoding, and output parsing. | +| `disk_cache` | No | Optional serialized sequence cache for repeated prompt prefixes. | + +## `server` + +Use `server.host` and `server.port` to choose the bind address. + +```json +{ + "server": { + "host": "0.0.0.0", + "port": 8000 + } +} +``` + +| Field | Default | Notes | +| --- | --- | --- | +| `host` | `127.0.0.1` | Use `0.0.0.0` to expose the server on the network. | +| `port` | `8000` | Passed directly to `uvicorn.run()`. | + +## `model` Source + +Load a local GGUF with `path` or download a GGUF from Hugging Face with `from_pretrained`. + +```json +{ + "model": { + "alias": "qwen3.5-0.8b-vl", + "from_pretrained": { + "repo_id": "lmstudio-community/Qwen3.5-0.8B-GGUF", + "filename": "Qwen3.5-0.8B-Q8_0.gguf" + } + } +} +``` + +| Field | Notes | +| --- | --- | +| `path` | Local GGUF path. | +| `from_pretrained.repo_id` | Hugging Face model repository. | +| `from_pretrained.filename` | File name or glob pattern for the GGUF. | +| `from_pretrained.additional_files` | Extra files to download from the same repository. | +| `from_pretrained.cache_dir` | Optional Hugging Face cache directory. | +| `alias` | Model id returned by `/v1/models` and used by OpenAI-compatible clients. | + +See the [Hugging Face Hub download guide](https://huggingface.co/docs/huggingface_hub/guides/download) for cache behavior and repository file resolution. + +## llama.cpp Runtime Settings + +Most model runtime fields map to `llama_model_params` or `llama_context_params` in [`llama.h`](https://github.com/ggml-org/llama.cpp/blob/master/include/llama.h). + +```json +{ + "model": { + "n_ctx": 32768, + "n_seq_max": 64, + "n_batch": 128, + "n_ubatch": 128, + "threads": 4, + "threads_batch": 8, + "kv_unified": true, + "use_mmap": true, + "use_mlock": true + } +} +``` + +| Field | Purpose | +| --- | --- | +| `n_ctx` | Total context size. | +| `n_seq_max` | Maximum number of concurrent llama.cpp sequence ids. | +| `n_batch` | Logical batch capacity. | +| `n_ubatch` | Physical microbatch capacity. | +| `threads` | Decode thread count. | +| `threads_batch` | Prefill and batch thread count. | +| `kv_unified` | Selects unified or per-sequence memory layout. | +| `store_logits` | Keeps logits after decode when needed by sampling or diagnostics. | +| `use_mmap` | Memory maps model weights. | +| `use_mlock` | Attempts to lock model pages into RAM. | + +GPU and backend-related fields are passed through to llama.cpp when set. + +```json +{ + "model": { + "n_gpu_layers": -1, + "split_mode": 1, + "main_gpu": 0, + "tensor_split": [1.0], + "flash_attn": true, + "offload_kqv": true, + "op_offload": true + } +} +``` + +## Chat Template + +`model.chat_template` is a Jinja chat template compatible with the style used by [Hugging Face chat templates](https://huggingface.co/docs/transformers/chat_templating). + +```json +{ + "model": { + "chat_template": "{{ bos_token }}{{ messages[0].content }}{{ eos_token }}" + } +} +``` + +Use an array of strings when the template is too large to read or edit as one JSON string. + +```json +{ + "model": { + "chat_template": [ + "{{ bos_token }}", + "{{ messages[0].content }}", + "{{ eos_token }}" + ] + } +} +``` + +The checked-in [`configs/qwen3.5-0.8b.json`](configs/qwen3.5-0.8b.json) includes a Qwen3.5 template with reasoning text, tool calls, forced tool choice, image markers, and video markers. + +## Response Parsing + +`model.response_schema` parses generated text into OpenAI-compatible fields with JSON Schema plus the Hugging Face `x-regex` extensions. + +```json +{ + "model": { + "response_schema": { + "type": "object", + "properties": { + "role": {"const": "assistant"}, + "content": { + "type": "string", + "x-regex": "^(.*)$" + } + }, + "required": ["role"] + } + } +} +``` + +Use `x-regex-iterator` and `x-regex-key-value` to parse repeated tool-call blocks. + +See [Hugging Face response parsing](https://huggingface.co/docs/transformers/chat_response_parsing) and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/reference) for the underlying schema concepts. + +## Multimodal `model.mtmd` + +`model.mtmd` loads a llama.cpp multimodal projector and enables OpenAI-style image and audio content parts. + +```json +{ + "model": { + "mtmd": { + "mmproj_from_pretrained": { + "repo_id": "lmstudio-community/Qwen3.5-0.8B-GGUF", + "filename": "mmproj-Qwen3.5-0.8B-BF16.gguf" + }, + "embedding_cache": { + "path": ".cache/mtmd-embeddings", + "max_bytes": 1073741824 + }, + "image_max_bytes": 20971520, + "audio_max_bytes": 104857600, + "image_timeout_seconds": 10.0 + } + } +} +``` + +| Field | Purpose | +| --- | --- | +| `mmproj_path` | Local multimodal projector path. | +| `mmproj_from_pretrained` | Hugging Face projector source. | +| `embedding_cache.path` | Directory for cached image and audio embeddings. | +| `embedding_cache.max_bytes` | Maximum embedding cache size. | +| `image_max_bytes` | Maximum image payload size. | +| `audio_max_bytes` | Maximum audio payload size. | +| `image_timeout_seconds` | Timeout for remote image and audio URL fetches. | + +Send image inputs with OpenAI chat content parts. + +```json +{ + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}} + ] + } + ] +} +``` + +Send audio inputs as a URL or as base64 `input_audio` content. + +```json +{ + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Transcribe this audio."}, + {"type": "input_audio", "input_audio": {"data": "...base64...", "format": "wav"}} + ] + } + ] +} +``` + +## Media Loading Policy + +Remote `http:` and `https:` media URLs are unrestricted unless `allowed_media_domains` is set. + +```json +{ + "model": { + "mtmd": { + "allowed_media_domains": ["example.com", "static.example.com"] + } + } +} +``` + +Local `file:` media URLs are disabled unless `allowed_local_media_path` is set. + +```json +{ + "model": { + "mtmd": { + "allowed_local_media_path": "/srv/llama-cpp-python/media" + } + } +} +``` + +`allowed_media_domains` matches exact hostnames and does not allow wildcard patterns. + +## LoRA `model.loras` + +Load LoRA adapters once at startup from local files or Hugging Face. + +```json +{ + "model": { + "loras": [ + { + "from_pretrained": { + "repo_id": "example/qwen-lora-gguf", + "filename": "adapter.gguf" + }, + "scale": 1.0 + } + ] + } +} +``` + +The current implementation does not hot-swap LoRAs per request. + +## Draft Decoding + +Set `model.draft_model` to enable speculative draft providers. + +```json +{ + "model": { + "draft_model": "prompt-lookup-decoding", + "draft_model_num_pred_tokens": 8, + "draft_model_max_ngram_size": 4 + } +} +``` + +### Multi-Token Prediction (MTP) + +Use MTP when the loaded model and llama.cpp build expose the required draft state. + +```json +{ + "model": { + "draft_model": "draft-mtp", + "draft_model_num_pred_tokens": 2, + "draft_model_threads": 4, + "draft_model_threads_batch": 8 + } +} +``` + +MTP currently applies to text-only requests. + +## Disk Sequence Cache + +`disk_cache` stores serialized llama.cpp sequence state for repeated prompt prefixes. + +```json +{ + "disk_cache": { + "path": ".cache/sequences", + "max_bytes": 1073741824, + "min_tokens": 128 + } +} +``` + +| Field | Purpose | +| --- | --- | +| `path` | Directory for cached sequence files. | +| `max_bytes` | Maximum cache size before background cleanup removes entries. | +| `min_tokens` | Minimum prefix length that is worth saving. | + +The cache is versioned by model and context compatibility data and should be treated as ephemeral. diff --git a/examples/server/configs/gemma-4-12b-it-qat.json b/examples/server/configs/gemma-4-12b-it-qat.json new file mode 100644 index 0000000000..fefeda8da0 --- /dev/null +++ b/examples/server/configs/gemma-4-12b-it-qat.json @@ -0,0 +1,451 @@ +{ + "server": { + "host": "0.0.0.0", + "port": 8000 + }, + "model": { + "alias": "gemma-4-12b-it-qat", + "from_pretrained": { + "repo_id": "unsloth/gemma-4-12B-it-qat-GGUF", + "filename": "gemma-4-12B-it-qat-UD-Q4_K_XL.gguf" + }, + "mtmd": { + "mmproj_from_pretrained": { + "repo_id": "unsloth/gemma-4-12B-it-qat-GGUF", + "filename": "mmproj-BF16.gguf" + } + }, + "n_ctx": 32768, + "max_output_tokens": 4096, + "n_seq_max": 8, + "n_batch": 512, + "n_ubatch": 512, + "threads": 8, + "threads_batch": 8, + "kv_unified": true, + "store_logits": false, + "use_mmap": true, + "use_mlock": false, + "n_gpu_layers": -1, + "flash_attn": true, + "response_schema": { + "type": "object", + "properties": { + "role": { + "const": "assistant" + }, + "reasoning_content": { + "type": "string", + "x-regex": "^(?:<\\|turn>model\\n)?<\\|channel>thought\\n(.*?)<channel\\|>" + }, + "content": { + "type": "string", + "x-regex": "^(?:<\\|turn>model\\n)?(?:(?:<\\|channel>thought\\n).*?<channel\\|>)?(.*?)(?=<\\|tool_call>|<turn\\|>|$)" + }, + "tool_calls": { + "type": "array", + "x-regex-iterator": "<\\|tool_call>(call:[^\\{]+\\{.*?\\})<tool_call\\|>", + "items": { + "type": "object", + "properties": { + "type": { + "const": "function" + }, + "function": { + "type": "object", + "properties": { + "name": { + "type": "string", + "x-regex": "^call:(\\w+)\\{" + }, + "arguments": { + "type": "object", + "x-regex": "^call:\\w+(\\{.*\\})$", + "x-parser": "gemma4-tool-call", + "additionalProperties": true + } + }, + "required": [ + "name", + "arguments" + ] + } + }, + "required": [ + "type", + "function" + ] + } + } + }, + "required": [ + "role" + ] + }, + "chat_template": [ + "{%- macro format_parameters(properties, required, filter_keys=false) -%}\n", + " {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n", + " {%- set ns = namespace(found_first=false) -%}\n", + " {%- for key, value in properties | dictsort -%}\n", + " {%- set add_comma = false -%}\n", + " {%- if not filter_keys or key not in standard_keys -%}\n", + " {%- if ns.found_first %},{% endif -%}\n", + " {%- set ns.found_first = true -%}\n", + " {{ key }}:{\n", + " {%- if value['description'] -%}\n", + " description:<|\"|>{{ value['description'] }}<|\"|>\n", + " {%- set add_comma = true -%}\n", + " {%- endif -%}\n", + " {%- if value['type'] | upper == 'STRING' -%}\n", + " {%- if value['enum'] -%}\n", + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n", + " enum:{{ format_argument(value['enum']) }}\n", + " {%- endif -%}\n", + " {%- elif value['type'] | upper == 'ARRAY' -%}\n", + " {%- if value['items'] is mapping and value['items'] -%}\n", + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n", + " items:{\n", + " {%- set ns_items = namespace(found_first=false) -%}\n", + " {%- for item_key, item_value in value['items'] | dictsort -%}\n", + " {%- if item_value is not none -%}\n", + " {%- if ns_items.found_first %},{% endif -%}\n", + " {%- set ns_items.found_first = true -%}\n", + " {%- if item_key == 'properties' -%}\n", + " properties:{\n", + " {%- if item_value is mapping -%}\n", + " {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n", + " {%- endif -%}\n", + " }\n", + " {%- elif item_key == 'required' -%}\n", + " required:[\n", + " {%- for req_item in item_value -%}\n", + " <|\"|>{{- req_item -}}<|\"|>\n", + " {%- if not loop.last %},{% endif -%}\n", + " {%- endfor -%}\n", + " ]\n", + " {%- elif item_key == 'type' -%}\n", + " {%- if item_value is string -%}\n", + " type:{{ format_argument(item_value | upper) }}\n", + " {%- else -%}\n", + " type:{{ format_argument(item_value | map('upper') | list) }}\n", + " {%- endif -%}\n", + " {%- else -%}\n", + " {{ item_key }}:{{ format_argument(item_value) }}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " }\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " {%- if value['nullable'] %}\n", + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n", + " nullable:true\n", + " {%- endif -%}\n", + " {%- if value['type'] | upper == 'OBJECT' -%}\n", + " {%- if value['properties'] is defined and value['properties'] is mapping -%}\n", + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n", + " properties:{\n", + " {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n", + " }\n", + " {%- elif value is mapping -%}\n", + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n", + " properties:{\n", + " {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}\n", + " }\n", + " {%- endif -%}\n", + " {%- if value['required'] -%}\n", + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n", + " required:[\n", + " {%- for item in value['required'] | default([]) -%}\n", + " <|\"|>{{- item -}}<|\"|>\n", + " {%- if not loop.last %},{% endif -%}\n", + " {%- endfor -%}\n", + " ]\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n", + " type:<|\"|>{{ value['type'] | upper }}<|\"|>}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + "{%- endmacro -%}\n", + "{%- macro format_function_declaration(tool_data) -%}\n", + " declaration:{{- tool_data['function']['name'] -}}{description:<|\"|>{{- tool_data['function']['description'] -}}<|\"|>\n", + " {%- set params = tool_data['function']['parameters'] -%}\n", + " {%- if params -%}\n", + " ,parameters:{\n", + " {%- if params['properties'] -%}\n", + " properties:{ {{- format_parameters(params['properties'], params['required']) -}} },\n", + " {%- endif -%}\n", + " {%- if params['required'] -%}\n", + " required:[\n", + " {%- for item in params['required'] -%}\n", + " <|\"|>{{- item -}}<|\"|>\n", + " {{- ',' if not loop.last -}}\n", + " {%- endfor -%}\n", + " ],\n", + " {%- endif -%}\n", + " {%- if params['type'] -%}\n", + " type:<|\"|>{{- params['type'] | upper -}}<|\"|>}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " {%- if 'response' in tool_data['function'] -%}\n", + " {%- set response_declaration = tool_data['function']['response'] -%}\n", + " ,response:{\n", + " {%- if response_declaration['description'] -%}\n", + " description:<|\"|>{{- response_declaration['description'] -}}<|\"|>,\n", + " {%- endif -%}\n", + " {%- if response_declaration['type'] | upper == 'OBJECT' -%}\n", + " type:<|\"|>{{- response_declaration['type'] | upper -}}<|\"|>}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " }\n", + "{%- endmacro -%}\n", + "{%- macro format_argument(argument, escape_keys=True) -%}\n", + " {%- if argument is string -%}\n", + " {{- '<|\"|>' + argument + '<|\"|>' -}}\n", + " {%- elif argument is boolean -%}\n", + " {{- 'true' if argument else 'false' -}}\n", + " {%- elif argument is mapping -%}\n", + " {{- '{' -}}\n", + " {%- set ns = namespace(found_first=false) -%}\n", + " {%- for key, value in argument | dictsort -%}\n", + " {%- if ns.found_first %},{% endif -%}\n", + " {%- set ns.found_first = true -%}\n", + " {%- if escape_keys -%}\n", + " {{- '<|\"|>' + key + '<|\"|>' -}}\n", + " {%- else -%}\n", + " {{- key -}}\n", + " {%- endif -%}\n", + " :{{- format_argument(value, escape_keys=escape_keys) -}}\n", + " {%- endfor -%}\n", + " {{- '}' -}}\n", + " {%- elif argument is sequence -%}\n", + " {{- '[' -}}\n", + " {%- for item in argument -%}\n", + " {{- format_argument(item, escape_keys=escape_keys) -}}\n", + " {%- if not loop.last %},{% endif -%}\n", + " {%- endfor -%}\n", + " {{- ']' -}}\n", + " {%- else -%}\n", + " {{- argument -}}\n", + " {%- endif -%}\n", + "{%- endmacro -%}\n", + "{%- macro strip_thinking(text) -%}\n", + " {%- set ns = namespace(result='') -%}\n", + " {%- for part in text.split('<channel|>') -%}\n", + " {%- if '<|channel>' in part -%}\n", + " {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n", + " {%- else -%}\n", + " {%- set ns.result = ns.result + part -%}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " {{- ns.result | trim -}}\n", + "{%- endmacro -%}\n", + "\n", + "{%- macro format_tool_response_block(tool_name, response) -%}\n", + " {{- '<|tool_response>' -}}\n", + " {%- if response is mapping -%}\n", + " {{- 'response:' + tool_name + '{' -}}\n", + " {%- for key, value in response | dictsort -%}\n", + " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n", + " {%- if not loop.last %},{% endif -%}\n", + " {%- endfor -%}\n", + " {{- '}' -}}\n", + " {%- else -%}\n", + " {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}\n", + " {%- endif -%}\n", + " {{- '<tool_response|>' -}}\n", + "{%- endmacro -%}\n", + "\n", + "{%- set ns = namespace(prev_message_type=None) -%}\n", + "{%- set loop_messages = messages -%}\n", + "{{- bos_token -}}\n", + "{#- Handle System/Tool Definitions Block -#}\n", + "{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n", + " {{- '<|turn>system\\n' -}}\n", + " {#- Inject Thinking token at the very top of the FIRST system turn -#}\n", + " {%- if enable_thinking is defined and enable_thinking -%}\n", + " {{- '<|think|>\\n' -}}\n", + " {%- set ns.prev_message_type = 'think' -%}\n", + " {%- endif -%}\n", + " {%- if messages[0]['role'] in ['system', 'developer'] -%}\n", + " {%- if messages[0]['content'] is string -%}\n", + " {{- messages[0]['content'] | trim -}}\n", + " {%- elif messages[0]['content'] is sequence -%}\n", + " {%- for item in messages[0]['content'] -%}\n", + " {{- item['text'] | trim + ' '-}}\n", + " {%- endfor -%}\n", + " {%- endif -%}\n", + " {%- set loop_messages = messages[1:] -%}\n", + " {%- endif -%}\n", + " {%- if tools -%}\n", + " {%- for tool in tools %}\n", + " {{- '<|tool>' -}}\n", + " {{- format_function_declaration(tool) | trim -}}\n", + " {{- '<tool|>' -}}\n", + " {%- endfor %}\n", + " {%- set ns.prev_message_type = 'tool' -%}\n", + " {%- endif -%}\n", + " {{- '<turn|>\\n' -}}\n", + "{%- endif %}\n", + "\n", + "{#- Pre-scan: find last user message index for reasoning guard -#}\n", + "{%- set ns_turn = namespace(last_user_idx=-1) -%}\n", + "{%- for i in range(loop_messages | length) -%}\n", + " {%- if loop_messages[i]['role'] == 'user' -%}\n", + " {%- set ns_turn.last_user_idx = i -%}\n", + " {%- endif -%}\n", + "{%- endfor -%}\n", + "\n", + "{#- Loop through messages -#}\n", + "{%- for message in loop_messages -%}\n", + " {%- if message['role'] != 'tool' -%}\n", + " {%- set ns.prev_message_type = None -%}\n", + " {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n", + " {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}\n", + " {%- set prev_nt = namespace(role=None, found=false) -%}\n", + " {%- if loop.index0 > 0 -%}\n", + " {%- for j in range(loop.index0 - 1, -1, -1) -%}\n", + " {%- if not prev_nt.found -%}\n", + " {%- if loop_messages[j]['role'] != 'tool' -%}\n", + " {%- set prev_nt.role = loop_messages[j]['role'] -%}\n", + " {%- set prev_nt.found = true -%}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " {%- endif -%}\n", + " {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}\n", + " {%- if not continue_same_model_turn -%}\n", + " {{- '<|turn>' + role + '\\n' }}\n", + " {%- endif -%}\n", + "\n", + " {#- Render reasoning/reasoning_content as thinking channel -#}\n", + " {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n", + " {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n", + " {{- '<|channel>thought\\n' + thinking_text + '\\n<channel|>' -}}\n", + " {%- endif -%}\n", + "\n", + " {%- if message['tool_calls'] -%}\n", + " {%- for tool_call in message['tool_calls'] -%}\n", + " {%- set function = tool_call['function'] -%}\n", + " {{- '<|tool_call>call:' + function['name'] + '{' -}}\n", + " {%- if function['arguments'] is mapping -%}\n", + " {%- set ns_args = namespace(found_first=false) -%}\n", + " {%- for key, value in function['arguments'] | dictsort -%}\n", + " {%- if ns_args.found_first %},{% endif -%}\n", + " {%- set ns_args.found_first = true -%}\n", + " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n", + " {%- endfor -%}\n", + " {%- elif function['arguments'] is string -%}\n", + " {{- function['arguments'] -}}\n", + " {%- endif -%}\n", + " {{- '}<tool_call|>' -}}\n", + " {%- endfor -%}\n", + " {%- set ns.prev_message_type = 'tool_call' -%}\n", + " {%- endif -%}\n", + "\n", + " {%- set ns_tr_out = namespace(flag=false) -%}\n", + " {%- if message.get('tool_responses') -%}\n", + " {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n", + " {%- for tool_response in message['tool_responses'] -%}\n", + " {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}\n", + " {%- set ns_tr_out.flag = true -%}\n", + " {%- set ns.prev_message_type = 'tool_response' -%}\n", + " {%- endfor -%}\n", + " {%- elif message.get('tool_calls') -%}\n", + " {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#}\n", + " {%- set ns_tool_scan = namespace(stopped=false) -%}\n", + " {%- for k in range(loop.index0 + 1, loop_messages | length) -%}\n", + " {%- if ns_tool_scan.stopped -%}\n", + " {%- elif loop_messages[k]['role'] != 'tool' -%}\n", + " {%- set ns_tool_scan.stopped = true -%}\n", + " {%- else -%}\n", + " {%- set follow = loop_messages[k] -%}\n", + " {#- Resolve tool_call_id to function name -#}\n", + " {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}\n", + " {%- for tc in message['tool_calls'] -%}\n", + " {%- if tc.get('id') == follow.get('tool_call_id') -%}\n", + " {%- set ns_tname.name = tc['function']['name'] -%}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " {#- Handle content as string or content-parts array -#}\n", + " {%- set tool_body = follow.get('content') -%}\n", + " {%- if tool_body is string -%}\n", + " {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n", + " {%- elif tool_body is sequence and tool_body is not string -%}\n", + " {%- set ns_txt = namespace(s='') -%}\n", + " {%- for part in tool_body -%}\n", + " {%- if part.get('type') == 'text' -%}\n", + " {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n", + " {%- for part in tool_body -%}\n", + " {%- if part.get('type') == 'image' -%}\n", + " {{- '<|image|>' -}}\n", + " {%- elif part.get('type') == 'audio' -%}\n", + " {{- '<|audio|>' -}}\n", + " {%- elif part.get('type') == 'video' -%}\n", + " {{- '<|video|>' -}}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " {%- else -%}\n", + " {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n", + " {%- endif -%}\n", + " {%- set ns_tr_out.flag = true -%}\n", + " {%- set ns.prev_message_type = 'tool_response' -%}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " {%- endif -%}\n", + "\n", + " {%- set captured_content -%}\n", + " {%- if message['content'] is string -%}\n", + " {%- if role == 'model' -%}\n", + " {{- strip_thinking(message['content']) -}}\n", + " {%- else -%}\n", + " {{- message['content'] | trim -}}\n", + " {%- endif -%}\n", + " {%- elif message['content'] is sequence -%}\n", + " {%- for item in message['content'] -%}\n", + " {%- if item['type'] == 'text' -%}\n", + " {%- if role == 'model' -%}\n", + " {{- strip_thinking(item['text']) -}}\n", + " {%- else -%}\n", + " {{- item['text'] | trim -}}\n", + " {%- endif -%}\n", + " {%- elif item['type'] == 'image' -%}\n", + " {{- '<|image|>' -}}\n", + " {%- set ns.prev_message_type = 'image' -%}\n", + " {%- elif item['type'] == 'audio' -%}\n", + " {{- '<|audio|>' -}}\n", + " {%- set ns.prev_message_type = 'audio' -%}\n", + " {%- elif item['type'] == 'video' -%}\n", + " {{- '<|video|>' -}}\n", + " {%- set ns.prev_message_type = 'video' -%}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " {%- endif -%}\n", + " {%- endset -%}\n", + "\n", + " {{- captured_content -}}\n", + " {%- set has_content = captured_content | trim | length > 0 -%}\n", + "\n", + " {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n", + " {{- '<|tool_response>' -}}\n", + " {%- elif not (ns_tr_out.flag and not has_content) -%}\n", + " {{- '<turn|>\\n' -}}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + "{%- endfor -%}\n", + "\n", + "{%- if add_generation_prompt -%}\n", + " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n", + " {{- '<|turn>model\\n' -}}\n", + " {%- if not enable_thinking | default(false) -%}\n", + " {{- '<|channel>thought\\n<channel|>' -}}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + "{%- endif -%}" + ] + } +} diff --git a/examples/server/configs/gpt-oss-120b.json b/examples/server/configs/gpt-oss-120b.json new file mode 100644 index 0000000000..2fa7a76f1e --- /dev/null +++ b/examples/server/configs/gpt-oss-120b.json @@ -0,0 +1,417 @@ +{ + "server": { + "host": "0.0.0.0", + "port": 8000 + }, + "model": { + "alias": "gpt-oss-120b", + "from_pretrained": { + "repo_id": "ggml-org/gpt-oss-120b-GGUF", + "filename": "gpt-oss-120b-mxfp4-00001-of-00003.gguf", + "additional_files": [ + "gpt-oss-120b-mxfp4-00002-of-00003.gguf", + "gpt-oss-120b-mxfp4-00003-of-00003.gguf" + ] + }, + "n_ctx": 32768, + "max_output_tokens": 4096, + "n_seq_max": 4, + "n_batch": 512, + "n_ubatch": 512, + "threads": 8, + "threads_batch": 8, + "kv_unified": true, + "store_logits": false, + "use_mmap": true, + "use_mlock": false, + "n_gpu_layers": -1, + "flash_attn": true, + "response_schema": { + "type": "object", + "properties": { + "role": { + "const": "assistant" + }, + "reasoning_content": { + "type": "string", + "x-regex": "^(?:<\\|start\\|>assistant)?<\\|channel\\|>analysis<\\|message\\|>(.*?)(?=<\\|end\\|>|<\\|start\\|>assistant|$)" + }, + "content": { + "type": "string", + "x-regex": "^(?:.*?<\\|start\\|>assistant)?<\\|channel\\|>final<\\|message\\|>(.*?)(?=<\\|return\\|>|<\\|end\\|>|$)" + }, + "tool_calls": { + "type": "array", + "x-regex-iterator": "(<\\|start\\|>assistant to=functions\\.\\w+<\\|channel\\|>commentary json<\\|message\\|>.*?<\\|call\\|>)", + "items": { + "type": "object", + "properties": { + "type": { + "const": "function" + }, + "function": { + "type": "object", + "properties": { + "name": { + "type": "string", + "x-regex": "^<\\|start\\|>assistant to=functions\\.(\\w+)<\\|channel\\|>commentary json<\\|message\\|>" + }, + "arguments": { + "type": "object", + "x-regex": "^<\\|start\\|>assistant to=functions\\.\\w+<\\|channel\\|>commentary json<\\|message\\|>(.*?)<\\|call\\|>$", + "x-parser": "json", + "additionalProperties": true + } + }, + "required": [ + "name", + "arguments" + ] + } + }, + "required": [ + "type", + "function" + ] + } + } + }, + "required": [ + "role" + ] + }, + "chat_template": [ + "{#-\n", + " In addition to the normal inputs of `messages` and `tools`, this template also accepts the\n", + " following kwargs:\n", + " - \"builtin_tools\": A list, can contain \"browser\" and/or \"python\".\n", + " - \"model_identity\": A string that optionally describes the model identity.\n", + " - \"reasoning_effort\": A string that describes the reasoning effort, defaults to \"medium\".\n", + " #}\n", + "\n", + "{#- Tool Definition Rendering ============================================== #}\n", + "{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}\n", + " {%- if param_spec.type == \"array\" -%}\n", + " {%- if param_spec['items'] -%}\n", + " {%- if param_spec['items']['type'] == \"string\" -%}\n", + " {{- \"string[]\" }}\n", + " {%- elif param_spec['items']['type'] == \"number\" -%}\n", + " {{- \"number[]\" }}\n", + " {%- elif param_spec['items']['type'] == \"integer\" -%}\n", + " {{- \"number[]\" }}\n", + " {%- elif param_spec['items']['type'] == \"boolean\" -%}\n", + " {{- \"boolean[]\" }}\n", + " {%- else -%}\n", + " {%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}\n", + " {%- if inner_type == \"object | object\" or inner_type|length > 50 -%}\n", + " {{- \"any[]\" }}\n", + " {%- else -%}\n", + " {{- inner_type + \"[]\" }}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " {%- if param_spec.nullable -%}\n", + " {{- \" | null\" }}\n", + " {%- endif -%}\n", + " {%- else -%}\n", + " {{- \"any[]\" }}\n", + " {%- if param_spec.nullable -%}\n", + " {{- \" | null\" }}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " {%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}\n", + " {#- Handle array of types like [\"object\", \"object\"] from Union[dict, list] #}\n", + " {%- if param_spec.type | length > 1 -%}\n", + " {{- param_spec.type | join(\" | \") }}\n", + " {%- else -%}\n", + " {{- param_spec.type[0] }}\n", + " {%- endif -%}\n", + " {%- elif param_spec.oneOf -%}\n", + " {#- Handle oneOf schemas - check for complex unions and fallback to any #}\n", + " {%- set has_object_variants = false -%}\n", + " {%- for variant in param_spec.oneOf -%}\n", + " {%- if variant.type == \"object\" -%}\n", + " {%- set has_object_variants = true -%}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " {%- if has_object_variants and param_spec.oneOf|length > 1 -%}\n", + " {{- \"any\" }}\n", + " {%- else -%}\n", + " {%- for variant in param_spec.oneOf -%}\n", + " {{- render_typescript_type(variant, required_params) -}}\n", + " {%- if variant.description %}\n", + " {{- \"// \" + variant.description }}\n", + " {%- endif -%}\n", + " {%- if variant.default is defined %}\n", + " {{ \"// default: \" + variant.default|tojson }}\n", + " {%- endif -%}\n", + " {%- if not loop.last %}\n", + " {{- \" | \" }}\n", + " {% endif -%}\n", + " {%- endfor -%}\n", + " {%- endif -%}\n", + " {%- elif param_spec.type == \"string\" -%}\n", + " {%- if param_spec.enum -%}\n", + " {{- '\"' + param_spec.enum|join('\" | \"') + '\"' -}}\n", + " {%- else -%}\n", + " {{- \"string\" }}\n", + " {%- if param_spec.nullable %}\n", + " {{- \" | null\" }}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " {%- elif param_spec.type == \"number\" -%}\n", + " {{- \"number\" }}\n", + " {%- elif param_spec.type == \"integer\" -%}\n", + " {{- \"number\" }}\n", + " {%- elif param_spec.type == \"boolean\" -%}\n", + " {{- \"boolean\" }}\n", + "\n", + " {%- elif param_spec.type == \"object\" -%}\n", + " {%- if param_spec.properties -%}\n", + " {{- \"{\\n\" }}\n", + " {%- for prop_name, prop_spec in param_spec.properties.items() -%}\n", + " {{- prop_name -}}\n", + " {%- if prop_name not in (param_spec.required or []) -%}\n", + " {{- \"?\" }}\n", + " {%- endif -%}\n", + " {{- \": \" }}\n", + " {{ render_typescript_type(prop_spec, param_spec.required or []) }}\n", + " {%- if not loop.last -%}\n", + " {{-\", \" }}\n", + " {%- endif -%}\n", + " {%- endfor -%}\n", + " {{- \"}\" }}\n", + " {%- else -%}\n", + " {{- \"object\" }}\n", + " {%- endif -%}\n", + " {%- else -%}\n", + " {{- \"any\" }}\n", + " {%- endif -%}\n", + "{%- endmacro -%}\n", + "\n", + "{%- macro render_tool_namespace(namespace_name, tools) -%}\n", + " {{- \"## \" + namespace_name + \"\\n\\n\" }}\n", + " {{- \"namespace \" + namespace_name + \" {\\n\\n\" }}\n", + " {%- for tool in tools %}\n", + " {%- set tool = tool.function %}\n", + " {{- \"// \" + tool.description + \"\\n\" }}\n", + " {{- \"type \"+ tool.name + \" = \" }}\n", + " {%- if tool.parameters and tool.parameters.properties %}\n", + " {{- \"(_: {\\n\" }}\n", + " {%- for param_name, param_spec in tool.parameters.properties.items() %}\n", + " {%- if param_spec.description %}\n", + " {{- \"// \" + param_spec.description + \"\\n\" }}\n", + " {%- endif %}\n", + " {{- param_name }}\n", + " {%- if param_name not in (tool.parameters.required or []) -%}\n", + " {{- \"?\" }}\n", + " {%- endif -%}\n", + " {{- \": \" }}\n", + " {{- render_typescript_type(param_spec, tool.parameters.required or []) }}\n", + " {%- if param_spec.default is defined -%}\n", + " {%- if param_spec.enum %}\n", + " {{- \", // default: \" + param_spec.default }}\n", + " {%- elif param_spec.oneOf %}\n", + " {{- \"// default: \" + param_spec.default }}\n", + " {%- else %}\n", + " {{- \", // default: \" + param_spec.default|tojson }}\n", + " {%- endif -%}\n", + " {%- endif -%}\n", + " {%- if not loop.last %}\n", + " {{- \",\\n\" }}\n", + " {%- else %}\n", + " {{- \",\\n\" }}\n", + " {%- endif -%}\n", + " {%- endfor %}\n", + " {{- \"}) => any;\\n\\n\" }}\n", + " {%- else -%}\n", + " {{- \"() => any;\\n\\n\" }}\n", + " {%- endif -%}\n", + " {%- endfor %}\n", + " {{- \"} // namespace \" + namespace_name }}\n", + "{%- endmacro -%}\n", + "\n", + "{%- macro render_builtin_tools(browser_tool, python_tool) -%}\n", + " {%- if browser_tool %}\n", + " {{- \"## browser\\n\\n\" }}\n", + " {{- \"// Tool for browsing.\\n\" }}\n", + " {{- \"// The `cursor` appears in brackets before each browsing display: `[{cursor}]`.\\n\" }}\n", + " {{- \"// Cite information from the tool using the following format:\\n\" }}\n", + " {{- \"// `\u3010{cursor}\u2020L{line_start}(-L{line_end})?\u3011`, for example: `\u30106\u2020L9-L11\u3011` or `\u30108\u2020L3\u3011`.\\n\" }}\n", + " {{- \"// Do not quote more than 10 words directly from the tool output.\\n\" }}\n", + " {{- \"// sources=web (default: web)\\n\" }}\n", + " {{- \"namespace browser {\\n\\n\" }}\n", + " {{- \"// Searches for information related to `query` and displays `topn` results.\\n\" }}\n", + " {{- \"type search = (_: {\\n\" }}\n", + " {{- \"query: string,\\n\" }}\n", + " {{- \"topn?: number, // default: 10\\n\" }}\n", + " {{- \"source?: string,\\n\" }}\n", + " {{- \"}) => any;\\n\\n\" }}\n", + " {{- \"// Opens the link `id` from the page indicated by `cursor` starting at line number `loc`, showing `num_lines` lines.\\n\" }}\n", + " {{- \"// Valid link ids are displayed with the formatting: `\u3010{id}\u2020.*\u3011`.\\n\" }}\n", + " {{- \"// If `cursor` is not provided, the most recent page is implied.\\n\" }}\n", + " {{- \"// If `id` is a string, it is treated as a fully qualified URL associated with `source`.\\n\" }}\n", + " {{- \"// If `loc` is not provided, the viewport will be positioned at the beginning of the document or centered on the most relevant passage, if available.\\n\" }}\n", + " {{- \"// Use this function without `id` to scroll to a new location of an opened page.\\n\" }}\n", + " {{- \"type open = (_: {\\n\" }}\n", + " {{- \"id?: number | string, // default: -1\\n\" }}\n", + " {{- \"cursor?: number, // default: -1\\n\" }}\n", + " {{- \"loc?: number, // default: -1\\n\" }}\n", + " {{- \"num_lines?: number, // default: -1\\n\" }}\n", + " {{- \"view_source?: boolean, // default: false\\n\" }}\n", + " {{- \"source?: string,\\n\" }}\n", + " {{- \"}) => any;\\n\\n\" }}\n", + " {{- \"// Finds exact matches of `pattern` in the current page, or the page given by `cursor`.\\n\" }}\n", + " {{- \"type find = (_: {\\n\" }}\n", + " {{- \"pattern: string,\\n\" }}\n", + " {{- \"cursor?: number, // default: -1\\n\" }}\n", + " {{- \"}) => any;\\n\\n\" }}\n", + " {{- \"} // namespace browser\\n\\n\" }}\n", + " {%- endif -%}\n", + "\n", + " {%- if python_tool %}\n", + " {{- \"## python\\n\\n\" }}\n", + " {{- \"Use this tool to execute Python code in your chain of thought. The code will not be shown to the user. This tool should be used for internal reasoning, but not for code that is intended to be visible to the user (e.g. when creating plots, tables, or files).\\n\\n\" }}\n", + " {{- \"When you send a message containing Python code to python, it will be executed in a stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 120.0 seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is UNKNOWN. Depends on the cluster.\\n\\n\" }}\n", + " {%- endif -%}\n", + "{%- endmacro -%}\n", + "\n", + "{#- System Message Construction ============================================ #}\n", + "{%- macro build_system_message() -%}\n", + " {%- if model_identity is not defined %}\n", + " {%- set model_identity = \"You are ChatGPT, a large language model trained by OpenAI.\" %}\n", + " {%- endif %}\n", + " {{- model_identity + \"\\n\" }}\n", + " {{- \"Knowledge cutoff: 2024-06\\n\" }}\n", + " {{- \"Current date: \" + strftime_now(\"%Y-%m-%d\") + \"\\n\\n\" }}\n", + " {%- if reasoning_effort is not string %}\n", + " {%- set reasoning_effort = \"medium\" %}\n", + " {%- endif %}\n", + " {{- \"Reasoning: \" + reasoning_effort + \"\\n\\n\" }}\n", + " {%- if builtin_tools %}\n", + " {{- \"# Tools\\n\\n\" }}\n", + " {%- set available_builtin_tools = namespace(browser=false, python=false) %}\n", + " {%- for tool in builtin_tools %}\n", + " {%- if tool == \"browser\" %}\n", + " {%- set available_builtin_tools.browser = true %}\n", + " {%- elif tool == \"python\" %}\n", + " {%- set available_builtin_tools.python = true %}\n", + " {%- endif %}\n", + " {%- endfor %}\n", + " {{- render_builtin_tools(available_builtin_tools.browser, available_builtin_tools.python) }}\n", + " {%- endif -%}\n", + " {{- \"# Valid channels: analysis, commentary, final. Channel must be included for every message.\" }}\n", + " {%- if tools -%}\n", + " {{- \"\\nCalls to these tools must go to the commentary channel: 'functions'.\" }}\n", + " {%- endif -%}\n", + "{%- endmacro -%}\n", + "\n", + "{#- Main Template Logic ================================================= #}\n", + "{#- Set defaults #}\n", + "\n", + "{#- Render system message #}\n", + "{{- \"<|start|>system<|message|>\" }}\n", + "{{- build_system_message() }}\n", + "{{- \"<|end|>\" }}\n", + "\n", + "{#- Extract developer message #}\n", + "{%- if messages[0].role == \"developer\" or messages[0].role == \"system\" %}\n", + " {%- set developer_message = messages[0].content %}\n", + " {%- set loop_messages = messages[1:] %}\n", + "{%- else %}\n", + " {%- set developer_message = \"\" %}\n", + " {%- set loop_messages = messages %}\n", + "{%- endif %}\n", + "\n", + "{#- Render developer message #}\n", + "{%- if developer_message or tools %}\n", + " {{- \"<|start|>developer<|message|>\" }}\n", + " {%- if developer_message %}\n", + " {{- \"# Instructions\\n\\n\" }}\n", + " {{- developer_message }}\n", + " {{- \"\\n\\n\" }}\n", + " {%- endif %}\n", + " {%- if tools -%}\n", + " {{- \"# Tools\\n\\n\" }}\n", + " {{- render_tool_namespace(\"functions\", tools) }}\n", + " {%- endif -%}\n", + " {{- \"<|end|>\" }}\n", + "{%- endif %}\n", + "\n", + "{#- Render messages #}\n", + "{%- set last_tool_call = namespace(name=none) %}\n", + "{%- for message in loop_messages -%}\n", + " {#- At this point only assistant/user/tool messages should remain #}\n", + " {%- if message.role == 'assistant' -%}\n", + " {#- Checks to ensure the messages are being passed in the format we expect #}\n", + " {%- if \"content\" in message %}\n", + " {%- if \"<|channel|>analysis<|message|>\" in message.content or \"<|channel|>final<|message|>\" in message.content %}\n", + " {{- raise_exception(\"You have passed a message containing <|channel|> tags in the content field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.\") }}\n", + " {%- endif %}\n", + " {%- endif %}\n", + " {%- if \"thinking\" in message %}\n", + " {%- if \"<|channel|>analysis<|message|>\" in message.thinking or \"<|channel|>final<|message|>\" in message.thinking %}\n", + " {{- raise_exception(\"You have passed a message containing <|channel|> tags in the thinking field. Instead of doing this, you should pass analysis messages (the string between '<|message|>' and '<|end|>') in the 'thinking' field, and final messages (the string between '<|message|>' and '<|end|>') in the 'content' field.\") }}\n", + " {%- endif %}\n", + " {%- endif %}\n", + " {%- if \"tool_calls\" in message %}\n", + " {#- We need very careful handling here - we want to drop the tool call analysis message if the model #}\n", + " {#- has output a later <|final|> message, but otherwise we want to retain it. This is the only case #}\n", + " {#- when we render CoT/analysis messages in inference. #}\n", + " {%- set future_final_message = namespace(found=false) %}\n", + " {%- for future_message in loop_messages[loop.index:] %}\n", + " {%- if future_message.role == 'assistant' and \"tool_calls\" not in future_message %}\n", + " {%- set future_final_message.found = true %}\n", + " {%- endif %}\n", + " {%- endfor %}\n", + " {#- We assume max 1 tool call per message, and so we infer the tool call name #}\n", + " {#- in \"tool\" messages from the most recent assistant tool call name #}\n", + " {%- set tool_call = message.tool_calls[0] %}\n", + " {%- if tool_call.function %}\n", + " {%- set tool_call = tool_call.function %}\n", + " {%- endif %}\n", + " {%- if message.content and message.thinking %}\n", + " {{- raise_exception(\"Cannot pass both content and thinking in an assistant message with tool calls! Put the analysis message in one or the other, but not both.\") }}\n", + " {%- elif message.content and not future_final_message.found %}\n", + " {{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.content + \"<|end|>\" }}\n", + " {%- elif message.thinking and not future_final_message.found %}\n", + " {{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.thinking + \"<|end|>\" }}\n", + " {%- endif %}\n", + " {{- \"<|start|>assistant to=\" }}\n", + " {{- \"functions.\" + tool_call.name + \"<|channel|>commentary \" }}\n", + " {{- (tool_call.content_type if tool_call.content_type is defined else \"json\") + \"<|message|>\" }}\n", + " {{- tool_call.arguments|tojson }}\n", + " {{- \"<|call|>\" }}\n", + " {%- set last_tool_call.name = tool_call.name %}\n", + " {%- elif loop.last and not add_generation_prompt %}\n", + " {#- Only render the CoT if the final turn is an assistant turn and add_generation_prompt is false #}\n", + " {#- This is a situation that should only occur in training, never in inference. #}\n", + " {%- if \"thinking\" in message %}\n", + " {{- \"<|start|>assistant<|channel|>analysis<|message|>\" + message.thinking + \"<|end|>\" }}\n", + " {%- endif %}\n", + " {#- <|return|> indicates the end of generation, but <|end|> does not #}\n", + " {#- <|return|> should never be an input to the model, but we include it as the final token #}\n", + " {#- when training, so the model learns to emit it. #}\n", + " {{- \"<|start|>assistant<|channel|>final<|message|>\" + message.content + \"<|return|>\" }}\n", + " {%- else %}\n", + " {#- CoT is dropped during all previous turns, so we never render it for inference #}\n", + " {{- \"<|start|>assistant<|channel|>final<|message|>\" + message.content + \"<|end|>\" }}\n", + " {%- set last_tool_call.name = none %}\n", + " {%- endif %}\n", + " {%- elif message.role == 'tool' -%}\n", + " {%- if last_tool_call.name is none %}\n", + " {{- raise_exception(\"Message has tool role, but there was no previous assistant message with a tool call!\") }}\n", + " {%- endif %}\n", + " {{- \"<|start|>functions.\" + last_tool_call.name }}\n", + " {{- \" to=assistant<|channel|>commentary<|message|>\" + message.content|tojson + \"<|end|>\" }}\n", + " {%- elif message.role == 'user' -%}\n", + " {{- \"<|start|>user<|message|>\" + message.content + \"<|end|>\" }}\n", + " {%- endif -%}\n", + "{%- endfor -%}\n", + "\n", + "{#- Generation prompt #}\n", + "{%- if add_generation_prompt -%}\n", + "<|start|>assistant\n", + "{%- endif -%}" + ] + } +} diff --git a/examples/server/configs/qwen3.5-0.8b.json b/examples/server/configs/qwen3.5-0.8b.json new file mode 100644 index 0000000000..c9f2ca944a --- /dev/null +++ b/examples/server/configs/qwen3.5-0.8b.json @@ -0,0 +1,261 @@ +{ + "server": { + "host": "0.0.0.0", + "port": 8000 + }, + "model": { + "alias": "qwen3.5-0.8b-vl", + "from_pretrained": { + "repo_id": "lmstudio-community/Qwen3.5-0.8B-GGUF", + "filename": "Qwen3.5-0.8B-Q8_0.gguf" + }, + "mtmd": { + "mmproj_from_pretrained": { + "repo_id": "lmstudio-community/Qwen3.5-0.8B-GGUF", + "filename": "mmproj-Qwen3.5-0.8B-BF16.gguf" + } + }, + "n_ctx": 32768, + "max_output_tokens": 4096, + "n_seq_max": 64, + "n_batch": 128, + "n_ubatch": 128, + "threads": 4, + "threads_batch": 8, + "kv_unified": true, + "store_logits": false, + "use_mmap": true, + "use_mlock": true, + "response_schema": { + "type": "object", + "properties": { + "role": { + "const": "assistant" + }, + "reasoning_content": { + "type": "string", + "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:<think>\\n)?(.*?)(?=</think>)" + }, + "content": { + "type": "string", + "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:(?:<think>\\n)?.*?</think>\\s*)?(.*?)(?=\\s*<tool_call>\\n|<\\|im_end\\|>$|$)" + }, + "tool_calls": { + "type": "array", + "x-regex-iterator": "<tool_call>\\n(.*?)\\n</tool_call>", + "items": { + "type": "object", + "properties": { + "type": { + "const": "function" + }, + "function": { + "type": "object", + "properties": { + "name": { + "type": "string", + "x-regex": "^<function=([^>\\n]+)>\\n" + }, + "arguments": { + "type": "object", + "x-regex": "^<function=[^>\\n]+>\\n(.*?)\\n</function>$", + "x-regex-key-value": "<parameter=(?P<key>[^>\\n]+)>\\n(?P<value>.*?)\\n</parameter>", + "additionalProperties": true + } + }, + "required": [ + "name", + "arguments" + ] + } + }, + "required": [ + "type", + "function" + ] + } + } + }, + "required": [ + "role" + ] + }, + "chat_template": [ + "{%- set image_count = namespace(value=0) %}\n", + "{%- set video_count = namespace(value=0) %}\n", + "{%- if enable_thinking is not defined and reasoning_effort is string %}\n", + " {%- set qwen_reasoning_effort = reasoning_effort|lower %}\n", + " {%- if qwen_reasoning_effort in ['none', 'minimal', 'low'] %}\n", + " {%- set enable_thinking = false %}\n", + " {%- elif qwen_reasoning_effort in ['medium', 'high'] %}\n", + " {%- set enable_thinking = true %}\n", + " {%- endif %}\n", + "{%- endif %}\n", + "{%- set forced_tool_name = none %}\n", + "{%- if tool_choice is mapping %}\n", + " {%- if tool_choice.function is defined and tool_choice.function.name is defined %}\n", + " {%- set forced_tool_name = tool_choice.function.name %}\n", + " {%- elif tool_choice.name is defined %}\n", + " {%- set forced_tool_name = tool_choice.name %}\n", + " {%- endif %}\n", + "{%- elif function_call is mapping and function_call.name is defined %}\n", + " {%- set forced_tool_name = function_call.name %}\n", + "{%- endif %}\n", + "{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n", + " {%- if content is string %}\n", + " {{- content }}\n", + " {%- elif content is iterable and content is not mapping %}\n", + " {%- for item in content %}\n", + " {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n", + " {%- if is_system_content %}\n", + " {{- raise_exception('System message cannot contain images.') }}\n", + " {%- endif %}\n", + " {%- if do_vision_count %}\n", + " {%- set image_count.value = image_count.value + 1 %}\n", + " {%- endif %}\n", + " {%- if add_vision_id %}\n", + " {{- 'Picture ' ~ image_count.value ~ ': ' }}\n", + " {%- endif %}\n", + " {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n", + " {%- elif 'video' in item or item.type == 'video' %}\n", + " {%- if is_system_content %}\n", + " {{- raise_exception('System message cannot contain videos.') }}\n", + " {%- endif %}\n", + " {%- if do_vision_count %}\n", + " {%- set video_count.value = video_count.value + 1 %}\n", + " {%- endif %}\n", + " {%- if add_vision_id %}\n", + " {{- 'Video ' ~ video_count.value ~ ': ' }}\n", + " {%- endif %}\n", + " {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n", + " {%- elif 'text' in item %}\n", + " {{- item.text }}\n", + " {%- else %}\n", + " {{- raise_exception('Unexpected item type in content.') }}\n", + " {%- endif %}\n", + " {%- endfor %}\n", + " {%- elif content is none or content is undefined %}\n", + " {{- '' }}\n", + " {%- else %}\n", + " {{- raise_exception('Unexpected content type.') }}\n", + " {%- endif %}\n", + "{%- endmacro %}\n", + "{%- if not messages %}\n", + " {{- raise_exception('No messages provided.') }}\n", + "{%- endif %}\n", + "{%- if tools and tools is iterable and tools is not mapping %}\n", + " {{- '<|im_start|>system\\n' }}\n", + " {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n", + " {%- for tool in tools %}\n", + " {{- \"\\n\" }}\n", + " {{- tool | tojson }}\n", + " {%- endfor %}\n", + " {{- \"\\n</tools>\" }}\n", + " {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n", + " {%- if messages[0].role == 'system' %}\n", + " {%- set content = render_content(messages[0].content, false, true)|trim %}\n", + " {%- if content %}\n", + " {{- '\\n\\n' + content }}\n", + " {%- endif %}\n", + " {%- endif %}\n", + " {{- '<|im_end|>\\n' }}\n", + "{%- else %}\n", + " {%- if messages[0].role == 'system' %}\n", + " {%- set content = render_content(messages[0].content, false, true)|trim %}\n", + " {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n", + " {%- endif %}\n", + "{%- endif %}\n", + "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n", + "{%- for message in messages[::-1] %}\n", + " {%- set index = (messages|length - 1) - loop.index0 %}\n", + " {%- if ns.multi_step_tool and message.role == \"user\" %}\n", + " {%- set content = render_content(message.content, false)|trim %}\n", + " {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n", + " {%- set ns.multi_step_tool = false %}\n", + " {%- set ns.last_query_index = index %}\n", + " {%- endif %}\n", + " {%- endif %}\n", + "{%- endfor %}\n", + "{%- if ns.multi_step_tool %}\n", + " {{- raise_exception('No user query found in messages.') }}\n", + "{%- endif %}\n", + "{%- for message in messages %}\n", + " {%- set content = render_content(message.content, true)|trim %}\n", + " {%- if message.role == \"system\" %}\n", + " {%- if not loop.first %}\n", + " {{- raise_exception('System message must be at the beginning.') }}\n", + " {%- endif %}\n", + " {%- elif message.role == \"user\" %}\n", + " {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n", + " {%- elif message.role == \"assistant\" %}\n", + " {%- set reasoning_content = '' %}\n", + " {%- if message.reasoning_content is string %}\n", + " {%- set reasoning_content = message.reasoning_content %}\n", + " {%- else %}\n", + " {%- if '</think>' in content %}\n", + " {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n", + " {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n", + " {%- endif %}\n", + " {%- endif %}\n", + " {%- set reasoning_content = reasoning_content|trim %}\n", + " {%- if loop.index0 > ns.last_query_index %}\n", + " {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n", + " {%- else %}\n", + " {{- '<|im_start|>' + message.role + '\\n' + content }}\n", + " {%- endif %}\n", + " {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n", + " {%- for tool_call in message.tool_calls %}\n", + " {%- if tool_call.function is defined %}\n", + " {%- set tool_call = tool_call.function %}\n", + " {%- endif %}\n", + " {%- if loop.first %}\n", + " {%- if content|trim %}\n", + " {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n", + " {%- else %}\n", + " {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n", + " {%- endif %}\n", + " {%- else %}\n", + " {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n", + " {%- endif %}\n", + " {%- if tool_call.arguments is defined %}\n", + " {%- set arguments = tool_call.arguments | from_json if tool_call.arguments is string else tool_call.arguments %}\n", + " {%- for args_name, args_value in arguments|items %}\n", + " {{- '<parameter=' + args_name + '>\\n' }}\n", + " {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n", + " {{- args_value }}\n", + " {{- '\\n</parameter>\\n' }}\n", + " {%- endfor %}\n", + " {%- endif %}\n", + " {{- '</function>\\n</tool_call>' }}\n", + " {%- endfor %}\n", + " {%- endif %}\n", + " {{- '<|im_end|>\\n' }}\n", + " {%- elif message.role == \"tool\" %}\n", + " {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n", + " {{- '<|im_start|>user' }}\n", + " {%- endif %}\n", + " {{- '\\n<tool_response>\\n' }}\n", + " {{- content }}\n", + " {{- '\\n</tool_response>' }}\n", + " {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n", + " {{- '<|im_end|>\\n' }}\n", + " {%- elif loop.last %}\n", + " {{- '<|im_end|>\\n' }}\n", + " {%- endif %}\n", + " {%- else %}\n", + " {{- raise_exception('Unexpected message role.') }}\n", + " {%- endif %}\n", + "{%- endfor %}\n", + "{%- if add_generation_prompt %}\n", + " {{- '<|im_start|>assistant\\n' }}\n", + " {%- if forced_tool_name %}\n", + " {{- '<tool_call>\\n<function=' + forced_tool_name + '>\\n' }}\n", + " {%- elif enable_thinking is defined and enable_thinking is false %}\n", + " {{- '<think>\\n\\n</think>\\n\\n' }}\n", + " {%- else %}\n", + " {{- '<think>\\n' }}\n", + " {%- endif %}\n", + "{%- endif %}" + ] + } +} diff --git a/examples/server/configs/qwen3.6-27b.json b/examples/server/configs/qwen3.6-27b.json new file mode 100644 index 0000000000..c5b42d8b72 --- /dev/null +++ b/examples/server/configs/qwen3.6-27b.json @@ -0,0 +1,263 @@ +{ + "server": { + "host": "0.0.0.0", + "port": 8000 + }, + "model": { + "alias": "qwen3.6-27b", + "from_pretrained": { + "repo_id": "unsloth/Qwen3.6-27B-GGUF", + "filename": "Qwen3.6-27B-Q8_0.gguf" + }, + "mtmd": { + "mmproj_from_pretrained": { + "repo_id": "unsloth/Qwen3.6-27B-GGUF", + "filename": "mmproj-BF16.gguf" + } + }, + "n_ctx": 32768, + "max_output_tokens": 4096, + "n_seq_max": 8, + "n_batch": 512, + "n_ubatch": 512, + "threads": 8, + "threads_batch": 8, + "kv_unified": true, + "store_logits": false, + "use_mmap": true, + "use_mlock": false, + "n_gpu_layers": -1, + "flash_attn": true, + "response_schema": { + "type": "object", + "properties": { + "role": { + "const": "assistant" + }, + "reasoning_content": { + "type": "string", + "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:<think>\\n)?(.*?)(?=</think>)" + }, + "content": { + "type": "string", + "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:(?:<think>\\n)?.*?</think>\\s*)?(.*?)(?=\\s*<tool_call>\\n|<\\|im_end\\|>$|$)" + }, + "tool_calls": { + "type": "array", + "x-regex-iterator": "<tool_call>\\n(.*?)\\n</tool_call>", + "items": { + "type": "object", + "properties": { + "type": { + "const": "function" + }, + "function": { + "type": "object", + "properties": { + "name": { + "type": "string", + "x-regex": "^<function=([^>\\n]+)>\\n" + }, + "arguments": { + "type": "object", + "x-regex": "^<function=[^>\\n]+>\\n(.*?)\\n</function>$", + "x-regex-key-value": "<parameter=(?P<key>[^>\\n]+)>\\n(?P<value>.*?)\\n</parameter>", + "additionalProperties": true + } + }, + "required": [ + "name", + "arguments" + ] + } + }, + "required": [ + "type", + "function" + ] + } + } + }, + "required": [ + "role" + ] + }, + "chat_template": [ + "{%- set image_count = namespace(value=0) %}\n", + "{%- set video_count = namespace(value=0) %}\n", + "{%- if enable_thinking is not defined and reasoning_effort is string %}\n", + " {%- set qwen_reasoning_effort = reasoning_effort|lower %}\n", + " {%- if qwen_reasoning_effort in ['none', 'minimal', 'low'] %}\n", + " {%- set enable_thinking = false %}\n", + " {%- elif qwen_reasoning_effort in ['medium', 'high'] %}\n", + " {%- set enable_thinking = true %}\n", + " {%- endif %}\n", + "{%- endif %}\n", + "{%- set forced_tool_name = none %}\n", + "{%- if tool_choice is mapping %}\n", + " {%- if tool_choice.function is defined and tool_choice.function.name is defined %}\n", + " {%- set forced_tool_name = tool_choice.function.name %}\n", + " {%- elif tool_choice.name is defined %}\n", + " {%- set forced_tool_name = tool_choice.name %}\n", + " {%- endif %}\n", + "{%- elif function_call is mapping and function_call.name is defined %}\n", + " {%- set forced_tool_name = function_call.name %}\n", + "{%- endif %}\n", + "{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n", + " {%- if content is string %}\n", + " {{- content }}\n", + " {%- elif content is iterable and content is not mapping %}\n", + " {%- for item in content %}\n", + " {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n", + " {%- if is_system_content %}\n", + " {{- raise_exception('System message cannot contain images.') }}\n", + " {%- endif %}\n", + " {%- if do_vision_count %}\n", + " {%- set image_count.value = image_count.value + 1 %}\n", + " {%- endif %}\n", + " {%- if add_vision_id %}\n", + " {{- 'Picture ' ~ image_count.value ~ ': ' }}\n", + " {%- endif %}\n", + " {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n", + " {%- elif 'video' in item or item.type == 'video' %}\n", + " {%- if is_system_content %}\n", + " {{- raise_exception('System message cannot contain videos.') }}\n", + " {%- endif %}\n", + " {%- if do_vision_count %}\n", + " {%- set video_count.value = video_count.value + 1 %}\n", + " {%- endif %}\n", + " {%- if add_vision_id %}\n", + " {{- 'Video ' ~ video_count.value ~ ': ' }}\n", + " {%- endif %}\n", + " {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n", + " {%- elif 'text' in item %}\n", + " {{- item.text }}\n", + " {%- else %}\n", + " {{- raise_exception('Unexpected item type in content.') }}\n", + " {%- endif %}\n", + " {%- endfor %}\n", + " {%- elif content is none or content is undefined %}\n", + " {{- '' }}\n", + " {%- else %}\n", + " {{- raise_exception('Unexpected content type.') }}\n", + " {%- endif %}\n", + "{%- endmacro %}\n", + "{%- if not messages %}\n", + " {{- raise_exception('No messages provided.') }}\n", + "{%- endif %}\n", + "{%- if tools and tools is iterable and tools is not mapping %}\n", + " {{- '<|im_start|>system\\n' }}\n", + " {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n", + " {%- for tool in tools %}\n", + " {{- \"\\n\" }}\n", + " {{- tool | tojson }}\n", + " {%- endfor %}\n", + " {{- \"\\n</tools>\" }}\n", + " {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n", + " {%- if messages[0].role == 'system' %}\n", + " {%- set content = render_content(messages[0].content, false, true)|trim %}\n", + " {%- if content %}\n", + " {{- '\\n\\n' + content }}\n", + " {%- endif %}\n", + " {%- endif %}\n", + " {{- '<|im_end|>\\n' }}\n", + "{%- else %}\n", + " {%- if messages[0].role == 'system' %}\n", + " {%- set content = render_content(messages[0].content, false, true)|trim %}\n", + " {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n", + " {%- endif %}\n", + "{%- endif %}\n", + "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n", + "{%- for message in messages[::-1] %}\n", + " {%- set index = (messages|length - 1) - loop.index0 %}\n", + " {%- if ns.multi_step_tool and message.role == \"user\" %}\n", + " {%- set content = render_content(message.content, false)|trim %}\n", + " {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n", + " {%- set ns.multi_step_tool = false %}\n", + " {%- set ns.last_query_index = index %}\n", + " {%- endif %}\n", + " {%- endif %}\n", + "{%- endfor %}\n", + "{%- if ns.multi_step_tool %}\n", + " {{- raise_exception('No user query found in messages.') }}\n", + "{%- endif %}\n", + "{%- for message in messages %}\n", + " {%- set content = render_content(message.content, true)|trim %}\n", + " {%- if message.role == \"system\" %}\n", + " {%- if not loop.first %}\n", + " {{- raise_exception('System message must be at the beginning.') }}\n", + " {%- endif %}\n", + " {%- elif message.role == \"user\" %}\n", + " {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n", + " {%- elif message.role == \"assistant\" %}\n", + " {%- set reasoning_content = '' %}\n", + " {%- if message.reasoning_content is string %}\n", + " {%- set reasoning_content = message.reasoning_content %}\n", + " {%- else %}\n", + " {%- if '</think>' in content %}\n", + " {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n", + " {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n", + " {%- endif %}\n", + " {%- endif %}\n", + " {%- set reasoning_content = reasoning_content|trim %}\n", + " {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %}\n", + " {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n", + " {%- else %}\n", + " {{- '<|im_start|>' + message.role + '\\n' + content }}\n", + " {%- endif %}\n", + " {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n", + " {%- for tool_call in message.tool_calls %}\n", + " {%- if tool_call.function is defined %}\n", + " {%- set tool_call = tool_call.function %}\n", + " {%- endif %}\n", + " {%- if loop.first %}\n", + " {%- if content|trim %}\n", + " {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n", + " {%- else %}\n", + " {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n", + " {%- endif %}\n", + " {%- else %}\n", + " {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n", + " {%- endif %}\n", + " {%- if tool_call.arguments is defined %}\n", + " {%- set arguments = tool_call.arguments | from_json if tool_call.arguments is string else tool_call.arguments %}\n", + " {%- for args_name, args_value in arguments|items %}\n", + " {{- '<parameter=' + args_name + '>\\n' }}\n", + " {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n", + " {{- args_value }}\n", + " {{- '\\n</parameter>\\n' }}\n", + " {%- endfor %}\n", + " {%- endif %}\n", + " {{- '</function>\\n</tool_call>' }}\n", + " {%- endfor %}\n", + " {%- endif %}\n", + " {{- '<|im_end|>\\n' }}\n", + " {%- elif message.role == \"tool\" %}\n", + " {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n", + " {{- '<|im_start|>user' }}\n", + " {%- endif %}\n", + " {{- '\\n<tool_response>\\n' }}\n", + " {{- content }}\n", + " {{- '\\n</tool_response>' }}\n", + " {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n", + " {{- '<|im_end|>\\n' }}\n", + " {%- elif loop.last %}\n", + " {{- '<|im_end|>\\n' }}\n", + " {%- endif %}\n", + " {%- else %}\n", + " {{- raise_exception('Unexpected message role.') }}\n", + " {%- endif %}\n", + "{%- endfor %}\n", + "{%- if add_generation_prompt %}\n", + " {{- '<|im_start|>assistant\\n' }}\n", + " {%- if forced_tool_name %}\n", + " {{- '<tool_call>\\n<function=' + forced_tool_name + '>\\n' }}\n", + " {%- elif enable_thinking is defined and enable_thinking is false %}\n", + " {{- '<think>\\n\\n</think>\\n\\n' }}\n", + " {%- else %}\n", + " {{- '<think>\\n' }}\n", + " {%- endif %}\n", + "{%- endif %}" + ] + } +} diff --git a/examples/server/configs/qwen3.6-35b-a3b.json b/examples/server/configs/qwen3.6-35b-a3b.json new file mode 100644 index 0000000000..10043f9c1a --- /dev/null +++ b/examples/server/configs/qwen3.6-35b-a3b.json @@ -0,0 +1,263 @@ +{ + "server": { + "host": "0.0.0.0", + "port": 8000 + }, + "model": { + "alias": "qwen3.6-35b-a3b", + "from_pretrained": { + "repo_id": "unsloth/Qwen3.6-35B-A3B-GGUF", + "filename": "Qwen3.6-35B-A3B-Q8_0.gguf" + }, + "mtmd": { + "mmproj_from_pretrained": { + "repo_id": "unsloth/Qwen3.6-35B-A3B-GGUF", + "filename": "mmproj-BF16.gguf" + } + }, + "n_ctx": 32768, + "max_output_tokens": 4096, + "n_seq_max": 8, + "n_batch": 512, + "n_ubatch": 512, + "threads": 8, + "threads_batch": 8, + "kv_unified": true, + "store_logits": false, + "use_mmap": true, + "use_mlock": false, + "n_gpu_layers": -1, + "flash_attn": true, + "response_schema": { + "type": "object", + "properties": { + "role": { + "const": "assistant" + }, + "reasoning_content": { + "type": "string", + "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:<think>\\n)?(.*?)(?=</think>)" + }, + "content": { + "type": "string", + "x-regex": "^(?:<\\|im_start\\|>assistant\\n)?(?:(?:<think>\\n)?.*?</think>\\s*)?(.*?)(?=\\s*<tool_call>\\n|<\\|im_end\\|>$|$)" + }, + "tool_calls": { + "type": "array", + "x-regex-iterator": "<tool_call>\\n(.*?)\\n</tool_call>", + "items": { + "type": "object", + "properties": { + "type": { + "const": "function" + }, + "function": { + "type": "object", + "properties": { + "name": { + "type": "string", + "x-regex": "^<function=([^>\\n]+)>\\n" + }, + "arguments": { + "type": "object", + "x-regex": "^<function=[^>\\n]+>\\n(.*?)\\n</function>$", + "x-regex-key-value": "<parameter=(?P<key>[^>\\n]+)>\\n(?P<value>.*?)\\n</parameter>", + "additionalProperties": true + } + }, + "required": [ + "name", + "arguments" + ] + } + }, + "required": [ + "type", + "function" + ] + } + } + }, + "required": [ + "role" + ] + }, + "chat_template": [ + "{%- set image_count = namespace(value=0) %}\n", + "{%- set video_count = namespace(value=0) %}\n", + "{%- if enable_thinking is not defined and reasoning_effort is string %}\n", + " {%- set qwen_reasoning_effort = reasoning_effort|lower %}\n", + " {%- if qwen_reasoning_effort in ['none', 'minimal', 'low'] %}\n", + " {%- set enable_thinking = false %}\n", + " {%- elif qwen_reasoning_effort in ['medium', 'high'] %}\n", + " {%- set enable_thinking = true %}\n", + " {%- endif %}\n", + "{%- endif %}\n", + "{%- set forced_tool_name = none %}\n", + "{%- if tool_choice is mapping %}\n", + " {%- if tool_choice.function is defined and tool_choice.function.name is defined %}\n", + " {%- set forced_tool_name = tool_choice.function.name %}\n", + " {%- elif tool_choice.name is defined %}\n", + " {%- set forced_tool_name = tool_choice.name %}\n", + " {%- endif %}\n", + "{%- elif function_call is mapping and function_call.name is defined %}\n", + " {%- set forced_tool_name = function_call.name %}\n", + "{%- endif %}\n", + "{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n", + " {%- if content is string %}\n", + " {{- content }}\n", + " {%- elif content is iterable and content is not mapping %}\n", + " {%- for item in content %}\n", + " {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n", + " {%- if is_system_content %}\n", + " {{- raise_exception('System message cannot contain images.') }}\n", + " {%- endif %}\n", + " {%- if do_vision_count %}\n", + " {%- set image_count.value = image_count.value + 1 %}\n", + " {%- endif %}\n", + " {%- if add_vision_id %}\n", + " {{- 'Picture ' ~ image_count.value ~ ': ' }}\n", + " {%- endif %}\n", + " {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n", + " {%- elif 'video' in item or item.type == 'video' %}\n", + " {%- if is_system_content %}\n", + " {{- raise_exception('System message cannot contain videos.') }}\n", + " {%- endif %}\n", + " {%- if do_vision_count %}\n", + " {%- set video_count.value = video_count.value + 1 %}\n", + " {%- endif %}\n", + " {%- if add_vision_id %}\n", + " {{- 'Video ' ~ video_count.value ~ ': ' }}\n", + " {%- endif %}\n", + " {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n", + " {%- elif 'text' in item %}\n", + " {{- item.text }}\n", + " {%- else %}\n", + " {{- raise_exception('Unexpected item type in content.') }}\n", + " {%- endif %}\n", + " {%- endfor %}\n", + " {%- elif content is none or content is undefined %}\n", + " {{- '' }}\n", + " {%- else %}\n", + " {{- raise_exception('Unexpected content type.') }}\n", + " {%- endif %}\n", + "{%- endmacro %}\n", + "{%- if not messages %}\n", + " {{- raise_exception('No messages provided.') }}\n", + "{%- endif %}\n", + "{%- if tools and tools is iterable and tools is not mapping %}\n", + " {{- '<|im_start|>system\\n' }}\n", + " {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n", + " {%- for tool in tools %}\n", + " {{- \"\\n\" }}\n", + " {{- tool | tojson }}\n", + " {%- endfor %}\n", + " {{- \"\\n</tools>\" }}\n", + " {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n", + " {%- if messages[0].role == 'system' %}\n", + " {%- set content = render_content(messages[0].content, false, true)|trim %}\n", + " {%- if content %}\n", + " {{- '\\n\\n' + content }}\n", + " {%- endif %}\n", + " {%- endif %}\n", + " {{- '<|im_end|>\\n' }}\n", + "{%- else %}\n", + " {%- if messages[0].role == 'system' %}\n", + " {%- set content = render_content(messages[0].content, false, true)|trim %}\n", + " {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n", + " {%- endif %}\n", + "{%- endif %}\n", + "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n", + "{%- for message in messages[::-1] %}\n", + " {%- set index = (messages|length - 1) - loop.index0 %}\n", + " {%- if ns.multi_step_tool and message.role == \"user\" %}\n", + " {%- set content = render_content(message.content, false)|trim %}\n", + " {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n", + " {%- set ns.multi_step_tool = false %}\n", + " {%- set ns.last_query_index = index %}\n", + " {%- endif %}\n", + " {%- endif %}\n", + "{%- endfor %}\n", + "{%- if ns.multi_step_tool %}\n", + " {{- raise_exception('No user query found in messages.') }}\n", + "{%- endif %}\n", + "{%- for message in messages %}\n", + " {%- set content = render_content(message.content, true)|trim %}\n", + " {%- if message.role == \"system\" %}\n", + " {%- if not loop.first %}\n", + " {{- raise_exception('System message must be at the beginning.') }}\n", + " {%- endif %}\n", + " {%- elif message.role == \"user\" %}\n", + " {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n", + " {%- elif message.role == \"assistant\" %}\n", + " {%- set reasoning_content = '' %}\n", + " {%- if message.reasoning_content is string %}\n", + " {%- set reasoning_content = message.reasoning_content %}\n", + " {%- else %}\n", + " {%- if '</think>' in content %}\n", + " {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n", + " {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n", + " {%- endif %}\n", + " {%- endif %}\n", + " {%- set reasoning_content = reasoning_content|trim %}\n", + " {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %}\n", + " {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n", + " {%- else %}\n", + " {{- '<|im_start|>' + message.role + '\\n' + content }}\n", + " {%- endif %}\n", + " {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n", + " {%- for tool_call in message.tool_calls %}\n", + " {%- if tool_call.function is defined %}\n", + " {%- set tool_call = tool_call.function %}\n", + " {%- endif %}\n", + " {%- if loop.first %}\n", + " {%- if content|trim %}\n", + " {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n", + " {%- else %}\n", + " {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n", + " {%- endif %}\n", + " {%- else %}\n", + " {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n", + " {%- endif %}\n", + " {%- if tool_call.arguments is defined %}\n", + " {%- set arguments = tool_call.arguments | from_json if tool_call.arguments is string else tool_call.arguments %}\n", + " {%- for args_name, args_value in arguments|items %}\n", + " {{- '<parameter=' + args_name + '>\\n' }}\n", + " {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n", + " {{- args_value }}\n", + " {{- '\\n</parameter>\\n' }}\n", + " {%- endfor %}\n", + " {%- endif %}\n", + " {{- '</function>\\n</tool_call>' }}\n", + " {%- endfor %}\n", + " {%- endif %}\n", + " {{- '<|im_end|>\\n' }}\n", + " {%- elif message.role == \"tool\" %}\n", + " {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n", + " {{- '<|im_start|>user' }}\n", + " {%- endif %}\n", + " {{- '\\n<tool_response>\\n' }}\n", + " {{- content }}\n", + " {{- '\\n</tool_response>' }}\n", + " {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n", + " {{- '<|im_end|>\\n' }}\n", + " {%- elif loop.last %}\n", + " {{- '<|im_end|>\\n' }}\n", + " {%- endif %}\n", + " {%- else %}\n", + " {{- raise_exception('Unexpected message role.') }}\n", + " {%- endif %}\n", + "{%- endfor %}\n", + "{%- if add_generation_prompt %}\n", + " {{- '<|im_start|>assistant\\n' }}\n", + " {%- if forced_tool_name %}\n", + " {{- '<tool_call>\\n<function=' + forced_tool_name + '>\\n' }}\n", + " {%- elif enable_thinking is defined and enable_thinking is false %}\n", + " {{- '<think>\\n\\n</think>\\n\\n' }}\n", + " {%- else %}\n", + " {{- '<think>\\n' }}\n", + " {%- endif %}\n", + "{%- endif %}" + ] + } +} diff --git a/examples/server/server.py b/examples/server/server.py new file mode 100644 index 0000000000..6a500ceec9 --- /dev/null +++ b/examples/server/server.py @@ -0,0 +1,15328 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.8" +# dependencies = [ +# "fastapi", +# "jinja2", +# "llama-cpp-python", +# "numpy", +# "openai", +# "pydantic", +# "safetensors", +# "uvicorn", +# "websockets", +# ] +# /// + +from __future__ import annotations + +import abc +import os +import re +import json +import math +import time +import uuid +import queue +import ctypes +import fnmatch +import base64 +import hashlib +import binascii +import asyncio +import argparse +import threading +import multiprocessing +import copy +import shutil +import inspect +import importlib.util +import sys +import urllib.error +import urllib.parse +import urllib.request + +from pathlib import Path +from datetime import datetime +from dataclasses import dataclass, field +from collections import OrderedDict, deque +from openai.types.completion import Completion as OpenAICompletion +from openai.types.completion_choice import ( + CompletionChoice, + Logprobs as CompletionLogprobs, +) +from openai.types.completion_usage import CompletionUsage +from openai.types.chat.chat_completion import ( + ChatCompletion, + Choice as ChatCompletionChoice, + ChoiceLogprobs as ChatCompletionChoiceLogprobs, +) +from openai.types.chat.chat_completion_chunk import ( + ChatCompletionChunk, + Choice as ChatCompletionChunkChoice, + ChoiceDelta, + ChoiceDeltaFunctionCall, + ChoiceDeltaToolCallFunction, + ChoiceDeltaToolCall, + ChoiceLogprobs as ChatCompletionChunkChoiceLogprobs, +) +from openai.types.chat.chat_completion_message import ( + ChatCompletionMessage, + FunctionCall as ChatCompletionMessageFunctionCall, +) +from openai.types.chat.chat_completion_message_tool_call import ( + ChatCompletionMessageToolCall, + Function as ChatCompletionMessageToolCallFunction, +) +from openai.types.chat.chat_completion_token_logprob import ( + ChatCompletionTokenLogprob, + TopLogprob, +) +from typing import ( + Any, + Callable, + Dict, + Generator, + Iterable, + List, + Optional, + Sequence, + Tuple, + Union, + Deque, + Literal, + Iterator, + Protocol, + TypedDict, + cast, + AsyncIterator, +) + +import jinja2 +import uvicorn +import numpy as np + +from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse, Response, StreamingResponse + +from jinja2.sandbox import ImmutableSandboxedEnvironment + +from pydantic_core import from_json +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from llama_cpp import llama_cpp # noqa: E402 +from llama_cpp import llama_cpp_ext # noqa: E402 +from llama_cpp import mtmd_cpp # noqa: E402 + + +JSON_GBNF = r""" +root ::= object +value ::= object | array | string | number | ("true" | "false" | "null") ws + +object ::= + "{" ws ( + string ":" ws value + ("," ws string ":" ws value)* + )? "}" ws + +array ::= + "[" ws ( + value + ("," ws value)* + )? "]" ws + +string ::= + "\"" ( + [^"\\\x7F\x00-\x1F] | + "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) + )* "\"" ws + +number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [0-9] [1-9]{0,15})? ws + +ws ::= | " " | "\n" [ \t]{0,20} +""" + + +class JsonSchemaConverter: + @dataclass(frozen=True) + class BuiltinRule: + content: str + deps: Sequence[str] = () + + SPACE_RULE = '" "?' + INVALID_RULE_CHARS_RE = re.compile(r"[^a-zA-Z0-9-]+") + GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"]') + GRAMMAR_LITERAL_ESCAPES = {"\r": "\\r", "\n": "\\n", '"': '\\"'} + DOTALL = "[\\U00000000-\\U0010FFFF]" + DOT = "[^\\x0A\\x0D]" + NON_LITERAL_SET = set("|.()[]{}*+?") + ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set("[]()|{}*+?") + + PRIMITIVE_RULES: Optional[Dict[str, "JsonSchemaConverter.BuiltinRule"]] = None + STRING_FORMAT_RULES: Optional[Dict[str, "JsonSchemaConverter.BuiltinRule"]] = None + RESERVED_NAMES: Optional[set[str]] = None + + @staticmethod + def _build_repetition( + item_rule: str, + min_items: int, + max_items: Optional[int], + separator_rule: Optional[str] = None, + item_rule_is_literal: bool = False, + ) -> str: + if not separator_rule: + if min_items == 0 and max_items == 1: + return f"{item_rule}?" + if min_items == 1 and max_items is None: + return f"{item_rule}+" + + result = "" + + if min_items > 0: + if item_rule_is_literal and separator_rule is None: + result = '"' + (item_rule[1:-1] * min_items) + '"' + else: + result = (f" {separator_rule} " if separator_rule else " ").join( + [item_rule] * min_items + ) + + def opt_repetitions(up_to_n: int, prefix_with_sep: bool = False) -> str: + content = ( + f"{separator_rule} {item_rule}" + if prefix_with_sep and separator_rule + else item_rule + ) + if up_to_n == 0: + return "" + if up_to_n == 1: + return f"({content})?" + if separator_rule and not prefix_with_sep: + return f"({content} {opt_repetitions(up_to_n - 1, prefix_with_sep=True)})?" + return (f"({content} " * up_to_n).rstrip() + (")?" * up_to_n) + + if min_items > 0 and max_items != min_items: + result += " " + + if max_items is not None: + result += opt_repetitions(max_items - min_items, prefix_with_sep=min_items > 0) + else: + item_operator = f"({separator_rule + ' ' if separator_rule else ''}{item_rule})" + if min_items == 0 and separator_rule: + result = f"({item_rule} {item_operator}*)?" + else: + result += f"{item_operator}*" + + return result + + @classmethod + def _primitive_rules(cls) -> Dict[str, "JsonSchemaConverter.BuiltinRule"]: + if cls.PRIMITIVE_RULES is None: + up_to_15_digits = cls._build_repetition("[0-9]", 0, 15) + cls.PRIMITIVE_RULES = { + "boolean": cls.BuiltinRule('("true" | "false") space', []), + "decimal-part": cls.BuiltinRule("[0-9] " + up_to_15_digits, []), + "integral-part": cls.BuiltinRule( + "[0-9] | [1-9] " + up_to_15_digits, + [], + ), + "number": cls.BuiltinRule( + '("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space', + ["integral-part", "decimal-part"], + ), + "integer": cls.BuiltinRule('("-"? integral-part) space', ["integral-part"]), + "value": cls.BuiltinRule( + "object | array | string | number | boolean | null", + ["object", "array", "string", "number", "boolean", "null"], + ), + "object": cls.BuiltinRule( + '"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', + ["string", "value"], + ), + "array": cls.BuiltinRule( + '"[" space ( value ("," space value)* )? "]" space', + ["value"], + ), + "uuid": cls.BuiltinRule( + r'"\"" ' + + ' "-" '.join("[0-9a-fA-F]" * n for n in [8, 4, 4, 4, 12]) + + r' "\"" space', + [], + ), + "char": cls.BuiltinRule( + r'[^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])', + [], + ), + "string": cls.BuiltinRule(r'"\"" char* "\"" space', ["char"]), + "null": cls.BuiltinRule('"null" space', []), + } + return cls.PRIMITIVE_RULES + + @classmethod + def _string_format_rules(cls) -> Dict[str, "JsonSchemaConverter.BuiltinRule"]: + if cls.STRING_FORMAT_RULES is None: + cls.STRING_FORMAT_RULES = { + "date": cls.BuiltinRule( + '[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [0-1] )', + [], + ), + "time": cls.BuiltinRule( + '([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', + [], + ), + "date-time": cls.BuiltinRule('date "T" time', ["date", "time"]), + "date-string": cls.BuiltinRule('"\\"" date "\\"" space', ["date"]), + "time-string": cls.BuiltinRule('"\\"" time "\\"" space', ["time"]), + "date-time-string": cls.BuiltinRule( + '"\\"" date-time "\\"" space', + ["date-time"], + ), + } + return cls.STRING_FORMAT_RULES + + @classmethod + def _reserved_names(cls) -> set[str]: + if cls.RESERVED_NAMES is None: + cls.RESERVED_NAMES = set( + ["root", "dot", *cls._primitive_rules().keys(), *cls._string_format_rules().keys()] + ) + return cls.RESERVED_NAMES + + def __init__( + self, + *, + prop_order: Dict[str, int], + allow_fetch: bool, + dotall: bool, + raw_pattern: bool, + ): + self._prop_order = prop_order + self._allow_fetch = allow_fetch + self._dotall = dotall + self._raw_pattern = raw_pattern + self._rules: Dict[str, str] = {"space": self.SPACE_RULE} + self._refs: Dict[str, Any] = {} + self._refs_being_resolved: set[str] = set() + + def _format_literal(self, literal: str) -> str: + escaped = self.GRAMMAR_LITERAL_ESCAPE_RE.sub( + lambda match: self.GRAMMAR_LITERAL_ESCAPES[match.group(0)], + literal, + ) + return f'"{escaped}"' + + def _add_rule(self, name: str, rule: str) -> str: + escaped_name = self.INVALID_RULE_CHARS_RE.sub("-", name) + if escaped_name not in self._rules or self._rules[escaped_name] == rule: + key = escaped_name + else: + suffix = 0 + while ( + f"{escaped_name}{suffix}" in self._rules + and self._rules[f"{escaped_name}{suffix}"] != rule + ): + suffix += 1 + key = f"{escaped_name}{suffix}" + self._rules[key] = rule + return key + + def resolve_refs(self, schema: Dict[str, Any], url: str) -> Dict[str, Any]: + def visit(node: Any) -> Any: + if isinstance(node, list): + return [visit(child) for child in node] + if isinstance(node, dict): + ref = node.get("$ref") + if ref is not None and ref not in self._refs: + if ref.startswith("https://"): + raise ValueError("remote schema fetch is not allowed") + elif ref.startswith("#/"): + target = schema + ref = f"{url}{ref}" + node["$ref"] = ref + else: + raise ValueError(f"Unsupported ref {ref}") + + for selector in ref.split("#")[-1].split("/")[1:]: + assert target is not None and selector in target, ( + f"Error resolving ref {ref}: {selector} not in {target}" + ) + target = target[selector] + self._refs[ref] = target + else: + for value in node.values(): + visit(value) + return node + + return cast(Dict[str, Any], visit(schema)) + + def _generate_union_rule(self, name: str, alt_schemas: List[Dict[str, Any]]) -> str: + return " | ".join( + self.visit(alt_schema, f"{name}{'-' if name else 'alternative-'}{index}") + for index, alt_schema in enumerate(alt_schemas) + ) + + def _visit_pattern(self, pattern: str, name: str) -> str: + assert pattern.startswith("^") and pattern.endswith("$"), ( + 'Pattern must start with "^" and end with "$"' + ) + pattern = pattern[1:-1] + sub_rule_ids: Dict[str, str] = {} + index = 0 + length = len(pattern) + + def to_rule(item: Tuple[str, bool]) -> str: + text, is_literal = item + return f'"{text}"' if is_literal else text + + def transform() -> Tuple[str, bool]: + nonlocal index + start = index + sequence: List[Tuple[str, bool]] = [] + + def get_dot() -> str: + rule = self.DOTALL if self._dotall else self.DOT + return self._add_rule("dot", rule) + + def join_sequence() -> Tuple[str, bool]: + if len(sequence) == 1: + return sequence[0] + return (" ".join(to_rule(item) for item in sequence), False) + + while index < length: + char = pattern[index] + if char == ".": + sequence.append((get_dot(), False)) + index += 1 + elif char == "(": + index += 1 + if index < length: + assert pattern[index] != "?", ( + f'Unsupported pattern syntax "{pattern[index]}" at index {index} of /{pattern}/' + ) + sequence.append((f"({to_rule(transform())})", False)) + elif char == ")": + index += 1 + assert start > 0 and pattern[start - 1] == "(", ( + f"Unbalanced parentheses; start = {start}, index = {index}, pattern = {pattern}" + ) + return join_sequence() + elif char == "[": + square_brackets = char + index += 1 + while index < length and pattern[index] != "]": + if pattern[index] == "\\": + square_brackets += pattern[index : index + 2] + index += 2 + else: + square_brackets += pattern[index] + index += 1 + assert index < length, ( + f"Unbalanced square brackets; start = {start}, index = {index}, pattern = {pattern}" + ) + square_brackets += "]" + index += 1 + sequence.append((square_brackets, False)) + elif char == "|": + sequence.append(("|", False)) + index += 1 + elif char in ("*", "+", "?"): + sequence[-1] = (to_rule(sequence[-1]) + char, False) + index += 1 + elif char == "{": + curly_brackets = char + index += 1 + while index < length and pattern[index] != "}": + curly_brackets += pattern[index] + index += 1 + assert index < length, ( + f"Unbalanced curly brackets; start = {start}, index = {index}, pattern = {pattern}" + ) + curly_brackets += "}" + index += 1 + numbers = [part.strip() for part in curly_brackets[1:-1].split(",")] + min_times = 0 + max_times: Optional[int] = None + try: + if len(numbers) == 1: + min_times = int(numbers[0]) + max_times = min_times + else: + assert len(numbers) == 2 + min_times = int(numbers[0]) if numbers[0] else 0 + max_times = int(numbers[1]) if numbers[1] else None + except ValueError as exc: + raise ValueError( + f"Invalid quantifier {curly_brackets} in /{pattern}/" + ) from exc + + sub, sub_is_literal = sequence[-1] + if not sub_is_literal: + rule_id = sub_rule_ids.get(sub) + if rule_id is None: + rule_id = self._add_rule(f"{name}-{len(sub_rule_ids) + 1}", sub) + sub_rule_ids[sub] = rule_id + sub = rule_id + + sequence[-1] = ( + self._build_repetition( + f'"{sub}"' if sub_is_literal else sub, + min_times, + max_times, + item_rule_is_literal=sub_is_literal, + ), + False, + ) + else: + literal = "" + while index < length: + if pattern[index] == "\\" and index < length - 1: + next_char = pattern[index + 1] + if next_char in self.ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS: + index += 1 + literal += pattern[index] + index += 1 + else: + literal += pattern[index : index + 2] + index += 2 + elif pattern[index] == '"' and not self._raw_pattern: + literal += '\\"' + index += 1 + elif pattern[index] not in self.NON_LITERAL_SET and ( + index == length - 1 + or literal == "" + or pattern[index + 1] == "." + or pattern[index + 1] not in self.NON_LITERAL_SET + ): + literal += pattern[index] + index += 1 + else: + break + if literal: + sequence.append((literal, True)) + + return join_sequence() + + return self._add_rule( + name, + ( + to_rule(transform()) + if self._raw_pattern + else '"\\"" ' + to_rule(transform()) + ' "\\"" space' + ), + ) + + def _resolve_ref(self, ref: str) -> str: + ref_name = ref.split("/")[-1] + if ref_name not in self._rules and ref not in self._refs_being_resolved: + self._refs_being_resolved.add(ref) + resolved = self._refs[ref] + ref_name = self.visit(resolved, ref_name) + self._refs_being_resolved.remove(ref) + return ref_name + + def _generate_constant_rule(self, value: Any) -> str: + return self._format_literal(json.dumps(value)) + + def visit(self, schema: Dict[str, Any], name: str) -> str: + schema_type = schema.get("type") + schema_format = schema.get("format") + rule_name = name + "-" if name in self._reserved_names() else name or "root" + + ref = schema.get("$ref") + if ref is not None: + return self._add_rule(rule_name, self._resolve_ref(ref)) + + if "oneOf" in schema or "anyOf" in schema: + return self._add_rule( + rule_name, + self._generate_union_rule(name, cast(List[Dict[str, Any]], schema.get("oneOf") or schema["anyOf"])), + ) + + if isinstance(schema_type, list): + return self._add_rule( + rule_name, + self._generate_union_rule(name, [{"type": entry} for entry in schema_type]), + ) + + if "const" in schema: + return self._add_rule(rule_name, self._generate_constant_rule(schema["const"])) + + if "enum" in schema: + rule = " | ".join(self._generate_constant_rule(value) for value in schema["enum"]) + return self._add_rule(rule_name, rule) + + if schema_type in (None, "object") and ( + "properties" in schema + or ("additionalProperties" in schema and schema["additionalProperties"] is not True) + ): + required_props = set(schema.get("required", [])) + property_items = list(cast(Dict[str, Any], schema.get("properties", {})).items()) + return self._add_rule( + rule_name, + self._build_object_rule( + property_items, required_props, name, schema.get("additionalProperties") + ), + ) + + if schema_type in (None, "object") and "allOf" in schema: + allof_required_props: set[str] = set() + allof_property_items: List[Tuple[str, Any]] = [] + + def add_component(component_schema: Dict[str, Any], is_required: bool) -> None: + component_ref = component_schema.get("$ref") + if component_ref is not None: + component_schema = cast(Dict[str, Any], self._refs[component_ref]) + if "properties" in component_schema: + for prop_name, prop_schema in cast(Dict[str, Any], component_schema["properties"]).items(): + allof_property_items.append((prop_name, prop_schema)) + if is_required: + allof_required_props.add(prop_name) + + for entry in cast(List[Dict[str, Any]], schema["allOf"]): + if "anyOf" in entry: + for alt in cast(List[Dict[str, Any]], entry["anyOf"]): + add_component(alt, is_required=False) + else: + add_component(entry, is_required=True) + + return self._add_rule( + rule_name, + self._build_object_rule( + allof_property_items, + allof_required_props, + name, + additional_properties=[], + ), + ) + + if schema_type in (None, "array") and ("items" in schema or "prefixItems" in schema): + items = schema.get("items") or schema["prefixItems"] + if isinstance(items, list): + return self._add_rule( + rule_name, + '"[" space ' + + ' "," space '.join( + self.visit(item, f"{name}{'-' if name else ''}tuple-{index}") + for index, item in enumerate(items) + ) + + ' "]" space', + ) + item_rule_name = self.visit(cast(Dict[str, Any], items), f"{name}{'-' if name else ''}item") + min_items = int(schema.get("minItems", 0)) + max_items = cast(Optional[int], schema.get("maxItems")) + return self._add_rule( + rule_name, + '"[" space ' + + self._build_repetition( + item_rule_name, + min_items, + max_items, + separator_rule='"," space', + ) + + ' "]" space', + ) + + if schema_type in (None, "string") and "pattern" in schema: + return self._visit_pattern(cast(str, schema["pattern"]), rule_name) + + if schema_type in (None, "string") and re.match(r"^uuid[1-5]?$", schema_format or ""): + return self._add_primitive( + "root" if rule_name == "root" else cast(str, schema_format), + self._primitive_rules()["uuid"], + ) + + if schema_type in (None, "string") and f"{schema_format}-string" in self._string_format_rules(): + primitive_name = f"{schema_format}-string" + return self._add_rule( + rule_name, + self._add_primitive(primitive_name, self._string_format_rules()[primitive_name]), + ) + + if schema_type == "string" and ("minLength" in schema or "maxLength" in schema): + char_rule = self._add_primitive("char", self._primitive_rules()["char"]) + min_len = int(schema.get("minLength", 0)) + max_len = cast(Optional[int], schema.get("maxLength")) + return self._add_rule( + rule_name, + r'"\"" ' + + self._build_repetition(char_rule, min_len, max_len) + + r' "\"" space', + ) + + if schema_type == "object" or len(schema) == 0: + return self._add_rule( + rule_name, + self._add_primitive("object", self._primitive_rules()["object"]), + ) + + primitive_rules = self._primitive_rules() + assert schema_type in primitive_rules, f"Unrecognized schema: {schema}" + return self._add_primitive( + "root" if rule_name == "root" else cast(str, schema_type), + primitive_rules[cast(str, schema_type)], + ) + + def _add_primitive(self, name: str, rule: "JsonSchemaConverter.BuiltinRule") -> str: + rule_name = self._add_rule(name, rule.content) + primitive_rules = self._primitive_rules() + string_format_rules = self._string_format_rules() + for dependency in rule.deps: + dependency_rule = primitive_rules.get(dependency) or string_format_rules.get( + dependency + ) + assert dependency_rule is not None, f"Rule {dependency} not known" + if dependency not in self._rules: + self._add_primitive(dependency, dependency_rule) + return rule_name + + def _build_object_rule( + self, + properties: List[Tuple[str, Any]], + required: set[str], + name: str, + additional_properties: Union[bool, Any], + ) -> str: + prop_order = self._prop_order + sorted_props = [ + key + for _, (key, _) in sorted( + enumerate(properties), + key=lambda indexed_key: ( + prop_order.get(indexed_key[1][0], len(prop_order)), + indexed_key[0], + ), + ) + ] + + property_kv_rule_names: Dict[str, str] = {} + for prop_name, prop_schema in properties: + prop_rule_name = self.visit( + cast(Dict[str, Any], prop_schema), + f"{name}{'-' if name else ''}{prop_name}", + ) + property_kv_rule_names[prop_name] = self._add_rule( + f"{name}{'-' if name else ''}{prop_name}-kv", + rf'{self._format_literal(json.dumps(prop_name))} space ":" space {prop_rule_name}', + ) + + required_props = [key for key in sorted_props if key in required] + optional_props = [key for key in sorted_props if key not in required] + + if additional_properties is True or isinstance(additional_properties, dict): + sub_name = f"{name}{'-' if name else ''}additional" + value_rule = self.visit( + {} if additional_properties is True else cast(Dict[str, Any], additional_properties), + f"{sub_name}-value", + ) + property_kv_rule_names["*"] = self._add_rule( + f"{sub_name}-kv", + self._add_primitive("string", self._primitive_rules()["string"]) + + f' ":" space {value_rule}', + ) + optional_props.append("*") + + rule = '"{" space ' + rule += ' "," space '.join(property_kv_rule_names[key] for key in required_props) + + if optional_props: + if required_props: + rule += ' ( "," space ( ' + else: + rule += "( " + + def get_recursive_refs(keys: List[str], first_is_optional: bool) -> str: + head, *rest = keys + kv_rule_name = property_kv_rule_names[head] + result = "" + if head == "*": + if first_is_optional: + result = f"({kv_rule_name})?" + else: + result = kv_rule_name + elif first_is_optional: + result = f'( "," space {kv_rule_name} )?' + else: + result = kv_rule_name + if rest: + result += " " + self._add_rule( + f"{name}{'-' if name else ''}{head}-rest", + get_recursive_refs(rest, first_is_optional=True), + ) + return result + + rule += " | ".join( + get_recursive_refs(optional_props[index:], first_is_optional=False) + for index in range(len(optional_props)) + ) + if required_props: + rule += " )" + rule += " )?" + + rule += ' "}" space' + return rule + + def format_grammar(self) -> str: + return "\n".join( + f"{name} ::= {rule}" + for name, rule in sorted(self._rules.items(), key=lambda item: item[0]) + ) + + @classmethod + def to_gbnf(cls, schema: str, prop_order: Optional[List[str]] = None) -> str: + property_order = prop_order or [] + loaded_schema = json.loads(schema) + order_index = {name: index for index, name in enumerate(property_order)} + converter = cls( + prop_order=order_index, + allow_fetch=False, + dotall=False, + raw_pattern=False, + ) + resolved_schema = converter.resolve_refs(loaded_schema, "stdin") + converter.visit(resolved_schema, "") + return converter.format_grammar() + + +class RadixTrie: + __slots__ = ("root", "sequences", "sequence_lengths") + + @dataclass + class Node: + label: Tuple[int, ...] = () + parent: Optional["RadixTrie.Node"] = None + children: Dict[int, "RadixTrie.Node"] = field(default_factory=dict) + sequences: set[int] = field(default_factory=set) + tails: set[int] = field(default_factory=set) + + def __init__(self) -> None: + self.root = RadixTrie.Node() + self.sequences: Dict[int, RadixTrie.Node] = {} + self.sequence_lengths: Dict[int, int] = {} + + @staticmethod + def _pick_sequence(candidates: set[int], preferred_sequences: Optional[Any]) -> int: + if preferred_sequences is None: + return next(iter(candidates)) + if isinstance(preferred_sequences, OrderedDict): + for sequence_id in reversed(preferred_sequences): + if sequence_id in candidates: + return sequence_id + return next(iter(candidates)) + if isinstance(preferred_sequences, (list, tuple)): + for sequence_id in reversed(preferred_sequences): + if sequence_id in candidates: + return sequence_id + return next(iter(candidates)) + preferred = candidates & preferred_sequences + if preferred: + return next(iter(preferred)) + return next(iter(candidates)) + + @staticmethod + def _common_prefix_len( + label: Sequence[int], + tokens: Sequence[int], + offset: int, + ) -> int: + limit = min(len(label), len(tokens) - offset) + match_len = 0 + while match_len < limit and label[match_len] == tokens[offset + match_len]: + match_len += 1 + return match_len + + def _split_child( + self, + parent: "RadixTrie.Node", + child: "RadixTrie.Node", + prefix_len: int, + ) -> "RadixTrie.Node": + assert 0 < prefix_len < len(child.label) + prefix = child.label[:prefix_len] + suffix = child.label[prefix_len:] + middle = RadixTrie.Node( + label=prefix, + parent=parent, + sequences=set(child.sequences), + ) + parent.children[prefix[0]] = middle + child.label = suffix + child.parent = middle + middle.children[suffix[0]] = child + return middle + + def _locate_prefix_node( + self, + sequence_id: int, + keep_len: int, + ) -> "RadixTrie.Node": + total_len = self.sequence_lengths[sequence_id] + assert 0 <= keep_len <= total_len + if keep_len == 0: + return self.root + node = self.sequences[sequence_id] + drop_len = total_len - keep_len + while node is not self.root: + label_len = len(node.label) + if drop_len == 0: + return node + if drop_len < label_len: + parent = node.parent + assert parent is not None + return self._split_child(parent, node, label_len - drop_len) + drop_len -= label_len + parent = node.parent + assert parent is not None + node = parent + return self.root + + def extend( + self, + sequence_id: int, + tokens: Sequence[int], + ) -> None: + assert sequence_id >= 0 + node = self.sequences.get(sequence_id, self.root) + if tokens: + node.tails.discard(sequence_id) + length = self.sequence_lengths.get(sequence_id, 0) + index = 0 + while index < len(tokens): + token = tokens[index] + child = node.children.get(token) + if child is None: + child = RadixTrie.Node( + label=tuple(tokens[index:]), + parent=node, + sequences={sequence_id}, + ) + node.children[token] = child + node = child + length += len(tokens) - index + index = len(tokens) + break + match_len = self._common_prefix_len(child.label, tokens, index) + if match_len == len(child.label): + child.sequences.add(sequence_id) + node = child + length += match_len + index += match_len + continue + node = self._split_child(node, child, match_len) + node.sequences.add(sequence_id) + length += match_len + index += match_len + if index == len(tokens): + break + suffix = RadixTrie.Node( + label=tuple(tokens[index:]), + parent=node, + sequences={sequence_id}, + ) + node.children[suffix.label[0]] = suffix + node = suffix + length += len(tokens) - index + index = len(tokens) + break + if node is self.root: + self.sequences.pop(sequence_id, None) + self.sequence_lengths.pop(sequence_id, None) + self.root.tails.discard(sequence_id) + else: + self.sequences[sequence_id] = node + self.sequence_lengths[sequence_id] = length + node.tails.add(sequence_id) + + def length(self, sequence_id: int) -> int: + return self.sequence_lengths.get(sequence_id, 0) + + def truncate(self, sequence_id: int, keep_len: int) -> None: + assert sequence_id >= 0 + assert sequence_id in self.sequence_lengths + assert 0 <= keep_len <= self.sequence_lengths[sequence_id] + current_len = self.sequence_lengths[sequence_id] + if keep_len == current_len: + return + boundary = self._locate_prefix_node(sequence_id, keep_len) + node = self.sequences.get(sequence_id, self.root) + if node is not self.root: + node.tails.discard(sequence_id) + while node is not boundary: + node.sequences.remove(sequence_id) + parent = node.parent + assert parent is not None + if not node.sequences: + del parent.children[node.label[0]] + node = parent + if boundary is self.root: + self.sequences.pop(sequence_id, None) + self.sequence_lengths.pop(sequence_id, None) + else: + self.sequences[sequence_id] = boundary + self.sequence_lengths[sequence_id] = keep_len + boundary.tails.add(sequence_id) + + def copy(self, source_sequence_id: int, dest_sequence_id: int, keep_len: int) -> None: + assert source_sequence_id >= 0 + assert dest_sequence_id >= 0 + assert source_sequence_id in self.sequence_lengths + assert dest_sequence_id not in self.sequence_lengths + assert 0 <= keep_len <= self.sequence_lengths[source_sequence_id] + if keep_len == 0: + self.sequences[dest_sequence_id] = self.root + self.sequence_lengths[dest_sequence_id] = 0 + self.root.tails.add(dest_sequence_id) + return + node = self._locate_prefix_node(source_sequence_id, keep_len) + self.sequences[dest_sequence_id] = node + self.sequence_lengths[dest_sequence_id] = keep_len + node.tails.add(dest_sequence_id) + while node is not self.root: + node.sequences.add(dest_sequence_id) + parent = node.parent + assert parent is not None + node = parent + + def tokens(self, sequence_id: int, keep_len: Optional[int] = None) -> List[int]: + length = self.sequence_lengths[sequence_id] + target_len = length if keep_len is None else keep_len + assert 0 <= target_len <= length + node = self.sequences[sequence_id] + labels: List[Tuple[int, ...]] = [] + while node is not self.root: + labels.append(node.label) + parent = node.parent + assert parent is not None + node = parent + values: List[int] = [] + for label in reversed(labels): + values.extend(label) + return values[:target_len] + + def longest_prefix( + self, + tokens: Sequence[int], + preferred_sequences: Optional[Any] = None, + *, + exact_only: bool = False, + ) -> Tuple[int, int]: + node = self.root + longest_sequence_id = -1 + longest_length = 0 + index = 0 + while index < len(tokens): + child = node.children.get(tokens[index]) + if child is None: + break + match_len = self._common_prefix_len(child.label, tokens, index) + if match_len < len(child.label): + break + node = child + index += match_len + candidates = node.tails if exact_only else node.sequences + if candidates: + longest_sequence_id = self._pick_sequence(candidates, preferred_sequences) + longest_length = index + return longest_sequence_id, longest_length + + +class SequenceHistory: + __slots__ = ("_position_lengths", "_root", "_tails", "size") + + @dataclass + class Node: + token: Optional[int] = None + parent: Optional["SequenceHistory.Node"] = None + children: Dict[int, "SequenceHistory.Node"] = field(default_factory=dict) + sequences: set[int] = field(default_factory=set) + position_increment: int = 1 + + def __init__(self) -> None: + self._root = SequenceHistory.Node() + self._tails: Dict[int, SequenceHistory.Node] = {} + self._position_lengths: Dict[int, int] = {} + self.size = 0 + + def extend( + self, + sequence_id: int, + tokens: Sequence[int], + position_increments: Optional[Sequence[int]] = None, + ) -> None: + assert sequence_id >= 0 + if position_increments is None: + position_increments = [1] * len(tokens) + assert len(position_increments) == len(tokens) + node = self._tails.get(sequence_id, self._root) + position_length = self._position_lengths.get(sequence_id, 0) + for token, position_increment in zip(tokens, position_increments): + position_increment = max(0, int(position_increment)) + child = node.children.get(sequence_id) + if child is None: + child = SequenceHistory.Node( + token=token, + parent=node, + position_increment=position_increment, + ) + node.children[sequence_id] = child + self.size += 1 + else: + assert child.parent is node + assert child.token == token + assert child.position_increment == position_increment + child.sequences.add(sequence_id) + position_length += position_increment + node = child + if node is self._root: + self._tails.pop(sequence_id, None) + self._position_lengths.pop(sequence_id, None) + else: + self._tails[sequence_id] = node + self._position_lengths[sequence_id] = position_length + + def position_length(self, sequence_id: int) -> int: + return self._position_lengths.get(sequence_id, 0) + + def position_length_for_prefix(self, sequence_id: int, keep_len: int) -> int: + if keep_len <= 0 or sequence_id not in self._tails: + return 0 + node = self._tails[sequence_id] + increments: List[int] = [] + while node is not self._root: + increments.append(node.position_increment) + parent = node.parent + assert parent is not None + node = parent + increments.reverse() + assert keep_len <= len(increments) + return sum(increments[:keep_len]) + + def copy( + self, + source_sequence_id: int, + dest_sequence_id: int, + source_length: int, + keep_len: int, + ) -> None: + assert source_sequence_id >= 0 + assert dest_sequence_id >= 0 + assert source_sequence_id in self._tails + assert dest_sequence_id not in self._tails + assert 0 <= keep_len <= source_length + node = self._tails[source_sequence_id] + path: List[SequenceHistory.Node] = [] + for _ in range(source_length - keep_len): + parent = node.parent + assert parent is not None + node = parent + while node is not self._root: + path.append(node) + parent = node.parent + assert parent is not None + node = parent + parent = self._root + position_length = 0 + for child in reversed(path): + parent.children[dest_sequence_id] = child + child.sequences.add(dest_sequence_id) + position_length += child.position_increment + parent = child + if keep_len == 0: + self._tails.pop(dest_sequence_id, None) + self._position_lengths.pop(dest_sequence_id, None) + else: + self._tails[dest_sequence_id] = path[0] + self._position_lengths[dest_sequence_id] = position_length + + def truncate( + self, + sequence_id: int, + current_length: int, + keep_len: int, + ) -> None: + assert sequence_id >= 0 + assert sequence_id in self._tails + assert 0 <= keep_len <= current_length + node = self._tails[sequence_id] + drop = current_length - keep_len + position_length = self._position_lengths.get(sequence_id, 0) + while node is not self._root and drop > 0: + node.sequences.remove(sequence_id) + parent = node.parent + assert parent is not None + child = parent.children.get(sequence_id) + if child is node: + del parent.children[sequence_id] + if not node.sequences: + self.size -= 1 + position_length -= node.position_increment + node = parent + drop -= 1 + if node is self._root: + self._tails.pop(sequence_id, None) + self._position_lengths.pop(sequence_id, None) + else: + self._tails[sequence_id] = node + self._position_lengths[sequence_id] = max(0, position_length) + + +class DraftProvider(abc.ABC): + @abc.abstractmethod + def draft( + self, + input_ids: np.ndarray, + /, + *, + seq_id: int, + max_tokens: Optional[int], + ) -> np.ndarray: + raise NotImplementedError() + + def can_draft(self, input_length: int, /, *, seq_id: int) -> bool: + return True + + def process(self, batch: Any, /) -> None: + pass + + def accept(self, seq_id: int, accepted_draft_tokens: int) -> None: + pass + + def truncate(self, seq_id: int, keep_len: int) -> None: + pass + + def copy_sequence( + self, + source_seq_id: int, + dest_seq_id: int, + p0: int, + p1: int, + ) -> None: + pass + + def set_target_processing_enabled(self, enabled: bool) -> None: + pass + + def close(self) -> None: + pass + + +class PromptLookupDecoding(DraftProvider): + def __init__(self, max_ngram_size: int = 2, num_pred_tokens: int = 10) -> None: + self._max_ngram_size = max_ngram_size + self._num_pred_tokens = num_pred_tokens + + def draft( + self, + input_ids: np.ndarray, + /, + *, + seq_id: int, + max_tokens: Optional[int], + ) -> np.ndarray: + input_length = input_ids.shape[0] + if input_length < 2: + return np.array([], dtype=np.intc) + num_pred_tokens = self._num_pred_tokens + if max_tokens is not None: + num_pred_tokens = min(num_pred_tokens, max_tokens) + if num_pred_tokens <= 0: + return np.array([], dtype=np.intc) + max_ngram_size = min(self._max_ngram_size, input_length - 1) + for ngram_size in range(max_ngram_size, 0, -1): + windows = np.lib.stride_tricks.sliding_window_view(input_ids, (ngram_size,)) + ngram = input_ids[-ngram_size:] + matches = np.all(windows == ngram, axis=1) + match_indices = np.nonzero(matches)[0] + for index in match_indices: + start = index + ngram_size + if start >= input_length: + continue + end = min(start + num_pred_tokens, input_length) + if start < end: + return input_ids[start:end].astype(np.intc, copy=False) + return np.array([], dtype=np.intc) + + +class MTPDraftProvider(DraftProvider): + batched_draft = True + sampled_batch_draft = True + + @dataclass + class DraftManyState: + result_index: int + seq_id: int + first_pos: int + keep_len: int + n_predict: int + token: int + drafted: List[int] + embedding: np.ndarray + cache_key: Tuple[Tuple[int, ...], int] + + def __init__( + self, + *, + model: "Model", + context_params: Any, + num_pred_tokens: int, + top_k: int, + p_min: float, + ) -> None: + self.target_ctx = model.ctx + self.model = model.llama_model + self.n_seq_max = model.n_seq_max + self.n_vocab = model.n_vocab + self.n_embd = int(llama_cpp.llama_model_n_embd(self.model)) + self.num_pred_tokens = max(0, int(num_pred_tokens)) + self.top_k = max(1, int(top_k)) + self.p_min = max(0.0, min(1.0, float(p_min))) + ( + self.target_hidden_norm_weight, + self.draft_hidden_norm_weight, + self.hidden_norm_epsilon, + ) = self._load_hidden_norm_weights(model.model_path) + self.ctx = llama_cpp.llama_init_from_model(self.model, context_params) + if self.ctx is None: + raise RuntimeError("failed to create MTP draft context") + self.n_batch = int(llama_cpp.llama_n_batch(self.ctx)) + mem = llama_cpp.llama_get_memory(self.ctx) + if mem is None: + llama_cpp.llama_free(self.ctx) + raise RuntimeError("failed to access MTP draft memory") + self.mem = mem + self.batch = llama_cpp.llama_batch_init(self.n_batch, self.n_embd, 1) + self._batch_tokens = (llama_cpp.llama_token * self.n_batch)() + self.batch.token = self._batch_tokens + self.batch_embeddings = np.ctypeslib.as_array( + self.batch.embd, + shape=(self.n_batch * self.n_embd,), + ) + self._samplers: List[Any] = [] + self.pending_h = np.zeros((self.n_seq_max, self.n_embd), dtype=np.float32) + self.verify_h: List[np.ndarray] = [ + np.empty((0, self.n_embd), dtype=np.float32) + for _ in range(self.n_seq_max) + ] + self.verify_h_pos: List[List[int]] = [[] for _ in range(self.n_seq_max)] + self.verify_h_rows = [0] * self.n_seq_max + self.ready = [False] * self.n_seq_max + self.ready_pos = [0] * self.n_seq_max + self.context_pos = [0] * self.n_seq_max + self.decode_seconds_total = 0.0 + self.decode_calls_total = 0 + self.decode_tokens_total = 0 + self.decode_failures_total = 0 + self.target_processing_enabled = False + self.set_target_processing_enabled(True) + llama_cpp_ext.llama_set_embeddings_pre_norm( + self.ctx, + True, + True, + ) + self._init_samplers() + + @staticmethod + def _load_gguf_reader() -> Any: + try: + from gguf import GGUFReader # type: ignore[import-not-found] + return GGUFReader + except ImportError: + pass + + gguf_init = ( + Path(__file__).resolve().parents[2] + / "vendor/llama.cpp/gguf-py/gguf/__init__.py" + ) + spec = importlib.util.spec_from_file_location( + "gguf", + gguf_init, + submodule_search_locations=[str(gguf_init.parent)], + ) + if spec is None or spec.loader is None: + raise RuntimeError("MTP requires the gguf Python reader from llama.cpp") + module = importlib.util.module_from_spec(spec) + previous = sys.modules.get("gguf") + # Package-relative imports in gguf-py expect the package to be registered. + sys.modules["gguf"] = module + try: + spec.loader.exec_module(module) + except Exception: + if previous is None: + sys.modules.pop("gguf", None) + else: + sys.modules["gguf"] = previous + raise + return module.GGUFReader + + @staticmethod + def _gguf_field_contents(reader: Any, name: str) -> Any: + field = reader.fields.get(name) + if field is None: + return None + return field.contents() + + def _load_hidden_norm_weights( + self, + model_path: str, + ) -> Tuple[np.ndarray, np.ndarray, float]: + GGUFReader = self._load_gguf_reader() + + target_weight: Optional[np.ndarray] = None + draft_weights: List[np.ndarray] = [] + reader = GGUFReader(model_path) + arch = self._gguf_field_contents(reader, "general.architecture") + if arch not in {"qwen35", "qwen35moe"}: + raise RuntimeError( + "draft-mtp currently supports qwen35/qwen35moe GGUF models, " + f"got {arch!r}" + ) + nextn_layers = self._gguf_field_contents( + reader, + f"{arch}.nextn_predict_layers", + ) + # The current MTP path follows llama.cpp's Qwen3.5 one-nextn-layer graph. + if int(nextn_layers or 0) != 1: + raise RuntimeError( + "draft-mtp currently supports exactly one Qwen3.5 nextn prediction layer" + ) + epsilon = self._gguf_field_contents( + reader, + f"{arch}.attention.layer_norm_rms_epsilon", + ) + if epsilon is None: + raise RuntimeError( + f"MTP requires {arch}.attention.layer_norm_rms_epsilon" + ) + for tensor in reader.tensors: + if tensor.name == "output_norm.weight": + target_weight = np.asarray(tensor.data, dtype=np.float32).copy() + elif tensor.name.endswith(".nextn.shared_head_norm.weight"): + draft_weights.append(np.asarray(tensor.data, dtype=np.float32).copy()) + if target_weight is None: + raise RuntimeError("MTP requires output_norm.weight in GGUF model") + if len(draft_weights) > 1: + raise RuntimeError( + "MTP requires at most one blk.*.nextn.shared_head_norm.weight in GGUF model" + ) + draft_weight = draft_weights[0] if draft_weights else target_weight + if target_weight.shape != (self.n_embd,): + raise RuntimeError( + "MTP target norm weight shape does not match model embedding size " + f"({target_weight.shape} != ({self.n_embd},))" + ) + if draft_weight.shape != (self.n_embd,): + raise RuntimeError( + "MTP draft norm weight shape does not match model embedding size " + f"({draft_weight.shape} != ({self.n_embd},))" + ) + return target_weight, draft_weight, float(epsilon) + + def _normalize_hidden_rows(self, rows: np.ndarray, weight: np.ndarray) -> np.ndarray: + rows = np.asarray(rows, dtype=np.float32) + scale = np.reciprocal( + np.sqrt( + np.mean(np.square(rows), axis=-1, keepdims=True) + + self.hidden_norm_epsilon + ) + ) + return rows * scale * weight.reshape(1, -1) + + def _normalize_target_hidden_rows(self, rows: np.ndarray) -> np.ndarray: + return self._normalize_hidden_rows(rows, self.target_hidden_norm_weight) + + def _normalize_draft_hidden_row( + self, + row: Union[np.ndarray, ctypes.POINTER(ctypes.c_float)], + ) -> np.ndarray: + if isinstance(row, np.ndarray): + row_array = row.reshape(1, -1) + else: + row_array = np.ctypeslib.as_array(row, shape=(self.n_embd,)).reshape(1, -1) + return self._normalize_hidden_rows( + row_array, + self.draft_hidden_norm_weight, + )[0] + + def _init_samplers(self) -> None: + for seq_id in range(self.n_seq_max): + params = llama_cpp.llama_sampler_chain_default_params() + params.no_perf = True + sampler = llama_cpp.llama_sampler_chain_init(params) + if self.top_k > 1: + llama_cpp.llama_sampler_chain_add( + sampler, + llama_cpp.llama_sampler_init_top_k(self.top_k), + ) + llama_cpp.llama_sampler_chain_add( + sampler, + llama_cpp.llama_sampler_init_greedy(), + ) + self._samplers.append(sampler) + + def _passes_p_min(self, output_index: int) -> bool: + if self.p_min <= 0.0: + return True + logits_ptr = llama_cpp.llama_get_logits_ith(self.ctx, output_index) + if not logits_ptr: + return False + logits = np.ctypeslib.as_array(logits_ptr, shape=(self.n_vocab,)) + n_values = min(self.top_k, self.n_vocab) + if n_values <= 0: + return False + if n_values == self.n_vocab: + values = logits + else: + top_indices = np.argpartition(logits, -n_values)[-n_values:] + values = logits[top_indices] + max_logit = float(np.max(values)) + weights = np.exp(values.astype(np.float64, copy=False) - max_logit) + total = float(np.sum(weights)) + if total <= 0.0 or not math.isfinite(total): + return False + return 1.0 / total >= self.p_min + + def _sample_token(self, output_index: int = 0, *, seq_id: int = 0) -> Optional[int]: + if seq_id < 0 or seq_id >= len(self._samplers): + return None + if not self._passes_p_min(output_index): + return None + sampler = self._samplers[seq_id] + token = int(llama_cpp.llama_sampler_sample(sampler, self.ctx, output_index)) + if token == llama_cpp.LLAMA_TOKEN_NULL: + return None + return token + + def _reset_sampler(self, seq_id: int) -> None: + if 0 <= seq_id < len(self._samplers): + llama_cpp.llama_sampler_reset(self._samplers[seq_id]) + + def close(self) -> None: + self.set_target_processing_enabled(False) + self.batch.token = ctypes.POINTER(llama_cpp.llama_token)() + llama_cpp.llama_batch_free(self.batch) + llama_cpp.llama_free(self.ctx) + for sampler in self._samplers: + llama_cpp.llama_sampler_free(sampler) + self._samplers.clear() + + def set_target_processing_enabled(self, enabled: bool) -> None: + if self.target_processing_enabled == enabled: + return + llama_cpp_ext.llama_set_embeddings_pre_norm( + self.target_ctx, + enabled, + False, + ) + self.target_processing_enabled = enabled + + def _clear_batch(self) -> None: + self.batch.n_tokens = 0 + + def _batch_embeddings(self) -> np.ndarray: + return self.batch_embeddings + + def _set_batch_embedding_row( + self, + row: int, + embedding: Union[np.ndarray, ctypes.POINTER(ctypes.c_float)], + ) -> None: + row_start = row * self.n_embd + row_end = row_start + self.n_embd + if isinstance(embedding, np.ndarray): + self._batch_embeddings()[row_start:row_end] = embedding + return + self._batch_embeddings()[row_start:row_end] = np.ctypeslib.as_array( + embedding, + shape=(self.n_embd,), + ) + + def _add_batch_token( + self, + *, + token: int, + pos: int, + seq_id: int, + logits: bool, + ) -> None: + slot = int(self.batch.n_tokens) + if slot >= self.n_batch: + raise RuntimeError("MTP draft batch capacity exceeded") + self.batch.token[slot] = int(token) + self.batch.pos[slot] = int(pos) + self.batch.seq_id[slot][0] = int(seq_id) + self.batch.n_seq_id[slot] = 1 + self.batch.logits[slot] = int(logits) + self.batch.n_tokens += 1 + + def _try_decode_batch(self) -> bool: + n_tokens = int(self.batch.n_tokens) + if n_tokens <= 0: + return True + started_at = time.perf_counter() + result = int(llama_cpp.llama_decode(self.ctx, self.batch)) + self.decode_seconds_total += time.perf_counter() - started_at + self.decode_calls_total += 1 + self.decode_tokens_total += n_tokens + if result != 0: + self.decode_failures_total += 1 + return False + return True + + def _decode_batch(self) -> None: + n_tokens = int(self.batch.n_tokens) + if n_tokens <= 0: + return + started_at = time.perf_counter() + result = int(llama_cpp.llama_decode(self.ctx, self.batch)) + self.decode_seconds_total += time.perf_counter() - started_at + self.decode_calls_total += 1 + self.decode_tokens_total += n_tokens + if result != 0: + self.decode_failures_total += 1 + raise RuntimeError(f"MTP draft decode failed with code {result}") + + def metric_definitions( + self, + ) -> List[Tuple[str, str, str, Union[int, float]]]: + decode_tokens_seconds = ( + self.decode_tokens_total / self.decode_seconds_total + if self.decode_seconds_total > 0.0 + else 0.0 + ) + return [ + ( + "counter", + "batch_server:mtp_decode_seconds_total", + "Time spent inside llama_decode() for the MTP draft context.", + self.decode_seconds_total, + ), + ( + "counter", + "batch_server:mtp_decode_calls_total", + "Number of llama_decode() calls made by the MTP draft context.", + self.decode_calls_total, + ), + ( + "counter", + "batch_server:mtp_decode_tokens_total", + "Number of batch rows decoded by the MTP draft context.", + self.decode_tokens_total, + ), + ( + "counter", + "batch_server:mtp_decode_failures_total", + "Number of failed MTP draft context decode calls.", + self.decode_failures_total, + ), + ( + "gauge", + "batch_server:mtp_decode_tokens_seconds", + "Average MTP draft context decode throughput in batch rows/s.", + decode_tokens_seconds, + ), + ] + + def can_draft(self, input_length: int, /, *, seq_id: int) -> bool: + if ( + input_length <= 0 + or seq_id < 0 + or seq_id >= self.n_seq_max + or not self.ready[seq_id] + ): + return False + return self.ready_pos[seq_id] == input_length - 1 + + def process(self, batch: Any, /) -> None: + n_tokens = int(batch.n_tokens) + if ( + n_tokens <= 0 + or not self.target_processing_enabled + or not bool(batch.token) + or bool(batch.embd) + ): + return + + h_tgt = llama_cpp_ext.llama_get_embeddings_pre_norm(self.target_ctx) + if not h_tgt: + raise RuntimeError("missing target pre-norm embeddings for MTP") + h_tgt_rows = np.ctypeslib.as_array( + h_tgt, + shape=(n_tokens, self.n_embd), + ) + h_tgt_rows = self._normalize_target_hidden_rows(h_tgt_rows) + + previous_row_by_seq: Dict[int, int] = {} + first_pos_by_seq: Dict[int, int] = {} + target_rows_by_seq: Dict[int, List[int]] = {} + aligned_by_seq: Dict[int, bool] = {} + + for start in range(0, n_tokens, self.n_batch): + self._process_rows( + batch, + h_tgt_rows, + start, + min(start + self.n_batch, n_tokens), + previous_row_by_seq, + first_pos_by_seq, + target_rows_by_seq, + aligned_by_seq, + ) + + for seq_id, rows in target_rows_by_seq.items(): + if not aligned_by_seq.get(seq_id, False): + self.ready[seq_id] = False + continue + if rows[-1] - rows[0] + 1 == len(rows): + target_rows = h_tgt_rows[rows[0] : rows[-1] + 1] + else: + target_rows = h_tgt_rows[rows].copy() + target_positions = [int(batch.pos[source_index]) for source_index in rows] + self.verify_h[seq_id] = target_rows + self.verify_h_pos[seq_id] = target_positions + self.verify_h_rows[seq_id] = len(rows) + self.pending_h[seq_id] = target_rows[-1] + self.ready[seq_id] = True + self.ready_pos[seq_id] = target_positions[-1] + 1 + + def _process_rows( + self, + batch: Any, + h_tgt_rows: np.ndarray, + start: int, + end: int, + previous_row_by_seq: Dict[int, int], + first_pos_by_seq: Dict[int, int], + target_rows_by_seq: Dict[int, List[int]], + aligned_by_seq: Dict[int, bool], + ) -> None: + added_pos_by_seq: Dict[int, int] = {} + self._clear_batch() + for index in range(start, end): + if int(batch.n_seq_id[index]) != 1: + raise RuntimeError("MTP requires one sequence id per batch token") + seq_id = int(batch.seq_id[index][0]) + if seq_id < 0 or seq_id >= self.n_seq_max: + raise RuntimeError(f"MTP sequence id out of range: {seq_id}") + pos = int(batch.pos[index]) + first_pos = first_pos_by_seq.setdefault(seq_id, pos) + aligned = first_pos <= 0 or ( + self.ready[seq_id] and self.ready_pos[seq_id] == first_pos + ) + aligned_by_seq.setdefault(seq_id, aligned) + mtp_pos = ( + pos - 1 + if previous_row_by_seq.get(seq_id) is None + else int(batch.pos[previous_row_by_seq[seq_id]]) + ) + if aligned and mtp_pos >= 0 and mtp_pos >= self.context_pos[seq_id]: + previous_row = previous_row_by_seq.get(seq_id) + slot = int(self.batch.n_tokens) + self._add_batch_token( + token=int(batch.token[index]), + pos=mtp_pos, + seq_id=seq_id, + logits=False, + ) + if previous_row is None: + self._set_batch_embedding_row(slot, self.pending_h[seq_id]) + else: + self._set_batch_embedding_row(slot, h_tgt_rows[previous_row]) + added_pos_by_seq[seq_id] = mtp_pos + previous_row_by_seq[seq_id] = index + target_rows_by_seq.setdefault(seq_id, []).append(index) + + if int(self.batch.n_tokens) > 0: + self._decode_batch() + for seq_id, pos in added_pos_by_seq.items(): + self.context_pos[seq_id] = max(self.context_pos[seq_id], pos + 1) + + def draft( + self, + input_ids: np.ndarray, + /, + *, + seq_id: int, + max_tokens: Optional[int], + ) -> np.ndarray: + if ( + self.num_pred_tokens <= 0 + or input_ids.size == 0 + or seq_id < 0 + or seq_id >= self.n_seq_max + or not self.ready[seq_id] + ): + return np.array([], dtype=np.intc) + n_predict = self.num_pred_tokens + if max_tokens is not None: + n_predict = min(n_predict, max_tokens) + if n_predict <= 0: + return np.array([], dtype=np.intc) + + n_past = int(input_ids.size) - 1 + if self.ready_pos[seq_id] != n_past: + return np.array([], dtype=np.intc) + first_pos = n_past - 1 + if first_pos < 0: + return np.array([], dtype=np.intc) + + token = int(input_ids[-1]) + drafted: List[int] = [] + if self.context_pos[seq_id] > first_pos: + self.truncate(seq_id, first_pos) + if self.context_pos[seq_id] < first_pos: + self.ready[seq_id] = False + return np.array([], dtype=np.intc) + + self._reset_sampler(seq_id) + self._clear_batch() + self._add_batch_token( + token=token, + pos=first_pos, + seq_id=seq_id, + logits=True, + ) + self._set_batch_embedding_row(0, self.pending_h[seq_id]) + if not self._try_decode_batch(): + self.truncate(seq_id, first_pos) + return np.array([], dtype=np.intc) + self.context_pos[seq_id] = n_past + + while len(drafted) < n_predict: + sampled_token = self._sample_token(seq_id=seq_id) + if sampled_token is None: + break + token = sampled_token + drafted.append(token) + if len(drafted) >= n_predict: + break + h_row = llama_cpp_ext.llama_get_embeddings_pre_norm_ith(self.ctx, 0) + if not h_row: + break + h_row = self._normalize_draft_hidden_row(h_row) + self._clear_batch() + self._add_batch_token( + token=token, + pos=first_pos + len(drafted), + seq_id=seq_id, + logits=True, + ) + self._set_batch_embedding_row(0, h_row) + if not self._try_decode_batch(): + break + self.context_pos[seq_id] = first_pos + len(drafted) + 1 + + if not drafted: + self.truncate(seq_id, n_past) + return np.array([], dtype=np.intc) + self.truncate(seq_id, n_past) + return np.asarray(drafted, dtype=np.intc) + + def draft_many( + self, + requests: Sequence[Tuple[np.ndarray, int, Optional[int]]], + /, + ) -> List[np.ndarray]: + results = [np.array([], dtype=np.intc) for _ in requests] + active: List["MTPDraftProvider.DraftManyState"] = [] + for result_index, (input_ids, seq_id, max_tokens) in enumerate(requests): + if ( + self.num_pred_tokens <= 0 + or input_ids.size == 0 + or seq_id < 0 + or seq_id >= self.n_seq_max + or not self.ready[seq_id] + ): + continue + n_predict = self.num_pred_tokens + if max_tokens is not None: + n_predict = min(n_predict, max_tokens) + if n_predict <= 0: + continue + if len(active) >= self.n_batch: + break + + n_past = int(input_ids.size) - 1 + if self.ready_pos[seq_id] != n_past: + continue + first_pos = n_past - 1 + if first_pos < 0: + continue + if self.context_pos[seq_id] > first_pos: + self.truncate(seq_id, first_pos) + if self.context_pos[seq_id] < first_pos: + self.ready[seq_id] = False + continue + self._reset_sampler(seq_id) + active.append( + self.DraftManyState( + result_index=result_index, + seq_id=seq_id, + first_pos=first_pos, + keep_len=n_past, + n_predict=n_predict, + token=int(input_ids[-1]), + drafted=[], + embedding=self.pending_h[seq_id], + cache_key=( + tuple(int(token) for token in input_ids.tolist()), + n_predict, + ), + ) + ) + + if not active: + return results + + touched = list(active) + try: + if all(state.n_predict == 1 for state in active): + grouped: Dict[ + Tuple[Tuple[int, ...], int], + List["MTPDraftProvider.DraftManyState"], + ] = {} + for state in active: + grouped.setdefault(state.cache_key, []).append(state) + + representatives = [states[0] for states in grouped.values()] + self._clear_batch() + for row, state in enumerate(representatives): + self._add_batch_token( + token=state.token, + pos=state.first_pos, + seq_id=state.seq_id, + logits=True, + ) + self._set_batch_embedding_row(row, state.embedding) + + if self._try_decode_batch(): + sampled_tokens = [ + self._sample_token(row, seq_id=state.seq_id) + for row, state in enumerate(representatives) + ] + for representative, sampled_token in zip( + representatives, + sampled_tokens, + ): + if sampled_token is None: + continue + self.context_pos[representative.seq_id] = max( + self.context_pos[representative.seq_id], + representative.keep_len, + ) + for state in grouped[representative.cache_key]: + state.drafted.append(sampled_token) + active = [] + + while active: + self._clear_batch() + for row, state in enumerate(active): + self._add_batch_token( + token=state.token, + pos=state.first_pos + len(state.drafted), + seq_id=state.seq_id, + logits=True, + ) + self._set_batch_embedding_row(row, state.embedding) + + if not self._try_decode_batch(): + break + + next_active: List["MTPDraftProvider.DraftManyState"] = [] + sampled_tokens = [ + self._sample_token(row, seq_id=state.seq_id) + for row, state in enumerate(active) + ] + for row, (state, sampled_token) in enumerate(zip(active, sampled_tokens)): + decoded_pos = state.first_pos + len(state.drafted) + self.context_pos[state.seq_id] = max( + self.context_pos[state.seq_id], + decoded_pos + 1, + ) + if sampled_token is None: + continue + h_row_ptr = llama_cpp_ext.llama_get_embeddings_pre_norm_ith( + self.ctx, row + ) + state.drafted.append(sampled_token) + if len(state.drafted) >= state.n_predict: + continue + if not h_row_ptr: + continue + h_row = self._normalize_draft_hidden_row(h_row_ptr) + state.token = sampled_token + state.embedding = h_row + next_active.append(state) + active = next_active + finally: + for state in touched: + self.truncate(state.seq_id, state.keep_len) + + for state in touched: + if state.drafted: + results[state.result_index] = np.asarray( + state.drafted, + dtype=np.intc, + ) + return results + + @dataclass(frozen=True) + class SampledContextRow: + seq_id: int + draft_pos: int + token: int + source_row: Optional[int] + + @dataclass(frozen=True) + class SampledPendingRow: + update_index: int + seq_id: int + draft_pos: int + token: int + source_row: int + + @dataclass(frozen=True) + class SampledOutput: + update_index: int + seq_id: int + output_index: int + keep_len: int + ready_pos: int + + @dataclass(frozen=True) + class SampledBatchUpdate: + seq_id: int + start_pos: int + tokens: List[int] + row_indices: List[int] + target_count: int + sample_index: int + pending_token: Optional[int] + max_tokens: Optional[int] + + @dataclass + class SampledBatchPlan: + context_rows: List["MTPDraftProvider.SampledContextRow"] + pending_rows: List["MTPDraftProvider.SampledPendingRow"] + sample_pending_index_by_update: Dict[int, int] + + @dataclass + class SampledDraftState: + update_index: int + seq_id: int + keep_len: int + pos: int + token: int + drafted: List[int] + n_predict: int + embedding: np.ndarray + + def _build_sampled_batch_plan( + self, + updates: Sequence["MTPDraftProvider.SampledBatchUpdate"], + /, + ) -> "MTPDraftProvider.SampledBatchPlan": + context_rows: List["MTPDraftProvider.SampledContextRow"] = [] + pending_rows: List["MTPDraftProvider.SampledPendingRow"] = [] + sample_pending_index_by_update: Dict[int, int] = {} + + for update_index, update in enumerate(updates): + seq_id = update.seq_id + if seq_id < 0 or seq_id >= self.n_seq_max: + continue + start_pos = update.start_pos + tokens = update.tokens + row_indices = update.row_indices + target_count = update.target_count + sample_index = update.sample_index + pending_token = update.pending_token + if ( + pending_token is None + or start_pos <= 0 + or target_count <= 0 + or target_count > len(tokens) + or target_count > len(row_indices) + or sample_index < 0 + or sample_index >= target_count + ): + continue + + for target_index in range(sample_index + 1): + mtp_pos = start_pos + target_index - 1 + if mtp_pos < 0: + continue + source_row = ( + None + if target_index == 0 + else row_indices[target_index - 1] + ) + context_rows.append( + self.SampledContextRow( + seq_id=seq_id, + draft_pos=mtp_pos, + token=tokens[target_index], + source_row=source_row, + ) + ) + + actual_pos = start_pos + sample_index + pending_rows.append( + self.SampledPendingRow( + update_index=update_index, + seq_id=seq_id, + draft_pos=actual_pos, + token=pending_token, + source_row=row_indices[sample_index], + ) + ) + sample_pending_index_by_update[update_index] = len(pending_rows) - 1 + + return self.SampledBatchPlan( + context_rows=context_rows, + pending_rows=pending_rows, + sample_pending_index_by_update=sample_pending_index_by_update, + ) + + def _decode_sampled_context_rows( + self, + context_rows: Sequence["MTPDraftProvider.SampledContextRow"], + h_tgt_rows: np.ndarray, + /, + ) -> None: + self._clear_batch() + decoded_context_rows: List[Tuple[int, int]] = [] + for row in context_rows: + if row.draft_pos < self.context_pos[row.seq_id]: + continue + if row.source_row is None: + embedding = self.pending_h[row.seq_id] + else: + embedding = h_tgt_rows[row.source_row] + self._add_batch_token( + token=row.token, + pos=row.draft_pos, + seq_id=row.seq_id, + logits=False, + ) + self._set_batch_embedding_row(int(self.batch.n_tokens) - 1, embedding) + decoded_context_rows.append((row.seq_id, row.draft_pos)) + + if int(self.batch.n_tokens) > 0: + self._decode_batch() + for seq_id, draft_pos in decoded_context_rows: + self.context_pos[seq_id] = max( + self.context_pos[seq_id], + draft_pos + 1, + ) + + def _decode_sampled_pending_rows( + self, + plan: "MTPDraftProvider.SampledBatchPlan", + h_tgt_rows: np.ndarray, + /, + ) -> List["MTPDraftProvider.SampledOutput"]: + sampled_outputs: List["MTPDraftProvider.SampledOutput"] = [] + pending_rows = plan.pending_rows + + self._clear_batch() + for pending_index, row in enumerate(pending_rows): + if row.draft_pos < self.context_pos[row.seq_id]: + continue + is_sample_pending = ( + pending_index + == plan.sample_pending_index_by_update.get(row.update_index) + ) + slot = int(self.batch.n_tokens) + self._add_batch_token( + token=row.token, + pos=row.draft_pos, + seq_id=row.seq_id, + logits=is_sample_pending, + ) + self._set_batch_embedding_row(slot, h_tgt_rows[row.source_row]) + if is_sample_pending: + sampled_outputs.append( + self.SampledOutput( + update_index=row.update_index, + seq_id=row.seq_id, + output_index=slot, + keep_len=row.draft_pos + 1, + ready_pos=row.draft_pos + 1, + ) + ) + + self._decode_batch() + for row in pending_rows: + self.context_pos[row.seq_id] = max( + self.context_pos[row.seq_id], + row.draft_pos + 1, + ) + + return sampled_outputs + + def _start_sampled_draft_states( + self, + updates: Sequence["MTPDraftProvider.SampledBatchUpdate"], + sampled_outputs: Sequence["MTPDraftProvider.SampledOutput"], + results: List[np.ndarray], + /, + ) -> List["MTPDraftProvider.SampledDraftState"]: + active: List["MTPDraftProvider.SampledDraftState"] = [] + for output in sampled_outputs: + self._reset_sampler(output.seq_id) + sampled_token = self._sample_token( + output.output_index, + seq_id=output.seq_id, + ) + if sampled_token is None: + continue + update = updates[output.update_index] + seq_id = update.seq_id + self.ready[seq_id] = True + n_predict = self.num_pred_tokens + max_tokens = update.max_tokens + if max_tokens is not None: + n_predict = min(n_predict, max_tokens) + if n_predict <= 0: + continue + if n_predict > 1: + h_row = llama_cpp_ext.llama_get_embeddings_pre_norm_ith( + self.ctx, output.output_index + ) + if h_row: + active.append( + self.SampledDraftState( + update_index=output.update_index, + seq_id=seq_id, + keep_len=output.keep_len, + pos=output.keep_len, + token=sampled_token, + drafted=[sampled_token], + n_predict=n_predict, + embedding=self._normalize_draft_hidden_row(h_row), + ) + ) + results[output.update_index] = np.asarray([sampled_token], dtype=np.intc) + + return active + + def _extend_sampled_draft_states( + self, + active: List["MTPDraftProvider.SampledDraftState"], + results: List[np.ndarray], + cleanup_keep_len_by_seq: Dict[int, int], + /, + ) -> None: + touched = list(active) + try: + while active: + self._clear_batch() + for batch_row, state in enumerate(active): + self._add_batch_token( + token=state.token, + pos=state.pos, + seq_id=state.seq_id, + logits=True, + ) + self._set_batch_embedding_row(batch_row, state.embedding) + + if not self._try_decode_batch(): + break + + next_active: List["MTPDraftProvider.SampledDraftState"] = [] + sampled_tokens = [ + self._sample_token(batch_row, seq_id=state.seq_id) + for batch_row, state in enumerate(active) + ] + for batch_row, (state, sampled_token) in enumerate( + zip(active, sampled_tokens) + ): + self.context_pos[state.seq_id] = max( + self.context_pos[state.seq_id], + state.pos + 1, + ) + if sampled_token is None: + continue + state.drafted.append(sampled_token) + if len(state.drafted) >= state.n_predict: + continue + h_row = llama_cpp_ext.llama_get_embeddings_pre_norm_ith( + self.ctx, batch_row + ) + if not h_row: + continue + h_row = self._normalize_draft_hidden_row(h_row) + state.token = sampled_token + state.embedding = h_row + state.pos += 1 + next_active.append(state) + active = next_active + finally: + for state in touched: + cleanup_keep_len_by_seq[state.seq_id] = state.keep_len + for seq_id, keep_len in cleanup_keep_len_by_seq.items(): + self._truncate_memory(seq_id, keep_len) + + for state in touched: + if state.drafted: + results[state.update_index] = np.asarray( + state.drafted, + dtype=np.intc, + ) + + def process_sampled_batch( + self, + updates: Sequence["MTPDraftProvider.SampledBatchUpdate"], + /, + ) -> List[np.ndarray]: + results = [np.array([], dtype=np.intc) for _ in updates] + if self.num_pred_tokens <= 0 or not updates: + return results + h_tgt = llama_cpp_ext.llama_get_embeddings_pre_norm(self.target_ctx) + if not h_tgt: + raise RuntimeError("missing target pre-norm embeddings for MTP") + n_target_rows = max( + ( + max(update.row_indices) + 1 + for update in updates + if update.row_indices + ), + default=0, + ) + if n_target_rows <= 0: + return results + h_tgt_rows = np.ctypeslib.as_array(h_tgt, shape=(n_target_rows, self.n_embd)) + h_tgt_rows = self._normalize_target_hidden_rows(h_tgt_rows) + + plan = self._build_sampled_batch_plan(updates) + if not plan.context_rows and not plan.pending_rows: + return results + if ( + len(plan.context_rows) > self.n_batch + or len(plan.pending_rows) > self.n_batch + ): + raise RuntimeError("MTP draft batch capacity exceeded") + + self._decode_sampled_context_rows(plan.context_rows, h_tgt_rows) + sampled_outputs = self._decode_sampled_pending_rows(plan, h_tgt_rows) + cleanup_keep_len_by_seq: Dict[int, int] = {} + for output in sampled_outputs: + update = updates[output.update_index] + sample_index = update.sample_index + sample_source_row = update.row_indices[sample_index] + self.pending_h[output.seq_id] = h_tgt_rows[sample_source_row] + self.ready[output.seq_id] = True + self.ready_pos[output.seq_id] = output.ready_pos + cleanup_keep_len_by_seq[output.seq_id] = output.keep_len + + for seq_id, keep_len in cleanup_keep_len_by_seq.items(): + self._truncate_memory(seq_id, keep_len) + + if sampled_outputs: + active = self._start_sampled_draft_states( + updates, + sampled_outputs, + results, + ) + self._extend_sampled_draft_states( + active, + results, + cleanup_keep_len_by_seq, + ) + + return results + + def accept(self, seq_id: int, accepted_draft_tokens: int) -> None: + if seq_id < 0 or seq_id >= self.n_seq_max: + return + n_rows = self.verify_h_rows[seq_id] + if n_rows <= 0: + return + row = min(max(accepted_draft_tokens, 0), n_rows - 1) + self.pending_h[seq_id] = self.verify_h[seq_id][row] + self.ready[seq_id] = True + self.ready_pos[seq_id] = self.verify_h_pos[seq_id][row] + 1 + + def _truncate_memory(self, seq_id: int, keep_len: int) -> None: + if seq_id < 0 or seq_id >= self.n_seq_max: + return + if not llama_cpp.llama_memory_seq_rm( + self.mem, + seq_id, + keep_len, + -1, + ): + raise RuntimeError( + f"failed to truncate MTP draft sequence {seq_id} at position {keep_len}" + ) + self.context_pos[seq_id] = min(self.context_pos[seq_id], keep_len) + + def truncate(self, seq_id: int, keep_len: int) -> None: + if seq_id < 0 or seq_id >= self.n_seq_max: + return + self._truncate_memory(seq_id, keep_len) + if keep_len <= 0: + self.pending_h[seq_id].fill(0.0) + self.verify_h[seq_id] = np.empty((0, self.n_embd), dtype=np.float32) + self.verify_h_pos[seq_id] = [] + self.verify_h_rows[seq_id] = 0 + self.ready[seq_id] = False + self.ready_pos[seq_id] = 0 + self.context_pos[seq_id] = 0 + return + + if self.ready_pos[seq_id] != keep_len: + self.ready[seq_id] = False + + def copy_sequence( + self, + source_seq_id: int, + dest_seq_id: int, + p0: int, + p1: int, + ) -> None: + if ( + source_seq_id < 0 + or source_seq_id >= self.n_seq_max + or dest_seq_id < 0 + or dest_seq_id >= self.n_seq_max + ): + return + llama_cpp.llama_memory_seq_cp( + self.mem, + source_seq_id, + dest_seq_id, + p0, + p1, + ) + source_ready_pos = self.ready_pos[source_seq_id] + copied_full_ready_state = p1 < 0 or p1 == source_ready_pos + if self.ready[source_seq_id] and copied_full_ready_state: + self.pending_h[dest_seq_id] = self.pending_h[source_seq_id] + self.verify_h[dest_seq_id] = self.verify_h[source_seq_id].copy() + self.verify_h_pos[dest_seq_id] = list(self.verify_h_pos[source_seq_id]) + self.verify_h_rows[dest_seq_id] = self.verify_h_rows[source_seq_id] + self.ready[dest_seq_id] = True + self.ready_pos[dest_seq_id] = source_ready_pos + self.context_pos[dest_seq_id] = min( + self.context_pos[source_seq_id], + source_ready_pos, + ) + return + + self.pending_h[dest_seq_id].fill(0.0) + self.verify_h[dest_seq_id] = np.empty((0, self.n_embd), dtype=np.float32) + self.verify_h_pos[dest_seq_id] = [] + self.verify_h_rows[dest_seq_id] = 0 + self.ready[dest_seq_id] = False + self.ready_pos[dest_seq_id] = 0 + self.context_pos[dest_seq_id] = 0 + + +class CompletionRequestCancelledError(RuntimeError): + pass + + +class CompletionRequestValidationError(ValueError): + pass + + +class CompletionResponseParsingError(RuntimeError): + pass + + +def omit_additional_properties(schema: Dict[str, Any]) -> None: + schema.pop("additionalProperties", None) + + +class CompletionChunkLogprobs(TypedDict): + tokens: List[str] + text_offset: List[int] + token_logprobs: List[Optional[float]] + top_logprobs: List[Optional[Dict[str, float]]] + + +class CompletionChunkChoice(TypedDict): + text: str + index: int + logprobs: Optional[CompletionChunkLogprobs] + finish_reason: Optional[str] + + +class CompletionChunk(TypedDict): + id: str + object: Literal["text_completion"] + created: int + model: str + choices: List[CompletionChunkChoice] + + +CompletionStream = Generator[CompletionChunk, None, OpenAICompletion] +CompletionPrompt = Union[str, List[int], List[str], List[List[int]]] + + +class CreateCompletionRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + prompt: CompletionPrompt = "" + suffix: Optional[str] = None + max_tokens: Optional[int] = Field(default=16, ge=0) + temperature: float = 0.8 + top_p: float = Field(default=0.95, ge=0.0, le=1.0) + echo: bool = False + stop: Optional[Union[str, List[str]]] = None + stream: bool = False + logprobs: Optional[int] = Field(default=None, ge=0) + presence_penalty: Optional[float] = Field(default=0.0, ge=-2.0, le=2.0) + frequency_penalty: Optional[float] = Field(default=0.0, ge=-2.0, le=2.0) + logit_bias: Optional[Dict[str, float]] = None + seed: Optional[int] = None + model: Optional[str] = None + n: int = Field(default=1, ge=1) + best_of: Optional[int] = Field(default=None, ge=1) + user: Optional[str] = None + + @field_validator("logit_bias") + @classmethod + def validate_logit_bias(cls, value: Optional[Dict[str, float]]) -> Optional[Dict[str, float]]: + if value is None: + return None + result: Dict[str, float] = {} + for key, bias in value.items(): + int(key) + result[key] = float(bias) + return result + + @model_validator(mode="after") + def validate_after(self) -> "CreateCompletionRequest": + if self.best_of is None: + self.best_of = self.n + if self.best_of < self.n: + raise ValueError("best_of must be greater than or equal to n") + if self.stream and self.best_of > 1: + raise ValueError("best_of is not supported for streaming completions") + if len(self.normalized_prompt()) > 1 and self.stream: + raise ValueError("streaming does not support multiple prompts") + return self + + def normalized_prompt(self) -> List[Union[str, List[int]]]: + if isinstance(self.prompt, str): + return [self.prompt] + if all(isinstance(token, int) for token in self.prompt): + return [cast(List[int], self.prompt)] + if all(isinstance(prompt, str) for prompt in self.prompt): + return cast(List[Union[str, List[int]]], list(cast(List[str], self.prompt))) + if all( + isinstance(prompt, list) + and all(isinstance(token, int) for token in prompt) + for prompt in self.prompt + ): + return cast( + List[Union[str, List[int]]], + list(cast(List[List[int]], self.prompt)), + ) + raise ValueError("prompt must be a string, token ids, list of strings, or list of token-id lists") + + +class ChatCompletionFunctionCall(BaseModel): + model_config = ConfigDict( + extra="allow", + json_schema_extra=omit_additional_properties, + ) + + name: str + arguments: Optional[str] = None + + +class ChatCompletionToolCall(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: Optional[str] = None + type: Literal["function"] = "function" + function: ChatCompletionFunctionCall + + +class ChatCompletionRequestMessage(BaseModel): + model_config = ConfigDict( + extra="allow", + json_schema_extra=omit_additional_properties, + ) + + role: Literal["system", "developer", "user", "assistant", "tool", "function"] = Field( + default="user" + ) + content: Optional[Union[str, List[Dict[str, Any]]]] = Field(default="") + name: Optional[str] = Field(default=None) + tool_call_id: Optional[str] = Field(default=None) + function_call: Optional[ChatCompletionFunctionCall] = Field(default=None) + tool_calls: Optional[List[ChatCompletionToolCall]] = Field(default=None) + + +class ChatTemplateFunctionDefinition(TypedDict, total=False): + name: str + description: Optional[str] + parameters: Optional[Dict[str, Any]] + strict: Optional[bool] + content_type: Optional[str] + + +class ChatTemplateTool(TypedDict, total=False): + type: Literal["function"] + original_type: str + function: ChatTemplateFunctionDefinition + + +class ChatTemplateFunctionCall(TypedDict, total=False): + name: str + + +class ChatTemplateToolChoice(TypedDict, total=False): + type: Literal["function"] + function: ChatTemplateFunctionCall + + +class ChatTemplateResponseFormatJsonSchema(TypedDict, total=False): + name: Optional[str] + description: Optional[str] + schema: Dict[str, Any] + strict: Optional[bool] + + +class ChatTemplateResponseFormat(TypedDict, total=False): + type: Literal["text", "json_object", "json_schema"] + schema: Dict[str, Any] + json_schema: ChatTemplateResponseFormatJsonSchema + + +class ChatCompletionFunctionDefinition(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: str + description: Optional[str] = None + parameters: Optional[Dict[str, Any]] = None + + def to_template_function(self) -> ChatTemplateFunctionDefinition: + function: ChatTemplateFunctionDefinition = {"name": self.name} + if self.description is not None: + function["description"] = self.description + if self.parameters is not None: + function["parameters"] = self.parameters + return function + + +class ChatCompletionToolFunction(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: str + description: Optional[str] = None + parameters: Optional[Dict[str, Any]] = None + strict: Optional[bool] = None + + def to_template_function(self) -> ChatTemplateFunctionDefinition: + function: ChatTemplateFunctionDefinition = {"name": self.name} + if self.description is not None: + function["description"] = self.description + if self.parameters is not None: + function["parameters"] = self.parameters + if self.strict is not None: + function["strict"] = self.strict + return function + + +class ChatCompletionTool(BaseModel): + model_config = ConfigDict(extra="ignore") + + type: Literal["function"] + function: ChatCompletionToolFunction + + def to_template_tool(self) -> ChatTemplateTool: + return { + "type": self.type, + "function": self.function.to_template_function(), + } + + +class ChatCompletionFunctionCallOption(BaseModel): + model_config = ConfigDict(extra="ignore") + + name: str + + def to_template_function_call(self) -> ChatTemplateFunctionCall: + return {"name": self.name} + + +class ChatCompletionToolChoiceObject(BaseModel): + model_config = ConfigDict(extra="ignore") + + type: Literal["function"] + function: ChatCompletionFunctionCallOption + + def to_template_tool_choice(self) -> ChatTemplateToolChoice: + return { + "type": self.type, + "function": self.function.to_template_function_call(), + } + + +class ChatCompletionResponseFormatJsonSchema(BaseModel): + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + name: Optional[str] = None + description: Optional[str] = None + schema_: Optional[Dict[str, Any]] = Field(default=None, alias="schema") + strict: Optional[bool] = None + + def to_template_json_schema(self) -> ChatTemplateResponseFormatJsonSchema: + json_schema: ChatTemplateResponseFormatJsonSchema = {} + if self.name is not None: + json_schema["name"] = self.name + if self.description is not None: + json_schema["description"] = self.description + if self.schema_ is not None: + json_schema["schema"] = self.schema_ + if self.strict is not None: + json_schema["strict"] = self.strict + return json_schema + + +class ChatCompletionResponseFormat(BaseModel): + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + type: Literal["text", "json_object", "json_schema"] + schema_: Optional[Dict[str, Any]] = Field(default=None, alias="schema") + json_schema: Optional[ChatCompletionResponseFormatJsonSchema] = None + + def to_template_response_format(self) -> ChatTemplateResponseFormat: + response_format: ChatTemplateResponseFormat = {"type": self.type} + if self.schema_ is not None: + response_format["schema"] = self.schema_ + if self.json_schema is not None: + response_format["json_schema"] = self.json_schema.to_template_json_schema() + return response_format + + +class CreateChatCompletionRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + messages: List[ChatCompletionRequestMessage] = Field(default_factory=list) + max_tokens: Optional[int] = Field(default=None, ge=0) + temperature: float = 0.8 + top_p: float = Field(default=0.95, ge=0.0, le=1.0) + stop: Optional[Union[str, List[str]]] = None + stream: bool = False + logprobs: Optional[bool] = Field(default=False) + top_logprobs: Optional[int] = Field(default=None, ge=0) + presence_penalty: Optional[float] = Field(default=0.0, ge=-2.0, le=2.0) + frequency_penalty: Optional[float] = Field(default=0.0, ge=-2.0, le=2.0) + logit_bias: Optional[Dict[str, float]] = None + seed: Optional[int] = None + model: Optional[str] = None + n: int = Field(default=1, ge=1) + user: Optional[str] = None + functions: Optional[List[ChatCompletionFunctionDefinition]] = None + function_call: Optional[ + Union[Literal["none", "auto"], ChatCompletionFunctionCallOption] + ] = None + tools: Optional[List[ChatCompletionTool]] = None + tool_choice: Optional[ + Union[Literal["none", "auto", "required"], ChatCompletionToolChoiceObject] + ] = None + response_format: Optional[ChatCompletionResponseFormat] = None + reasoning_effort: Optional[ + Literal["low", "medium", "high", "minimal", "none"] + ] = None + + @field_validator("logit_bias") + @classmethod + def validate_logit_bias(cls, value: Optional[Dict[str, float]]) -> Optional[Dict[str, float]]: + if value is None: + return None + result: Dict[str, float] = {} + for key, bias in value.items(): + int(key) + result[key] = float(bias) + return result + + @model_validator(mode="after") + def validate_after(self) -> "CreateChatCompletionRequest": + if self.top_logprobs is not None and not self.logprobs: + raise ValueError("top_logprobs requires logprobs=true") + return self + + +class ResponsesFunctionTool(BaseModel): + model_config = ConfigDict(extra="ignore") + + type: Literal["function"] + name: str + description: Optional[str] = None + parameters: Optional[Dict[str, Any]] = None + strict: Optional[bool] = None + + def to_chat_template_tool(self) -> ChatTemplateTool: + return { + "type": "function", + "function": { + "name": self.name, + "description": self.description, + "parameters": self.parameters, + "strict": self.strict, + }, + } + + +class ResponsesCustomToolFormat(BaseModel): + model_config = ConfigDict(extra="ignore") + + syntax: Optional[str] = None + definition: Optional[str] = None + + +class ResponsesCustomTool(BaseModel): + model_config = ConfigDict(extra="ignore") + + type: Literal["custom"] + name: str + description: Optional[str] = None + format: Optional[ResponsesCustomToolFormat] = None + strict: Optional[bool] = None + + def to_chat_template_tool(self) -> ChatTemplateTool: + tool_format = self.format + syntax = tool_format.syntax if tool_format is not None else None + definition = tool_format.definition if tool_format is not None else None + description = self.description or "" + text_tool_guidance = ( + "This is a text tool. When calling it, the " + "`function.arguments` field itself must be the raw input string. " + "Do not wrap the input in JSON and do not use an object such as " + "'{\"input\": \"...\"}' or '{\"patch\": \"...\"}'." + ) + if isinstance(syntax, str) and isinstance(definition, str): + if description: + description = ( + f"{description}\n\n{text_tool_guidance}\n\n" + f"{syntax}:\n{definition}" + ) + else: + description = f"{text_tool_guidance}\n\n{syntax}:\n{definition}" + elif description: + description = f"{description}\n\n{text_tool_guidance}" + else: + description = text_tool_guidance + return { + "type": "function", + "original_type": "custom", + "function": { + "name": self.name, + "description": description or None, + "parameters": { + "type": "object", + "properties": { + "input": { + "type": "string", + "description": ( + "Raw input text for this tool. " + "For apply_patch, put the full patch here." + ), + }, + }, + "required": ["input"], + "additionalProperties": False, + }, + "strict": self.strict, + "content_type": "text", + }, + } + + +class ResponsesWebSearchTool(BaseModel): + model_config = ConfigDict(extra="ignore") + + type: Literal["web_search"] + + +ResponsesToolDefinition = Union[ + ResponsesFunctionTool, + ResponsesCustomTool, + ResponsesWebSearchTool, +] + + +class ResponsesToolChoiceObject(BaseModel): + model_config = ConfigDict(extra="ignore") + + type: Literal["function", "custom"] + name: str + + def to_chat_template_tool_choice(self) -> ChatTemplateToolChoice: + return { + "type": "function", + "function": { + "name": self.name, + }, + } + + +ResponsesToolChoice = Union[ + Literal["auto", "none", "required"], + ResponsesToolChoiceObject, +] + + +class ResponsesTextFormatJsonSchema(BaseModel): + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + schema_: Dict[str, Any] = Field(alias="schema") + + def to_template_json_schema(self) -> ChatTemplateResponseFormatJsonSchema: + return {"schema": self.schema_} + + +class ResponsesTextFormat(BaseModel): + model_config = ConfigDict(extra="ignore", populate_by_name=True) + + type: Literal["text", "json_object", "json_schema"] + schema_: Optional[Dict[str, Any]] = Field(default=None, alias="schema") + json_schema: Optional[ResponsesTextFormatJsonSchema] = None + + def to_chat_response_format(self) -> ChatTemplateResponseFormat: + response_format: ChatTemplateResponseFormat = {"type": self.type} + if self.schema_ is not None: + response_format["schema"] = self.schema_ + if self.json_schema is not None: + response_format["json_schema"] = self.json_schema.to_template_json_schema() + return response_format + + +class ResponsesTextOptions(BaseModel): + model_config = ConfigDict(extra="ignore") + + format: Optional[ResponsesTextFormat] = None + + +class ResponsesReasoningOptions(BaseModel): + model_config = ConfigDict(extra="ignore") + + effort: Optional[Literal["low", "medium", "high"]] = None + + +class CreateResponseRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + input: Any = "" + instructions: Optional[str] = None + max_output_tokens: Optional[int] = Field(default=None, ge=0) + temperature: Optional[float] = Field(default=0.8, ge=0.0, le=2.0) + top_p: Optional[float] = Field(default=0.95, ge=0.0, le=1.0) + stream: bool = False + top_logprobs: Optional[int] = Field(default=None, ge=0) + model: Optional[str] = None + tools: Optional[List["ResponsesToolDefinition"]] = None + tool_choice: Optional["ResponsesToolChoice"] = None + parallel_tool_calls: bool = True + text: Optional["ResponsesTextOptions"] = None + reasoning: Optional["ResponsesReasoningOptions"] = None + metadata: Optional[Dict[str, str]] = None + user: Optional[str] = None + previous_response_id: Optional[str] = None + conversation: Optional[str] = None + store: Optional[bool] = None + truncation: Optional[Literal["auto", "disabled"]] = None + + +class ResponseCreateWebSocketRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + type: Literal["response.create"] + model: Optional[str] = None + input: Any = "" + instructions: Optional[str] = None + max_output_tokens: Optional[int] = Field(default=None, ge=0) + temperature: Optional[float] = Field(default=0.8, ge=0.0, le=2.0) + top_p: Optional[float] = Field(default=0.95, ge=0.0, le=1.0) + stream: bool = True + top_logprobs: Optional[int] = Field(default=None, ge=0) + tools: Optional[List["ResponsesToolDefinition"]] = None + tool_choice: Optional["ResponsesToolChoice"] = None + parallel_tool_calls: bool = True + text: Optional["ResponsesTextOptions"] = None + reasoning: Optional["ResponsesReasoningOptions"] = None + metadata: Optional[Dict[str, str]] = None + user: Optional[str] = None + previous_response_id: Optional[str] = None + conversation: Optional[str] = None + store: Optional[bool] = None + truncation: Optional[Literal["auto", "disabled"]] = None + generate: Optional[bool] = None + + def to_create_response_request(self) -> CreateResponseRequest: + return CreateResponseRequest.model_validate( + self.model_dump(mode="python", exclude={"type"}) + ) + + +class ModelCardResponse(BaseModel): + id: str + object: Literal["model"] = "model" + created: int + owned_by: str + + +class ModelListResponse(BaseModel): + object: Literal["list"] = "list" + data: List[ModelCardResponse] + + +class HealthzResponse(BaseModel): + status: Literal["ok"] = "ok" + + +@dataclass +class ResponsesWebSocketState: + input_items: List[Any] + output_items: List[Dict[str, Any]] + + +class ConfigFile(BaseModel): + class ServerOptions(BaseModel): + host: str = "127.0.0.1" + port: int = 8000 + + class DiskCacheOptions(BaseModel): + path: str + max_bytes: int = Field(ge=0) + min_tokens: int = Field(default=128, ge=1) + + class FromPretrainedOptions(BaseModel): + repo_id: str + filename: str + additional_files: Optional[List[str]] = None + local_dir: Optional[str] = None + local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto" + cache_dir: Optional[str] = None + + @staticmethod + def _pattern_has_glob(pattern: str) -> bool: + return any(char in pattern for char in "*?[]") + + @staticmethod + def _subfolder_for_repo_file(repo_file: str) -> Optional[str]: + subfolder = str(Path(repo_file).parent) + return None if subfolder == "." else subfolder + + def _cached_repo_file_list(self) -> List[str]: + from huggingface_hub import scan_cache_dir + + try: + cache_info = scan_cache_dir(self.cache_dir) + except Exception: + return [] + + cached_files: set[str] = set() + for repo in cache_info.repos: + if repo.repo_type != "model" or repo.repo_id != self.repo_id: + continue + for revision in repo.revisions: + for cached_file in revision.files: + cached_files.add(cached_file.file_name) + return sorted(cached_files) + + def _download_repo_file(self, repo_file: str) -> str: + from huggingface_hub import hf_hub_download + + filename = Path(repo_file).name + subfolder = self._subfolder_for_repo_file(repo_file) + + download_kwargs: Dict[str, Any] = dict( + repo_id=self.repo_id, + filename=filename, + subfolder=subfolder, + local_dir=self.local_dir, + cache_dir=self.cache_dir, + ) + hf_hub_download_signature = inspect.signature(hf_hub_download) + supports_local_dir_use_symlinks = ( + "local_dir_use_symlinks" in hf_hub_download_signature.parameters + or any( + parameter.kind is inspect.Parameter.VAR_KEYWORD + for parameter in hf_hub_download_signature.parameters.values() + ) + ) + if supports_local_dir_use_symlinks: + download_kwargs["local_dir_use_symlinks"] = self.local_dir_use_symlinks + + try: + resolved_path = hf_hub_download(**download_kwargs) + except Exception as exc: + try: + cached_path = cast( + str, + hf_hub_download( + repo_id=self.repo_id, + filename=filename, + subfolder=subfolder, + cache_dir=self.cache_dir, + local_files_only=True, + ), + ) + except Exception: + raise exc + + if self.local_dir is None: + return cached_path + + resolved_path = os.path.join(self.local_dir, repo_file) + resolved_parent = os.path.dirname(resolved_path) + if resolved_parent: + os.makedirs(resolved_parent, exist_ok=True) + shutil.copy2(cached_path, resolved_path) + + return cast(str, resolved_path) + + @staticmethod + def _match_single_file(pattern: str, file_list: Sequence[str]) -> str: + matching_files = [file for file in file_list if fnmatch.fnmatch(file, pattern)] + if len(matching_files) == 0: + raise ValueError( + f"No file found matching {pattern}\n\n" + f"Available Files:\n{json.dumps(list(file_list))}" + ) + if len(matching_files) > 1: + raise ValueError( + f"Multiple files found matching {pattern}\n\n" + f"Available Files:\n{json.dumps(list(file_list))}" + ) + return matching_files[0] + + def resolve_model_path(self) -> str: + try: + from huggingface_hub import HfFileSystem + from huggingface_hub.utils import validate_repo_id + except ImportError as exc: + try: + from huggingface_hub import HfFileSystem + from huggingface_hub.utils._validators import validate_repo_id + except ImportError: + raise ImportError( + "model.from_pretrained requires the huggingface-hub package. " + "You can install it with `pip install huggingface-hub`." + ) from exc + + validate_repo_id(self.repo_id) + requested_patterns = [self.filename, *(self.additional_files or [])] + if not any(self._pattern_has_glob(pattern) for pattern in requested_patterns): + model_path = self._download_repo_file(self.filename) + if self.additional_files: + for additional_file_name in self.additional_files: + self._download_repo_file(additional_file_name) + return model_path + + try: + hffs = HfFileSystem() + files = [ + file["name"] if isinstance(file, dict) else file + for file in hffs.ls(self.repo_id, recursive=True) + ] + file_list = [str(Path(file).relative_to(self.repo_id)) for file in files] + except Exception as exc: + file_list = self._cached_repo_file_list() + if not file_list: + raise exc + + matching_file = self._match_single_file(self.filename, file_list) + model_path = self._download_repo_file(matching_file) + + if self.additional_files: + for additional_file_name in self.additional_files: + matching_additional_file = self._match_single_file( + additional_file_name, file_list + ) + self._download_repo_file(matching_additional_file) + + return model_path + + class LoraOptions(BaseModel): + path: Optional[str] = None + from_pretrained: Optional["ConfigFile.FromPretrainedOptions"] = None + scale: float = 1.0 + + @model_validator(mode="after") + def validate_source(self) -> "ConfigFile.LoraOptions": + if (self.path is None) == (self.from_pretrained is None): + raise ValueError("exactly one of lora.path or lora.from_pretrained is required") + return self + + def resolve_path(self) -> str: + if self.from_pretrained is not None: + return self.from_pretrained.resolve_model_path() + assert self.path is not None + return self.path + + class MTMDEmbeddingCacheOptions(BaseModel): + path: str + max_bytes: int = Field(ge=0) + + class MTMDOptions(BaseModel): + mmproj_path: Optional[str] = None + mmproj_from_pretrained: Optional["ConfigFile.FromPretrainedOptions"] = None + embedding_cache: Optional["ConfigFile.MTMDEmbeddingCacheOptions"] = None + allowed_media_domains: Optional[List[str]] = None + allowed_local_media_path: Optional[str] = None + image_max_bytes: int = Field(default=20 * 1024 * 1024, ge=1) + audio_max_bytes: int = Field(default=100 * 1024 * 1024, ge=1) + image_timeout_seconds: float = Field(default=10.0, gt=0.0) + + @model_validator(mode="after") + def validate_source(self) -> "ConfigFile.MTMDOptions": + if (self.mmproj_path is None) == (self.mmproj_from_pretrained is None): + raise ValueError( + "exactly one of model.mtmd.mmproj_path or " + "model.mtmd.mmproj_from_pretrained is required" + ) + return self + + def resolve_mmproj_path(self) -> str: + if self.mmproj_from_pretrained is not None: + return self.mmproj_from_pretrained.resolve_model_path() + assert self.mmproj_path is not None + return self.mmproj_path + + class ModelOptions(BaseModel): + path: Optional[str] = None + alias: Optional[str] = None + chat_template: Optional[str] = None + from_pretrained: Optional["ConfigFile.FromPretrainedOptions"] = None + loras: List["ConfigFile.LoraOptions"] = Field(default_factory=list) + mtmd: Optional["ConfigFile.MTMDOptions"] = None + n_gpu_layers: Optional[int] = None + split_mode: Optional[int] = None + main_gpu: Optional[int] = None + tensor_split: Optional[List[float]] = None + vocab_only: Optional[bool] = None + use_mmap: Optional[bool] = None + use_mlock: Optional[bool] = None + kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None + n_ctx: Optional[int] = None + n_batch: Optional[int] = None + n_ubatch: Optional[int] = None + n_seq_max: Optional[int] = None + threads: Optional[int] = Field( + default_factory=lambda: max(multiprocessing.cpu_count(), 1) + ) + threads_batch: Optional[int] = Field( + default_factory=lambda: max(multiprocessing.cpu_count(), 1) + ) + rope_scaling_type: Optional[int] = None + pooling_type: Optional[int] = None + attention_type: Optional[int] = None + rope_freq_base: Optional[float] = None + rope_freq_scale: Optional[float] = None + yarn_ext_factor: Optional[float] = None + yarn_attn_factor: Optional[float] = None + yarn_beta_fast: Optional[float] = None + yarn_beta_slow: Optional[float] = None + yarn_orig_ctx: Optional[int] = None + offload_kqv: Optional[bool] = None + flash_attn: Optional[bool] = None + op_offload: Optional[bool] = None + swa_full: Optional[bool] = None + no_perf: Optional[bool] = None + type_k: Optional[int] = None + type_v: Optional[int] = None + max_seq_len: Optional[int] = None + max_output_tokens: Optional[int] = Field(default=None, ge=0) + kv_unified: bool = True + draft_model: Optional[Literal["prompt-lookup-decoding", "draft-mtp"]] = None + draft_model_num_pred_tokens: int = 16 + draft_model_max_ngram_size: int = 2 + draft_model_top_k: int = Field(default=1, ge=1) + draft_model_p_min: float = Field(default=0.0, ge=0.0, le=1.0) + draft_model_max_batch_size: Optional[int] = Field(default=None, ge=1) + draft_model_threads: Optional[int] = Field(default=None, gt=0) + draft_model_threads_batch: Optional[int] = Field(default=None, gt=0) + response_schema: Optional[Dict[str, Any]] = None + store_logits: bool = True + + @model_validator(mode="after") + def validate_source(self) -> "ConfigFile.ModelOptions": + if (self.path is None) == (self.from_pretrained is None): + raise ValueError("exactly one of model.path or model.from_pretrained is required") + return self + + @field_validator("chat_template", mode="before") + @classmethod + def normalize_chat_template(cls, value: Any) -> Any: + if isinstance(value, list): + if not all(isinstance(item, str) for item in value): + raise TypeError("model.chat_template list entries must be strings") + return "".join(value) + return value + + def resolve_model_path(self) -> str: + if self.from_pretrained is not None: + return self.from_pretrained.resolve_model_path() + assert self.path is not None + return self.path + + server: "ConfigFile.ServerOptions" = Field(default_factory=lambda: ConfigFile.ServerOptions()) + model: "ConfigFile.ModelOptions" + disk_cache: Optional["ConfigFile.DiskCacheOptions"] = None + + @classmethod + def load(cls, path: str) -> "ConfigFile": + return cls.model_validate_json(Path(path).read_text()) + + +ConfigFile.model_rebuild() + + +@dataclass(frozen=True) +class MediaInput: + kind: Literal["image", "audio"] + url: Optional[str] = None + data: Optional[str] = None + format: Optional[str] = None + + +class Jinja2ChatFormatter: + def __init__(self, template: str, *, bos_token: str, eos_token: str) -> None: + self._eos_token = eos_token + self._bos_token = bos_token + self._template_text = template + environment = ImmutableSandboxedEnvironment( + loader=jinja2.BaseLoader(), + trim_blocks=True, + lstrip_blocks=True, + ) + environment.filters["from_json"] = self._from_json + self._template = environment.from_string(template) + + @staticmethod + def _from_json(value: Any) -> Any: + if isinstance(value, str): + return json.loads(value) + return value + + @staticmethod + def media_inputs_from_messages( + messages: Sequence[ChatCompletionRequestMessage], + ) -> List[MediaInput]: + media_inputs: List[MediaInput] = [] + for message in messages: + content = message.content + if not isinstance(content, list): + continue + for part in content: + if not isinstance(part, dict): + continue + part_type = part.get("type") + if part_type in {"image_url", "input_image", "image"}: + image_url = part.get("image_url") or part.get("url") + if isinstance(image_url, str): + media_inputs.append(MediaInput(kind="image", url=image_url)) + elif isinstance(image_url, dict) and isinstance(image_url.get("url"), str): + media_inputs.append(MediaInput(kind="image", url=cast(str, image_url["url"]))) + else: + raise ValueError("image_url content part requires a URL string") + continue + if part_type == "audio_url": + audio_url = part.get("audio_url") + if isinstance(audio_url, str): + media_inputs.append(MediaInput(kind="audio", url=audio_url)) + elif isinstance(audio_url, dict) and isinstance(audio_url.get("url"), str): + media_inputs.append(MediaInput(kind="audio", url=cast(str, audio_url["url"]))) + else: + raise ValueError("audio_url content part requires a URL string") + continue + if part_type == "input_audio": + input_audio = part.get("input_audio") + if isinstance(input_audio, dict): + data = input_audio.get("data") + audio_format = input_audio.get("format") + else: + data = part.get("data") + audio_format = part.get("format") + if not isinstance(data, str): + raise ValueError("input_audio content part requires base64 data") + if audio_format is not None and not isinstance(audio_format, str): + raise ValueError("input_audio format must be a string") + media_inputs.append( + MediaInput( + kind="audio", + data=data, + format=cast(Optional[str], audio_format), + ) + ) + continue + return media_inputs + + @staticmethod + def _literal_content_for_template( + text: str, + media_marker: Optional[str], + ) -> str: + if media_marker is not None and media_marker in text: + raise ValueError("message content contains reserved multimodal marker text") + return text + + @staticmethod + def _content_part_for_template( + part: Any, + media_marker: Optional[str], + ) -> str: + if isinstance(part, str): + return Jinja2ChatFormatter._literal_content_for_template(part, media_marker) + if not isinstance(part, dict): + raise ValueError("content parts must be strings or objects") + part_type = part.get("type") + if part_type in {"image_url", "input_image", "image", "audio_url", "input_audio"}: + if media_marker is None: + raise ValueError("multimodal content requires model.mtmd") + return media_marker + if part_type in {"text", "input_text"}: + text = part.get("text") + if not isinstance(text, str): + raise ValueError(f"content part {part_type!r} requires string text") + return Jinja2ChatFormatter._literal_content_for_template(text, media_marker) + raise ValueError(f"unsupported content part type: {part_type!r}") + + @staticmethod + def _messages_for_template( + messages: Sequence[ChatCompletionRequestMessage], + media_marker: Optional[str], + ) -> List[Dict[str, Any]]: + converted: List[Dict[str, Any]] = [] + for message in messages: + data = message.model_dump(exclude_none=True) + content = data.get("content") + if isinstance(content, list): + data["content"] = "".join( + Jinja2ChatFormatter._content_part_for_template( + part, + media_marker, + ) + for part in content + ) + elif isinstance(content, str): + data["content"] = Jinja2ChatFormatter._literal_content_for_template( + content, + media_marker, + ) + converted.append(data) + return converted + + def _render( + self, + *, + messages: List[ChatCompletionRequestMessage], + media_marker: Optional[str] = None, + functions: Optional[List[ChatTemplateFunctionDefinition]] = None, + function_call: Optional[Union[str, ChatTemplateFunctionCall]] = None, + tools: Optional[List[ChatTemplateTool]] = None, + tool_choice: Optional[Union[str, ChatTemplateToolChoice]] = None, + reasoning_effort: Optional[str] = None, + add_generation_prompt: bool, + strftime_now: Callable[[str], str], + ) -> str: + def raise_exception(message: str) -> None: + raise ValueError(message) + + return cast( + str, + self._template.render( + messages=self._messages_for_template(messages, media_marker), + eos_token=self._eos_token, + bos_token=self._bos_token, + raise_exception=raise_exception, + add_generation_prompt=add_generation_prompt, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + reasoning_effort=reasoning_effort, + strftime_now=strftime_now, + ), + ) + + def format( + self, + *, + messages: List[ChatCompletionRequestMessage], + media_marker: Optional[str] = None, + functions: Optional[List[ChatTemplateFunctionDefinition]] = None, + function_call: Optional[Union[str, ChatTemplateFunctionCall]] = None, + tools: Optional[List[ChatTemplateTool]] = None, + tool_choice: Optional[Union[str, ChatTemplateToolChoice]] = None, + reasoning_effort: Optional[str] = None, + ) -> Tuple[str, str, List[str]]: + render_time = datetime.now() + + def strftime_now(format_string: str) -> str: + return render_time.strftime(format_string) + + chat_template = self._render( + messages=messages, + media_marker=media_marker, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + reasoning_effort=reasoning_effort, + add_generation_prompt=False, + strftime_now=strftime_now, + ) + prompt = self._render( + messages=messages, + media_marker=media_marker, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + reasoning_effort=reasoning_effort, + add_generation_prompt=True, + strftime_now=strftime_now, + ) + generation_prompt = prompt[len(chat_template) :] if prompt.startswith(chat_template) else "" + stop = [self._eos_token] if self._eos_token else [] + return prompt, generation_prompt, stop + + +@dataclass +class Token: + token: int + text_bytes: bytes + token_logprob: Optional[float] + top_logprobs: Optional[Dict[str, float]] + + @classmethod + def from_token( + cls, + *, + model: Model, + prev_tokens: Sequence[int], + prev_text_bytes: Optional[Union[bytes, bytearray]] = None, + token: int, + ) -> "Token": + return cls( + token=token, + text_bytes=( + model.token_bytes_with_prev_bytes(prev_tokens, prev_text_bytes, token) + if prev_text_bytes is not None + else model.token_bytes_with_prev(prev_tokens, token) + ), + token_logprob=None, + top_logprobs=None, + ) + + @classmethod + def from_logits( + cls, + *, + model: Model, + formatter: OpenAIFormatter, + prev_tokens: Sequence[int], + prev_text_bytes: Optional[Union[bytes, bytearray]] = None, + token: int, + logits: np.ndarray, + logprobs_count: Optional[int], + need_token_logprob: bool = False, + ) -> "Token": + text_bytes = ( + model.token_bytes_with_prev_bytes(prev_tokens, prev_text_bytes, token) + if prev_text_bytes is not None + else model.token_bytes_with_prev(prev_tokens, token) + ) + if not model.store_logits: + return cls( + token=token, + text_bytes=text_bytes, + token_logprob=None, + top_logprobs=None, + ) + if logprobs_count is None and not need_token_logprob: + return cls( + token=token, + text_bytes=text_bytes, + token_logprob=None, + top_logprobs=None, + ) + logprobs = CompletionScheduler.logits_to_logprobs(logits) + token_logprob = float(logprobs[token]) + top_logprobs: Optional[Dict[str, float]] = None + if logprobs_count is not None: + top_count = min(max(logprobs_count, 0), model.n_vocab) + if top_count > 0: + top_indices = np.argpartition(logprobs, -top_count)[-top_count:] + top_indices = top_indices[np.argsort(logprobs[top_indices])[::-1]] + top_logprobs = { + formatter.decode_text(model.token_bytes(int(index))): float( + logprobs[int(index)] + ) + for index in top_indices + } + else: + top_logprobs = {} + top_logprobs[formatter.decode_text(text_bytes)] = token_logprob + return cls( + token=token, + text_bytes=text_bytes, + token_logprob=token_logprob, + top_logprobs=top_logprobs, + ) + + +@dataclass +class Completion: + request_id: str + index: int + seq_id: int + sampler: "Sampler" + prompt_tokens: List[int] + prompt_length: int + prompt_text: str + multimodal_prompt: bool + max_total_tokens: int + stop_sequences: List[bytes] + logprobs: Optional[int] + completion_tokens: List[int] = field(default_factory=list) + token_records: List[Token] = field(default_factory=list) + rendered_bytes: bytearray = field(default_factory=bytearray) + detokenized_prefix_bytes: bytearray = field(default_factory=bytearray) + pending_input_tokens: List[int] = field(default_factory=list) + draft_tokens: List[int] = field(default_factory=list) + pending_finish_reason: Optional[str] = None + returned_token_count: int = 0 + finished: bool = False + finish_reason: Optional[str] = None + score_sum: float = 0.0 + rank_by_score: bool = False + + @property + def total_tokens(self) -> int: + return self.prompt_length + len(self.completion_tokens) + + @property + def completion_token_count(self) -> int: + return len(self.completion_tokens) + + @property + def needs_token_logprob(self) -> bool: + return self.logprobs is not None or self.rank_by_score + + @property + def max_stop_sequence_length(self) -> int: + return max((len(stop) for stop in self.stop_sequences), default=0) + + +@dataclass +class PromptSegment: + @dataclass + class Media: + embeddings: np.ndarray + positions: np.ndarray + non_causal: bool = False + + kind: Literal["text", "image", "audio"] + start_pos: int + n_pos: int + identity_tokens: List[int] + decode_start_pos: int + decode_n_pos: int + text_tokens: List[int] = field(default_factory=list) + media: Optional["PromptSegment.Media"] = None + + @property + def end_pos(self) -> int: + return self.start_pos + self.n_pos + + @property + def batch_rows(self) -> int: + if self.kind != "text": + if self.media is None: + return 0 + return int(self.media.embeddings.shape[0]) + return len(self.text_tokens) + + @property + def decoder_position_increments(self) -> List[int]: + if not self.identity_tokens: + return [] + if self.kind == "text": + return [1] * len(self.identity_tokens) + return [*([0] * (len(self.identity_tokens) - 1)), self.decode_n_pos] + + def rows_for_capacity(self, offset: int, capacity: int) -> int: + if capacity <= 0: + return 0 + return min(capacity, self.end_pos - self.start_pos - offset) + + def media_slice( + self, + row_offset: int, + row_count: int, + ) -> Tuple[np.ndarray, np.ndarray, List[int]]: + if self.media is None: + raise RuntimeError("media segment is missing embeddings or positions") + row_start = row_offset + row_end = row_offset + row_count + embeddings = self.media.embeddings[row_start:row_end] + if len(self.media.positions) == self.batch_rows: + positions = self.media.positions[row_start:row_end] + else: + positions = ( + self.media.positions.reshape(4, self.batch_rows)[:, row_start:row_end] + .reshape(-1) + ) + return ( + embeddings, + positions, + self.decoder_position_increments[row_start:row_end], + ) + + +@dataclass +class PromptPlan: + text: str + generation_prompt: str + text_tokens: List[int] + identity_tokens: List[int] + segments: List[PromptSegment] + text_token_index_by_pos: Dict[int, int] = field(default_factory=dict) + + @classmethod + def from_tokens( + cls, + text: str, + tokens: List[int], + *, + generation_prompt: str = "", + ) -> "PromptPlan": + segment = PromptSegment( + kind="text", + start_pos=0, + n_pos=len(tokens), + identity_tokens=list(tokens), + decode_start_pos=0, + decode_n_pos=len(tokens), + text_tokens=list(tokens), + ) + return cls( + text=text, + generation_prompt=generation_prompt, + text_tokens=list(tokens), + identity_tokens=list(tokens), + segments=[segment] if tokens else [], + text_token_index_by_pos={pos: pos for pos in range(len(tokens))}, + ) + + @property + def length(self) -> int: + return len(self.identity_tokens) + + @property + def eval_token_count(self) -> int: + return self.length + + def position_increments_up_to(self, pos: int) -> List[int]: + increments: List[int] = [] + for segment in self.segments: + if pos <= segment.start_pos: + break + take = min(pos, segment.end_pos) - segment.start_pos + if take <= 0: + continue + increments.extend(segment.decoder_position_increments[:take]) + if pos <= segment.end_pos: + break + return increments + + def is_boundary(self, pos: int) -> bool: + if pos <= 0 or pos >= self.length: + return True + return any(segment.start_pos == pos or segment.end_pos == pos for segment in self.segments) + + def is_reusable_boundary(self, pos: int) -> bool: + if pos <= 0 or pos >= self.length: + return True + if self.is_boundary(pos): + return True + return self.segment_at(pos).kind == "text" + + def clamp_to_reusable_boundary(self, pos: int) -> int: + if pos <= 0: + return 0 + if pos >= self.length: + return self.length + if self.is_reusable_boundary(pos): + return pos + segment = self.segment_at(pos) + return segment.start_pos + + def decoder_pos_up_to(self, pos: int) -> int: + if not self.is_reusable_boundary(pos): + raise RuntimeError("decoder position requested inside a non-reusable media segment") + if pos <= 0: + return 0 + if pos >= self.length: + if not self.segments: + return 0 + segment = self.segments[-1] + return segment.decode_start_pos + segment.decode_n_pos + segment = self.segment_at(pos) + if segment.kind == "text": + return segment.decode_start_pos + (pos - segment.start_pos) + if pos == segment.start_pos: + return segment.decode_start_pos + if pos == segment.end_pos: + return segment.decode_start_pos + segment.decode_n_pos + raise RuntimeError("decoder position requested inside a media segment") + + def segment_at(self, pos: int) -> PromptSegment: + for segment in self.segments: + if segment.start_pos <= pos < segment.end_pos: + return segment + raise RuntimeError(f"missing prompt segment at position {pos}") + + def has_text_token_at(self, pos: int) -> bool: + return pos in self.text_token_index_by_pos + + def text_token_at(self, pos: int) -> int: + return self.text_tokens[self.text_token_index_by_pos[pos]] + + def prev_text_tokens_at(self, pos: int) -> List[int]: + return self.text_tokens[: self.text_token_index_by_pos[pos]] + + +@dataclass(frozen=True) +class PreparedCompletionParts: + payload: CreateCompletionRequest + prompt_text: str + generation_prompt: str + prompt_plan: PromptPlan + grammar_text: Optional[str] + tool_name: Optional[str] + + +@dataclass(frozen=True) +class ResponsesChatRequestParts: + messages: List[ChatCompletionRequestMessage] + max_tokens: Optional[int] + temperature: float + top_p: float + stream: bool + logprobs: bool + top_logprobs: Optional[int] + model: Optional[str] + user: Optional[str] + tools: Optional[List[ChatTemplateTool]] + tool_choice: Optional[Union[Literal["auto", "none", "required"], ChatTemplateToolChoice]] + response_format: Optional[ChatTemplateResponseFormat] + reasoning_effort: Optional[str] + + +@dataclass +class SchedulerMetrics: + started_at: float = field(default_factory=time.time) + requests_submitted_total: int = 0 + requests_admitted_total: int = 0 + requests_completed_total: int = 0 + requests_cancelled_total: int = 0 + requests_failed_total: int = 0 + prompt_tokens_total: int = 0 + prompt_seconds_total: float = 0.0 + tokens_predicted_total: int = 0 + tokens_predicted_seconds_total: float = 0.0 + scheduler_step_seconds_total: float = 0.0 + process_batch_seconds_total: float = 0.0 + sample_seconds_total: float = 0.0 + draft_seconds_total: float = 0.0 + draft_process_seconds_total: float = 0.0 + draft_generate_seconds_total: float = 0.0 + draft_sampled_batch_seconds_total: float = 0.0 + draft_process_calls_total: int = 0 + draft_generate_calls_total: int = 0 + draft_sampled_batch_calls_total: int = 0 + draft_batches_verified_total: int = 0 + draft_target_tokens_verified_total: int = 0 + draft_target_tokens_wasted_total: int = 0 + draft_tokens_reused_as_pending_total: int = 0 + n_decode_total: int = 0 + scheduler_step_calls_total: int = 0 + process_batch_calls_total: int = 0 + sample_calls_total: int = 0 + n_tokens_max: int = 0 + n_busy_slots_total: int = 0 + checkpoint_hits_total: int = 0 + checkpoint_saves_total: int = 0 + checkpoint_evictions_total: int = 0 + sequence_cache_hits_total: int = 0 + sequence_cache_save_requests_total: int = 0 + sequence_cache_lookup_failures_total: int = 0 + sequence_cache_load_failures_total: int = 0 + sequence_cache_save_failures_total: int = 0 + sequence_cache_tokens_loaded_total: int = 0 + + def observe_decode( + self, + items: Sequence[Any], + elapsed_seconds: float, + ) -> None: + if not items: + return + total_tokens = sum( + item.batch_token_count + for item in items + ) + if total_tokens <= 0: + return + prompt_tokens = sum( + item.batch_token_count + for item in items + if getattr(item, "kind", None) == "prefill" + ) + generation_tokens = total_tokens - prompt_tokens + self.n_decode_total += 1 + self.n_busy_slots_total += len(items) + self.n_tokens_max = max(self.n_tokens_max, total_tokens) + self.prompt_tokens_total += prompt_tokens + if prompt_tokens > 0: + self.prompt_seconds_total += elapsed_seconds * (prompt_tokens / total_tokens) + if generation_tokens > 0: + self.tokens_predicted_seconds_total += elapsed_seconds * ( + generation_tokens / total_tokens + ) + + def observe_predicted_token(self) -> None: + self.tokens_predicted_total += 1 + + +class SequenceCache(Protocol): + """Optional external sequence-state cache; storage and eviction stay outside the scheduler.""" + + @dataclass(frozen=True) + class Match: + tokens: Tuple[int, ...] + has_prompt_logits: bool = False + + @property + def length(self) -> int: + return len(self.tokens) + + @dataclass + class Load: + tokens: List[int] + state_bytes: np.ndarray + prompt_logits: Optional[np.ndarray] = None + + def lookup(self, tokens: Sequence[int]) -> Optional["SequenceCache.Match"]: + ... + + def load( + self, + match: "SequenceCache.Match", + ) -> Optional["SequenceCache.Load"]: + ... + + def save( + self, + tokens: Sequence[int], + state_bytes: np.ndarray, + prompt_logits: Optional[np.ndarray], + ) -> None: + ... + + +@dataclass +class CompletionRequest: + payload: CreateCompletionRequest + prompt_text: str + prompt_tokens: List[int] + prompt_plan: PromptPlan + effective_max_len: int + internal_completion_count: int + prompt_visible_start: int + id: str = field(default_factory=lambda: f"cmpl-{uuid.uuid4().hex}") + created: int = field(default_factory=lambda: int(time.time())) + prompt_cursor: int = 0 + match_sequence_id: int = -1 + match_length: int = 0 + sequence_cache_match: Optional[SequenceCache.Match] = None + sequence_cache_match_length: int = 0 + prompt_logits: Optional[np.ndarray] = None + base_seq_id: Optional[int] = None + sibling_seq_ids: List[int] = field(default_factory=list) + completion_seq_ids: List[int] = field(default_factory=list) + completions: List[Completion] = field(default_factory=list) + admitted: bool = False + prompt_done: bool = False + prompt_checkpoint_saved: bool = False + cancelled: bool = False + prompt_records: List[Token] = field(default_factory=list) + grammar_text: Optional[str] = None + grammar_root: str = "root" + chat_tool_name: Optional[str] = None + on_stream_chunk: Optional[Callable[[CompletionChunk], None]] = None + on_done: Optional[Callable[[OpenAICompletion], None]] = None + on_error: Optional[Callable[[BaseException], None]] = None + + @classmethod + def from_prepared( + cls, + *, + payload: CreateCompletionRequest, + prompt_text: str, + prompt_plan: PromptPlan, + max_seq_len: int, + max_output_tokens: Optional[int], + prompt_visible_start: int, + prompt_records: Optional[List[Token]] = None, + grammar_text: Optional[str] = None, + chat_tool_name: Optional[str] = None, + on_stream_chunk: Optional[Callable[[CompletionChunk], None]] = None, + on_done: Optional[Callable[[OpenAICompletion], None]] = None, + on_error: Optional[Callable[[BaseException], None]] = None, + ) -> "CompletionRequest": + prompt_tokens = list(prompt_plan.identity_tokens) + prompt_eval_tokens = prompt_plan.eval_token_count + if prompt_eval_tokens > max_seq_len: + raise CompletionRequestValidationError("prompt exceeds context window") + ctx_limit = prompt_plan.length + (max_seq_len - prompt_eval_tokens) + if max_output_tokens is not None: + ctx_limit = min(ctx_limit, prompt_plan.length + max_output_tokens) + if payload.max_tokens is None: + effective_max_len = ctx_limit + else: + effective_max_len = min(ctx_limit, prompt_plan.length + payload.max_tokens) + if effective_max_len < prompt_plan.length: + raise CompletionRequestValidationError("prompt exceeds context window") + internal_completion_count = payload.best_of if payload.best_of is not None else payload.n + request = cls( + payload=payload, + prompt_text=prompt_text, + prompt_tokens=prompt_tokens, + prompt_plan=prompt_plan, + effective_max_len=effective_max_len, + internal_completion_count=internal_completion_count, + prompt_visible_start=prompt_visible_start, + prompt_records=list(prompt_records or []), + grammar_text=grammar_text, + chat_tool_name=chat_tool_name, + on_stream_chunk=on_stream_chunk, + on_done=on_done, + on_error=on_error, + ) + return request + + def selected_completions(self) -> List[Completion]: + completions = list(self.completions) + best_of = self.payload.best_of if self.payload.best_of is not None else self.payload.n + if best_of > self.payload.n: + completions.sort( + key=lambda completion: ( + completion.score_sum / max(1, completion.completion_token_count), + completion.score_sum, + ), + reverse=True, + ) + return sorted(completions[: self.payload.n], key=lambda completion: completion.index) + + def capture_prompt_logprobs( + self, + *, + model: Model, + formatter: OpenAIFormatter, + output_indices: Sequence[Optional[int]], + output_positions: Sequence[int], + output_count: int, + output_index_to_logits_index: Callable[[Optional[int], int], Optional[int]], + ) -> None: + if self.payload.logprobs is None or not self.payload.echo: + return + for token_offset, output_index in enumerate(output_indices): + if output_index is None: + continue + next_pos = output_positions[token_offset] + 1 + if not self.prompt_plan.has_text_token_at(next_pos): + continue + text_token_index = self.prompt_plan.text_token_index_by_pos[next_pos] + if text_token_index < self.prompt_visible_start: + continue + logits_index = output_index_to_logits_index(output_index, output_count) + assert logits_index is not None + next_token = self.prompt_plan.text_token_at(next_pos) + record = Token.from_logits( + model=model, + formatter=formatter, + prev_tokens=self.prompt_plan.prev_text_tokens_at(next_pos), + token=next_token, + logits=model.logits(logits_index), + logprobs_count=self.payload.logprobs, + ) + expected_index = text_token_index - self.prompt_visible_start + if expected_index == len(self.prompt_records): + self.prompt_records.append(record) + elif expected_index < len(self.prompt_records): + self.prompt_records[expected_index] = record + else: + raise RuntimeError("prompt logprob order mismatch") + + +class ResponseParser: + @dataclass + class PartialJsonValue: + text: str + schema_type: Optional[str] = None + complete: bool = False + + @dataclass + class PartialJsonObject: + value: "OrderedDict[str, Any]" + complete: bool = False + + @dataclass + class DirectState: + deltas: bool + pending: str + mode: int + done: bool + saw_tool_calls: bool + tool_call_count: int + assistant_prefix: str + leading_capture_field: Optional[str] + leading_capture_start: str + leading_capture_end: str + leading_capture_strip_after: bool + leading_capture_implicit: bool + content_end_markers: Tuple[str, ...] + trim_before_iterator: bool + iterator_start: str + iterator_end: str + stop_markers: Tuple[str, ...] + function_start: str + function_name_end: str + function_end: str + parameter_start: str + parameter_name_end: str + parameter_end: str + + @dataclass + class ItemState: + pending: str + mode: int + tool_call_index: int + tool_name: str + parameter_count: int + visible_tool_call: Optional[Dict[str, Any]] + visible_function: Optional[Dict[str, Any]] + current_parameter: Optional[str] + current_parameter_value: str + current_schema_type: Optional[str] + current_parameter_schema: Dict[str, Any] + + @dataclass + class StreamState: + kind: str + pending: str + mode: str + current_item: Optional[Dict[str, Any]] + current_segment: Optional[Dict[str, Any]] + done: bool + saw_tool_calls: bool + parsed: Optional[Dict[str, Any]] = None + buffer: str = "" + tool_call_count: int = 0 + + _STREAM_PLAN_CACHE: Dict[int, Tuple[Any, Optional[Dict[str, Any]]]] = {} + _TOOL_SCHEMA_CACHE: Dict[int, Tuple[Any, Dict[str, Dict[str, Any]]]] = {} + __slots__ = ( + "_schema", + "_tools", + "_completion_id", + "_choice_index", + "_generation_prompt", + "_tool_schemas", + "_started", + "_text_parts", + "_message", + "_stream_plan", + "_direct", + "_item", + "_stream_state", + "_stream_failed", + ) + DIRECT_MODE_ASSISTANT_PREFIX = 0 + DIRECT_MODE_PRELUDE = 1 + DIRECT_MODE_LEADING_CAPTURE = 2 + DIRECT_MODE_CONTENT = 3 + DIRECT_MODE_TOOL_ITEM = 4 + DIRECT_MODE_AFTER_TOOL_ITEM = 5 + ITEM_MODE_FUNCTION_HEADER = 0 + ITEM_MODE_AFTER_FUNCTION_HEADER = 1 + ITEM_MODE_PARAMETER_NAME = 2 + ITEM_MODE_PARAMETER_VALUE = 3 + ITEM_MODE_AFTER_PARAMETER = 4 + ITEM_MODE_DONE = 5 + + def __init__( + self, + schema: Dict[str, Any], + *, + tools: Optional[List[ChatTemplateTool]] = None, + completion_id: str = "", + choice_index: int = 0, + generation_prompt: str = "", + ) -> None: + self._schema = schema + self._tools = tools + self._completion_id = completion_id + self._choice_index = choice_index + self._generation_prompt = generation_prompt + self._tool_schemas = self._cached_tool_schema_map(tools) + self._started = False + self._text_parts: List[str] = [] + self._message: Dict[str, Any] = {} + self._stream_plan = self._cached_stream_plan(schema) + self._direct = ResponseParser.DirectState( + deltas=bool(self._stream_plan is not None and self._stream_plan.get("direct_deltas")), + pending="", + mode=self.DIRECT_MODE_PRELUDE, + done=False, + saw_tool_calls=False, + tool_call_count=0, + assistant_prefix="", + leading_capture_field=None, + leading_capture_start="", + leading_capture_end="", + leading_capture_strip_after=False, + leading_capture_implicit=False, + content_end_markers=(), + trim_before_iterator=False, + iterator_start="", + iterator_end="", + stop_markers=(), + function_start="", + function_name_end="", + function_end="", + parameter_start="", + parameter_name_end="", + parameter_end="", + ) + self._item = ResponseParser.ItemState( + pending="", + mode=self.ITEM_MODE_FUNCTION_HEADER, + tool_call_index=0, + tool_name="", + parameter_count=0, + visible_tool_call=None, + visible_function=None, + current_parameter=None, + current_parameter_value="", + current_schema_type=None, + current_parameter_schema={}, + ) + if self._direct.deltas and self._stream_plan is not None: + direct_init = self._stream_plan["direct_init"] + ( + self._direct.assistant_prefix, + self._direct.leading_capture_field, + self._direct.leading_capture_start, + self._direct.leading_capture_end, + self._direct.leading_capture_strip_after, + self._direct.leading_capture_implicit, + self._direct.trim_before_iterator, + self._direct.content_end_markers, + self._direct.stop_markers, + self._direct.iterator_start, + self._direct.iterator_end, + self._direct.function_start, + self._direct.function_name_end, + self._direct.function_end, + self._direct.parameter_start, + self._direct.parameter_name_end, + self._direct.parameter_end, + ) = direct_init + self._direct.mode = ( + self.DIRECT_MODE_ASSISTANT_PREFIX + if self._direct.assistant_prefix + else self.DIRECT_MODE_PRELUDE + ) + self._stream_state = None + else: + self._stream_state = ( + self._new_stream_state(self._stream_plan) if self._stream_plan is not None else None + ) + self._stream_failed = False + if self._generation_prompt and self._stream_plan is not None: + success, _ = self._advance_stream_state(self._generation_prompt) + if not success: + self._stream_failed = True + + @property + def started(self) -> bool: + return self._started + + @staticmethod + def _regex_capture(text: str, pattern: str) -> Optional[Any]: + match = re.search(pattern, text, re.S) + if match is None: + return None + group_dict = match.groupdict() + if group_dict: + return {key: value for key, value in group_dict.items() if value is not None} + return match.group(1) if match.lastindex else match.group(0) + + @staticmethod + def _gemma4_tool_call_to_json(text: str) -> str: + strings: List[str] = [] + + def capture(match: re.Match[str]) -> str: + strings.append(match.group(1)) + return f"\x00{len(strings) - 1}\x00" + + text = re.sub(r'<\|"\|>(.*?)<\|"\|>', capture, text, flags=re.S) + text = re.sub(r"(?<=[{,])(\w+):", r'"\1":', text) + for index, value in enumerate(strings): + text = text.replace(f"\x00{index}\x00", json.dumps(value)) + return text + + @staticmethod + def _regex_literal_prefix(pattern: str) -> str: + literal: List[str] = [] + index = 0 + while index < len(pattern): + char = pattern[index] + if char == "\\": + index += 1 + if index >= len(pattern): + break + escaped = pattern[index] + if escaped == "n": + literal.append("\n") + elif escaped == "t": + literal.append("\t") + else: + literal.append(escaped) + index += 1 + continue + if char in "[](){}.*+?|^$": + break + literal.append(char) + index += 1 + return "".join(literal) + + @staticmethod + def _literal_suffix_prefix_length(text: str, literal: str) -> int: + max_length = min(len(text), len(literal) - 1) + if max_length <= 0: + return 0 + tail = text[-max_length:] + if literal[0] not in tail: + return 0 + for prefix_length in range(max_length, 0, -1): + if text.endswith(literal[:prefix_length]): + return prefix_length + return 0 + + @classmethod + def _consume_until_literal( + cls, + text: str, + literal: str, + ) -> Tuple[str, bool, str, str]: + if not literal: + return text, True, "", "" + literal_length = len(literal) + search_from = 0 + first_char = literal[0] + while True: + marker_index = text.find(first_char, search_from) + if marker_index < 0: + return text, False, "", "" + if text.startswith(literal, marker_index): + return text[:marker_index], True, text[marker_index + literal_length :], "" + suffix = text[marker_index:] + if literal.startswith(suffix): + return text[:marker_index], False, "", suffix + search_from = marker_index + 1 + + @staticmethod + def _compile_iterator_pattern(pattern: str) -> Optional[Tuple[str, str]]: + if "(.*?)" not in pattern: + return None + prefix_pattern, suffix_pattern = pattern.split("(.*?)", 1) + prefix = ResponseParser._regex_literal_prefix(prefix_pattern.lstrip("^")) + suffix = ResponseParser._regex_literal_prefix(suffix_pattern) + if not prefix: + return None + return prefix, suffix + + @classmethod + def _compile_iterator_block_pattern( + cls, + pattern: str, + ) -> Optional[Dict[str, Any]]: + first_group = pattern.find("(") + if first_group < 0: + return None + depth = 0 + last_group = -1 + escaped = False + for index in range(first_group, len(pattern)): + char = pattern[index] + if escaped: + escaped = False + continue + if char == "\\": + escaped = True + continue + if char == "(": + depth += 1 + continue + if char == ")": + depth -= 1 + if depth == 0: + last_group = index + break + if last_group < first_group: + return None + start = cls._regex_literal_prefix(pattern[:first_group].lstrip("^")) + if not start: + return None + suffix_pattern = pattern[last_group + 1 :] + if suffix_pattern == "": + return {"start": start, "end": "", "allow_eof": True} + eof_variant = suffix_pattern.endswith("|$)") and suffix_pattern.startswith("(?:") + if eof_variant: + suffix_pattern = suffix_pattern[3:-3] + end = cls._regex_literal_prefix(suffix_pattern) + if not end: + return None + return {"start": start, "end": end, "allow_eof": eof_variant} + + @classmethod + def _compile_capture_pattern( + cls, + pattern: str, + ) -> Optional[Dict[str, Any]]: + normalized = pattern.lstrip("^") + if "(.*?)" in normalized: + prefix_pattern, suffix_pattern = normalized.split("(.*?)", 1) + start = cls._regex_literal_prefix(prefix_pattern) + if not start: + return None + if suffix_pattern == "": + return {"start": start, "end": "", "allow_eof": True} + eof_variant = suffix_pattern.endswith("|$)") and suffix_pattern.startswith("(?:") + if eof_variant: + suffix_pattern = suffix_pattern[3:-3] + end = cls._regex_literal_prefix(suffix_pattern) + if not end: + return None + return {"start": start, "end": end, "allow_eof": eof_variant} + if "(.*)" in normalized: + prefix_pattern, suffix_pattern = normalized.split("(.*)", 1) + start = cls._regex_literal_prefix(prefix_pattern) + if not start: + return None + if suffix_pattern == "": + return {"start": start, "end": "", "allow_eof": True} + eof_variant = suffix_pattern.endswith("|$)") and suffix_pattern.startswith("(?:") + if eof_variant: + suffix_pattern = suffix_pattern[3:-3] + end = cls._regex_literal_prefix(suffix_pattern) + if not end: + return None + return {"start": start, "end": end, "allow_eof": eof_variant} + return None + + @classmethod + def _compile_word_capture_pattern( + cls, + pattern: str, + ) -> Optional[Dict[str, Any]]: + normalized = pattern.lstrip("^") + if r"(\w+)" not in normalized: + return None + prefix_pattern, suffix_pattern = normalized.split(r"(\w+)", 1) + start = cls._regex_literal_prefix(prefix_pattern) + if not start: + return None + end = cls._regex_literal_prefix(suffix_pattern) if suffix_pattern else "" + return { + "kind": "word", + "start": start, + "end": end, + } + + @classmethod + def _consume_until_any_literal( + cls, + text: str, + literals: Sequence[str], + ) -> Tuple[str, Optional[str], str, str]: + for literal in literals: + if literal and len(text) < len(literal) and literal.startswith(text): + return "", None, "", text + literal_first_chars = {literal[0] for literal in literals if literal} + if literal_first_chars and not any(char in text for char in literal_first_chars): + return text, None, "", "" + earliest_index: Optional[int] = None + earliest_literal: Optional[str] = None + for literal in literals: + marker_index = text.find(literal) + if marker_index < 0: + continue + if earliest_index is None or marker_index < earliest_index: + earliest_index = marker_index + earliest_literal = literal + if earliest_index is not None and earliest_literal is not None: + return ( + text[:earliest_index], + earliest_literal, + text[earliest_index + len(earliest_literal) :], + "", + ) + overlap = 0 + for literal in literals: + overlap = max(overlap, cls._literal_suffix_prefix_length(text, literal)) + if overlap: + return text[:-overlap], None, "", text[-overlap:] + return text, None, "", "" + + @classmethod + def _compile_tool_call_item_plan( + cls, + item_schema: Dict[str, Any], + ) -> Optional[Dict[str, Any]]: + item_properties = item_schema.get("properties") + if not isinstance(item_properties, dict): + return None + function_schema = item_properties.get("function") + if not isinstance(function_schema, dict): + return None + function_properties = function_schema.get("properties") + if not isinstance(function_properties, dict): + return None + name_schema = function_properties.get("name") + arguments_schema = function_properties.get("arguments") + if not isinstance(name_schema, dict) or not isinstance(arguments_schema, dict): + return None + name_regex = name_schema.get("x-regex") + argument_regex = arguments_schema.get("x-regex") + argument_key_value = arguments_schema.get("x-regex-key-value") + if ( + name_regex == r"^<function=([^>\n]+)>\n" + and argument_regex == r"^<function=[^>\n]+>\n(.*?)\n</function>$" + and argument_key_value + == r"<parameter=(?P<key>[^>\n]+)>\n(?P<value>.*?)\n</parameter>" + ): + return { + "kind": "tagged-parameters", + "schema": item_schema, + "function_start": "<function=", + "function_name_end": ">\n", + "function_end": "</function>", + "parameter_start": "<parameter=", + "parameter_name_end": ">\n", + "parameter_end": "\n</parameter>", + } + name_capture = ( + cls._compile_word_capture_pattern(name_regex) + if isinstance(name_regex, str) + else None + ) + arguments_capture = ( + cls._compile_capture_pattern(argument_regex) + if isinstance(argument_regex, str) + else None + ) + if ( + isinstance(name_capture, dict) + and isinstance(arguments_capture, dict) + and arguments_schema.get("x-parser") == "json" + ): + arguments_value_schema = { + key: value + for key, value in arguments_schema.items() + if key != "x-regex" + } + return { + "kind": "json-message", + "schema": item_schema, + "name_capture": name_capture, + "arguments_capture": arguments_capture, + "arguments_schema": arguments_value_schema, + } + return { + "kind": "buffered", + "schema": item_schema, + } + + @classmethod + def _compile_segment_message_plan( + cls, + schema: Dict[str, Any], + ) -> Optional[Dict[str, Any]]: + if schema.get("type") != "object": + return None + if isinstance(schema.get("x-regex"), str): + return None + properties = schema.get("properties") + if not isinstance(properties, dict): + return None + segments: List[Dict[str, Any]] = [] + field_segment_count = 0 + for field_name, value_schema in properties.items(): + if not isinstance(value_schema, dict): + continue + if field_name == "tool_calls": + iterator_pattern = value_schema.get("x-regex-iterator") + if not isinstance(iterator_pattern, str): + continue + iterator_capture = cls._compile_iterator_block_pattern(iterator_pattern) + if not isinstance(iterator_capture, dict) or not iterator_capture["start"]: + return None + items_schema = value_schema.get("items") + if not isinstance(items_schema, dict): + return None + item_plan = cls._compile_tool_call_item_plan(items_schema) + if item_plan is None: + return None + segments.append( + { + "kind": "iterator", + "field": field_name, + "start": iterator_capture["start"], + "end": iterator_capture["end"], + "allow_eof": iterator_capture["allow_eof"], + "item": item_plan, + } + ) + continue + field_regex = value_schema.get("x-regex") + if not isinstance(field_regex, str): + continue + capture = cls._compile_capture_pattern(field_regex) + if not isinstance(capture, dict) or not capture["start"]: + return None + segments.append( + { + "kind": "field", + "field": field_name, + "start": capture["start"], + "end": capture["end"], + "allow_eof": capture["allow_eof"], + } + ) + field_segment_count += 1 + if not segments or field_segment_count == 0: + return None + start_literals = tuple(segment["start"] for segment in segments) + if len(start_literals) != len(set(start_literals)): + return None + return { + "kind": "segment-message", + "segments": segments, + "segment_starts": start_literals, + "segments_by_start": {segment["start"]: segment for segment in segments}, + } + + @classmethod + def _compile_tagged_message_plan( + cls, + schema: Dict[str, Any], + ) -> Optional[Dict[str, Any]]: + if schema.get("type") != "object": + return None + properties = schema.get("properties") + if not isinstance(properties, dict): + return None + tool_calls_schema = properties.get("tool_calls") + if not isinstance(tool_calls_schema, dict): + return None + iterator_pattern = tool_calls_schema.get("x-regex-iterator") + if not isinstance(iterator_pattern, str): + return None + iterator = cls._compile_iterator_pattern(iterator_pattern) + if iterator is None: + return None + items_schema = tool_calls_schema.get("items") + if not isinstance(items_schema, dict): + return None + item_plan = cls._compile_tool_call_item_plan(items_schema) + if item_plan is None: + return None + content_schema = properties.get("content") + content_regex = ( + content_schema.get("x-regex") + if isinstance(content_schema, dict) + else None + ) + object_regex = schema.get("x-regex") if isinstance(schema.get("x-regex"), str) else None + assistant_prefix: Optional[str] = None + if isinstance(content_regex, str) and r"<\|im_start\|>assistant\n" in content_regex: + assistant_prefix = "<|im_start|>assistant\n" + leading_capture: Optional[Dict[str, Any]] = None + for field_name, value_schema in properties.items(): + if not isinstance(value_schema, dict): + continue + field_regex = value_schema.get("x-regex") + if not isinstance(field_regex, str): + continue + if "<think>\\n" in field_regex and "</think>" in field_regex: + leading_capture = { + "field": field_name, + "start": "<think>\n", + "end": "</think>", + "strip_after": True, + "implicit_at_start": "(?:<think>\\n)?" in field_regex, + } + break + if leading_capture is None and isinstance(object_regex, str): + if ( + "(?P<thinking>" in object_regex + and r"<\|channel\>thought\n" in object_regex + and r"\<channel\|\>" in object_regex + ): + leading_capture = { + "field": "thinking", + "start": "<|channel>thought\n", + "end": "<channel|>", + "strip_after": False, + "implicit_at_start": False, + } + end_markers: List[str] = [] + iterator_start, iterator_end = iterator + if "content" in properties: + end_markers.append(iterator_start) + if isinstance(content_regex, str) and r"<\|im_end\|>" in content_regex: + end_markers.append("<|im_end|>") + if isinstance(object_regex, str) and r"<turn\|>" in object_regex: + end_markers.append("<turn|>") + if not end_markers and iterator_start: + end_markers.append(iterator_start) + trim_before_iterator = ( + isinstance(content_regex, str) + and r"\s*<tool_call>\n" in content_regex + ) + end_marker_tuple = tuple(end_markers) + direct_deltas = item_plan["kind"] == "tagged-parameters" + direct_init = ( + ( + assistant_prefix or "", + leading_capture["field"] if leading_capture is not None else None, + leading_capture["start"] if leading_capture is not None else "", + leading_capture["end"] if leading_capture is not None else "", + bool(leading_capture.get("strip_after")) if leading_capture is not None else False, + bool(leading_capture.get("implicit_at_start")) if leading_capture is not None else False, + trim_before_iterator, + end_marker_tuple, + tuple(marker for marker in end_marker_tuple if marker != iterator_start), + iterator_start, + iterator_end, + item_plan["function_start"], + item_plan["function_name_end"], + item_plan["function_end"], + item_plan["parameter_start"], + item_plan["parameter_name_end"], + item_plan["parameter_end"], + ) + if direct_deltas + else None + ) + return { + "kind": "tagged-message", + "direct_deltas": direct_deltas, + "assistant_prefix": assistant_prefix, + "leading_capture": leading_capture, + "content_field": "content" if "content" in properties else None, + "content_end_markers": end_marker_tuple, + "trim_before_iterator": trim_before_iterator, + "stop_markers": tuple(marker for marker in end_marker_tuple if marker != iterator_start), + "direct_init": direct_init, + "iterator": { + "start": iterator_start, + "end": iterator_end, + "item": item_plan, + }, + } + + @classmethod + def _compile_stream_plan( + cls, + response_schema: Optional[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + if not isinstance(response_schema, dict): + return None + if response_schema.get("x-parser") == "json": + return {"kind": "json-root"} + segment_plan = cls._compile_segment_message_plan(response_schema) + if segment_plan is not None: + return segment_plan + return cls._compile_tagged_message_plan(response_schema) + + @classmethod + def _cached_stream_plan( + cls, + response_schema: Optional[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + if not isinstance(response_schema, dict): + return None + cache_key = id(response_schema) + cached = cls._STREAM_PLAN_CACHE.get(cache_key) + if cached is not None and cached[0] is response_schema: + return cached[1] + plan = cls._compile_stream_plan(response_schema) + cls._STREAM_PLAN_CACHE[cache_key] = (response_schema, plan) + return plan + + @staticmethod + def _tool_schema_map( + tools: Optional[List[ChatTemplateTool]], + ) -> Dict[str, Dict[str, Any]]: + if tools is None: + return {} + mapping: Dict[str, Dict[str, Any]] = {} + for tool in tools: + if tool.get("type") != "function": + continue + function = tool.get("function", {}) + name = function.get("name") + parameters = function.get("parameters") + if isinstance(name, str) and isinstance(parameters, dict): + mapping[name] = { + "parameters": parameters, + "content_type": function.get("content_type"), + } + return mapping + + @classmethod + def _cached_tool_schema_map( + cls, + tools: Optional[List[ChatTemplateTool]], + ) -> Dict[str, Dict[str, Any]]: + if tools is None: + return {} + cache_key = id(tools) + cached = cls._TOOL_SCHEMA_CACHE.get(cache_key) + if cached is not None and cached[0] is tools: + return cached[1] + mapping = cls._tool_schema_map(tools) + cls._TOOL_SCHEMA_CACHE[cache_key] = (tools, mapping) + return mapping + + def _parameter_schema_for_tool( + self, tool_name: str, parameter_name: str + ) -> Dict[str, Any]: + tool_schema = self._tool_schemas.get(tool_name) + if not isinstance(tool_schema, dict): + return {} + parameters = tool_schema.get("parameters") + if not isinstance(parameters, dict): + return {} + properties = parameters.get("properties") + if not isinstance(properties, dict): + return {} + parameter_schema = properties.get(parameter_name) + if not isinstance(parameter_schema, dict): + return {} + return parameter_schema + + def _tool_content_type(self, tool_name: str) -> Optional[str]: + tool_schema = self._tool_schemas.get(tool_name) + if not isinstance(tool_schema, dict): + return None + content_type = tool_schema.get("content_type") + if isinstance(content_type, str): + return content_type + return None + + def _has_text_tools(self) -> bool: + return any( + isinstance(tool_schema, dict) and tool_schema.get("content_type") == "text" + for tool_schema in self._tool_schemas.values() + ) + + @staticmethod + def _append_parsed_text(parsed: Dict[str, Any], key: str, text: str) -> None: + if not text: + return + existing = parsed.get(key) + if isinstance(existing, str): + parsed[key] = existing + text + else: + parsed[key] = text + + def _append_visible_text(self, key: str, text: str) -> None: + if not text: + return + existing = self._message.get(key) + if isinstance(existing, str): + self._message[key] = existing + text + else: + self._message[key] = text + + @staticmethod + def _advance_json_scanner( + item_state: Dict[str, Any], + text: str, + *, + schema_type: Optional[str], + ) -> bool: + started = cast(bool, item_state["json_started"]) + complete = cast(bool, item_state["json_complete"]) + depth = cast(int, item_state["json_depth"]) + in_string = cast(bool, item_state["json_in_string"]) + escaped = cast(bool, item_state["json_escaped"]) + for char in text: + if complete: + if not char.isspace(): + return False + continue + if not started: + if char.isspace(): + continue + if schema_type == "object" and char != "{": + return False + if schema_type == "array" and char != "[": + return False + if char == "{": + started = True + depth = 1 + continue + if char == "[": + started = True + depth = 1 + continue + return False + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + continue + if char in "{[": + depth += 1 + continue + if char in "}]": + depth -= 1 + if depth < 0: + return False + if depth == 0: + complete = True + continue + item_state["json_started"] = started + item_state["json_complete"] = complete + item_state["json_depth"] = depth + item_state["json_in_string"] = in_string + item_state["json_escaped"] = escaped + return True + + def _new_tool_call_state(self, item_plan: Dict[str, Any]) -> Dict[str, Any]: + if item_plan["kind"] == "buffered": + return {"kind": "buffered", "buffer": ""} + if item_plan["kind"] == "json-message": + return { + "kind": "json-message", + "pending": "", + "mode": "function-name", + "tool_call": { + "type": "function", + "function": { + "name": "", + "arguments": "", + }, + }, + "arguments_text": "", + "json_started": False, + "json_complete": False, + "json_depth": 0, + "json_in_string": False, + "json_escaped": False, + } + return { + "kind": "tagged-parameters", + "pending": "", + "mode": "function-header", + "tool_call": { + "type": "function", + "function": { + "name": "", + "arguments": ResponseParser.PartialJsonObject(OrderedDict(), complete=False), + }, + }, + "current_parameter": None, + "current_schema_type": None, + } + + def _new_stream_state(self, plan: Dict[str, Any]) -> ResponseParser.StreamState: + if plan["kind"] == "json-root": + return ResponseParser.StreamState( + kind="json-root", + pending="", + mode="prelude", + current_item=None, + current_segment=None, + done=False, + saw_tool_calls=False, + parsed={"role": "assistant"}, + buffer="", + ) + if plan.get("direct_deltas"): + return ResponseParser.StreamState( + kind="tagged-message", + pending="", + mode="assistant-prefix" if plan.get("assistant_prefix") else "prelude", + current_item=None, + current_segment=None, + done=False, + saw_tool_calls=False, + tool_call_count=0, + ) + if plan["kind"] == "segment-message": + return ResponseParser.StreamState( + kind="segment-message", + pending="", + mode="segment-start", + current_item=None, + current_segment=None, + done=False, + saw_tool_calls=False, + parsed={"role": "assistant"}, + ) + return ResponseParser.StreamState( + kind="tagged-message", + pending="", + mode="assistant-prefix" if plan.get("assistant_prefix") else "prelude", + current_item=None, + current_segment=None, + done=False, + saw_tool_calls=False, + parsed={"role": "assistant"}, + ) + + def _start_direct_tool_call(self, tool_call_index: int) -> None: + self._item.pending = "" + self._item.mode = self.ITEM_MODE_FUNCTION_HEADER + self._item.tool_call_index = tool_call_index + self._item.tool_name = "" + self._item.parameter_count = 0 + tool_calls = self._message.setdefault("tool_calls", []) + assert isinstance(tool_calls, list) + while len(tool_calls) <= tool_call_index: + tool_calls.append({"function": {"name": "", "arguments": ""}}) + visible_tool_call = tool_calls[tool_call_index] + assert isinstance(visible_tool_call, dict) + self._item.visible_tool_call = visible_tool_call + function = visible_tool_call.setdefault("function", {}) + assert isinstance(function, dict) + self._item.visible_function = function + if tool_call_index == 0: + self._message["function_call"] = self._item.visible_function + self._item.current_parameter = None + self._item.current_parameter_value = "" + self._item.current_schema_type = None + self._item.current_parameter_schema = {} + + def _advance_direct_tool_call_state(self, text: str) -> Tuple[bool, List[Dict[str, Any]]]: + deltas: List[Dict[str, Any]] = [] + mode = self._item.mode + tool_call_index = self._item.tool_call_index + tool_name = self._item.tool_name + parameter_count = self._item.parameter_count + visible_tool_call = self._item.visible_tool_call + visible_function = self._item.visible_function + current_parameter = self._item.current_parameter + current_parameter_value = self._item.current_parameter_value + current_schema_type = self._item.current_schema_type + current_parameter_schema = self._item.current_parameter_schema + buffer = self._item.pending + text + function_start = self._direct.function_start + function_name_end = self._direct.function_name_end + function_end = self._direct.function_end + parameter_start = self._direct.parameter_start + parameter_name_end = self._direct.parameter_name_end + parameter_end = self._direct.parameter_end + + while True: + if mode == self.ITEM_MODE_FUNCTION_HEADER: + if buffer.startswith(function_start): + buffer = buffer[len(function_start) :] + elif function_start.startswith(buffer): + self._item.pending = buffer + self._item.mode = mode + return True, deltas + else: + self._item.pending = "" + return False, deltas + delimiter_index = buffer.find(function_name_end) + if delimiter_index < 0: + self._item.pending = function_start + buffer + self._item.mode = mode + return True, deltas + function_name = buffer[:delimiter_index] + if not function_name: + self._item.pending = "" + return False, deltas + tool_name = function_name + assert visible_tool_call is not None + assert visible_function is not None + visible_function["name"] = function_name + visible_function["arguments"] = "{" + visible_tool_call["id"] = ( + f"call_{self._choice_index}_{function_name}_{self._completion_id}_{tool_call_index}" + ) + visible_tool_call["type"] = "function" + deltas.append( + { + "tool_calls": [ + { + "index": tool_call_index, + "id": ( + f"call_{self._choice_index}_{function_name}_" + f"{self._completion_id}_{tool_call_index}" + ), + "type": "function", + "function": {"name": function_name, "arguments": "{"}, + } + ] + } + ) + buffer = buffer[delimiter_index + len(function_name_end) :] + mode = self.ITEM_MODE_AFTER_FUNCTION_HEADER + continue + if mode == self.ITEM_MODE_AFTER_FUNCTION_HEADER: + if buffer.startswith("\n"): + buffer = buffer[1:] + continue + if buffer.startswith(parameter_start): + mode = self.ITEM_MODE_PARAMETER_NAME + continue + if buffer.startswith(function_end): + buffer = buffer[len(function_end) :] + assert visible_function is not None + visible_function["arguments"] += "}" + deltas.append( + { + "tool_calls": [ + { + "index": tool_call_index, + "function": {"arguments": "}"}, + } + ] + } + ) + mode = self.ITEM_MODE_DONE + continue + if parameter_start.startswith(buffer) or function_end.startswith(buffer): + self._item.pending = buffer + self._item.mode = mode + self._item.tool_name = tool_name + return True, deltas + if not buffer: + self._item.pending = "" + self._item.mode = mode + self._item.tool_name = tool_name + return True, deltas + self._item.pending = "" + return False, deltas + if mode == self.ITEM_MODE_PARAMETER_NAME: + if buffer.startswith(parameter_start): + buffer = buffer[len(parameter_start) :] + elif parameter_start.startswith(buffer): + self._item.pending = buffer + self._item.mode = mode + self._item.tool_name = tool_name + self._item.parameter_count = parameter_count + return True, deltas + else: + self._item.pending = "" + return False, deltas + delimiter_index = buffer.find(parameter_name_end) + if delimiter_index < 0: + self._item.pending = parameter_start + buffer + self._item.mode = mode + self._item.tool_name = tool_name + self._item.parameter_count = parameter_count + return True, deltas + parameter_name = buffer[:delimiter_index] + if not parameter_name: + self._item.pending = "" + return False, deltas + parameter_schema = self._parameter_schema_for_tool(tool_name, parameter_name) + schema_type = parameter_schema.get("type") if isinstance(parameter_schema, dict) else None + current_parameter = parameter_name + current_parameter_value = "" + current_schema_type = schema_type if isinstance(schema_type, str) else None + current_parameter_schema = parameter_schema + key_prefix = json.dumps(parameter_name, ensure_ascii=False, separators=(",", ":")) + ":" + if parameter_count > 0: + key_prefix = "," + key_prefix + if schema_type in {None, "string"}: + key_prefix += '"' + parameter_count += 1 + assert visible_function is not None + visible_function["arguments"] += key_prefix + deltas.append( + { + "tool_calls": [ + { + "index": tool_call_index, + "function": {"arguments": key_prefix}, + } + ] + } + ) + buffer = buffer[delimiter_index + len(parameter_name_end) :] + mode = self.ITEM_MODE_PARAMETER_VALUE + continue + if mode == self.ITEM_MODE_PARAMETER_VALUE: + value_delta, matched, remainder, pending = self._consume_until_literal(buffer, parameter_end) + current_parameter_value += value_delta + if value_delta: + assert visible_function is not None + visible_function["arguments"] += value_delta + deltas.append( + { + "tool_calls": [ + { + "index": tool_call_index, + "function": {"arguments": value_delta}, + } + ] + } + ) + if not matched: + self._item.pending = pending + self._item.mode = mode + self._item.tool_name = tool_name + self._item.parameter_count = parameter_count + self._item.current_parameter = current_parameter + self._item.current_parameter_value = current_parameter_value + self._item.current_schema_type = current_schema_type + self._item.current_parameter_schema = current_parameter_schema + return True, deltas + self._coerce_tool_argument( + cast(str, current_parameter_value), + cast(Dict[str, Any], current_parameter_schema), + tool_name=tool_name, + argument_name=cast(str, current_parameter), + ) + if current_schema_type in {None, "string"}: + assert visible_function is not None + visible_function["arguments"] += '"' + deltas.append( + { + "tool_calls": [ + { + "index": tool_call_index, + "function": {"arguments": '"'}, + } + ] + } + ) + current_parameter = None + current_parameter_value = "" + current_schema_type = None + current_parameter_schema = {} + buffer = remainder + mode = self.ITEM_MODE_AFTER_PARAMETER + continue + if mode == self.ITEM_MODE_AFTER_PARAMETER: + if buffer.startswith("\n"): + buffer = buffer[1:] + continue + if buffer.startswith(parameter_start): + mode = self.ITEM_MODE_PARAMETER_NAME + continue + if buffer.startswith(function_end): + buffer = buffer[len(function_end) :] + assert visible_function is not None + visible_function["arguments"] += "}" + deltas.append( + { + "tool_calls": [ + { + "index": tool_call_index, + "function": {"arguments": "}"}, + } + ] + } + ) + mode = self.ITEM_MODE_DONE + continue + if parameter_start.startswith(buffer) or function_end.startswith(buffer): + self._item.pending = buffer + self._item.mode = mode + self._item.tool_name = tool_name + self._item.parameter_count = parameter_count + return True, deltas + if not buffer: + self._item.pending = "" + self._item.mode = mode + self._item.tool_name = tool_name + self._item.parameter_count = parameter_count + return True, deltas + self._item.pending = "" + return False, deltas + if mode == self.ITEM_MODE_DONE: + if buffer.strip(): + self._item.pending = "" + return False, deltas + self._item.pending = buffer + self._item.mode = mode + self._item.tool_name = tool_name + self._item.parameter_count = parameter_count + return True, deltas + + def _advance_direct_stream_state(self, text: str) -> Tuple[bool, List[Dict[str, Any]]]: + deltas: List[Dict[str, Any]] = [] + mode = self._direct.mode + tool_call_count = self._direct.tool_call_count + saw_tool_calls = self._direct.saw_tool_calls + done = self._direct.done + buffer = self._direct.pending + text + assistant_prefix = self._direct.assistant_prefix + leading_capture_field = self._direct.leading_capture_field + leading_capture_start = self._direct.leading_capture_start + leading_capture_end = self._direct.leading_capture_end + leading_capture_strip_after = self._direct.leading_capture_strip_after + leading_capture_implicit = self._direct.leading_capture_implicit + iterator_start = self._direct.iterator_start + iterator_end = self._direct.iterator_end + content_end_markers = self._direct.content_end_markers + stop_markers = self._direct.stop_markers + + while True: + if mode == self.DIRECT_MODE_ASSISTANT_PREFIX: + if buffer.startswith(assistant_prefix): + buffer = buffer[len(assistant_prefix) :] + mode = self.DIRECT_MODE_PRELUDE + continue + if assistant_prefix.startswith(buffer): + self._direct.pending = buffer + self._direct.mode = mode + return True, deltas + mode = self.DIRECT_MODE_PRELUDE + continue + if mode == self.DIRECT_MODE_PRELUDE: + if leading_capture_field is not None: + if buffer.startswith(leading_capture_start): + buffer = buffer[len(leading_capture_start) :] + mode = self.DIRECT_MODE_LEADING_CAPTURE + continue + if leading_capture_start.startswith(buffer): + self._direct.pending = buffer + self._direct.mode = mode + return True, deltas + if leading_capture_implicit: + tool_index = buffer.find(iterator_start) + end_index = buffer.find(leading_capture_end) + overlap = self._literal_suffix_prefix_length(buffer, leading_capture_end) + if ((end_index >= 0 and (tool_index < 0 or end_index < tool_index)) or overlap): + mode = self.DIRECT_MODE_LEADING_CAPTURE + continue + mode = self.DIRECT_MODE_CONTENT + continue + if mode == self.DIRECT_MODE_LEADING_CAPTURE: + segment, matched, remainder, pending = self._consume_until_literal(buffer, leading_capture_end) + if segment: + assert leading_capture_field is not None + self._append_visible_text(leading_capture_field, segment) + deltas.append({leading_capture_field: segment}) + if not matched: + self._direct.pending = pending + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return True, deltas + buffer = remainder.lstrip() if leading_capture_strip_after else remainder + mode = self.DIRECT_MODE_CONTENT + continue + if mode == self.DIRECT_MODE_CONTENT: + segment, matched_marker, remainder, pending = self._consume_until_any_literal( + buffer, + content_end_markers, + ) + content_segment = segment.rstrip() if matched_marker == iterator_start and self._direct.trim_before_iterator else segment + if content_segment: + self._append_visible_text("content", content_segment) + deltas.append({"content": content_segment}) + if matched_marker is None: + self._direct.pending = pending + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return True, deltas + if matched_marker == iterator_start: + saw_tool_calls = True + self._start_direct_tool_call(tool_call_count) + tool_call_count += 1 + mode = self.DIRECT_MODE_TOOL_ITEM + buffer = remainder + continue + done = True + self._direct.pending = "" + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return True, deltas + if mode == self.DIRECT_MODE_TOOL_ITEM: + item_text, matched, remainder, pending = self._consume_until_literal(buffer, iterator_end) + if item_text: + success, item_deltas = self._advance_direct_tool_call_state(item_text) + if not success: + self._direct.pending = "" + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return False, deltas + deltas.extend(item_deltas) + if not matched: + self._direct.pending = pending + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return True, deltas + buffer = remainder.lstrip() + mode = self.DIRECT_MODE_AFTER_TOOL_ITEM + continue + if mode == self.DIRECT_MODE_AFTER_TOOL_ITEM: + if not buffer: + self._direct.pending = "" + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return True, deltas + if buffer.startswith(iterator_start): + saw_tool_calls = True + self._start_direct_tool_call(tool_call_count) + tool_call_count += 1 + mode = self.DIRECT_MODE_TOOL_ITEM + buffer = buffer[len(iterator_start) :] + continue + for marker in stop_markers: + if buffer.startswith(marker): + done = True + self._direct.pending = "" + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return True, deltas + if any(marker.startswith(buffer) for marker in content_end_markers) or buffer.isspace(): + self._direct.pending = buffer + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return True, deltas + self._direct.pending = "" + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return False, deltas + + def _advance_tool_call_state( + self, + item_state: Dict[str, Any], + text: str, + item_plan: Dict[str, Any], + ) -> Tuple[bool, List[Dict[str, Any]]]: + deltas: List[Dict[str, Any]] = [] + if item_plan["kind"] == "buffered": + item_state["buffer"] = item_state["buffer"] + text + return True, deltas + if item_plan["kind"] == "json-message": + buffer = item_state["pending"] + text + item_state["pending"] = "" + while True: + mode = item_state["mode"] + if mode == "function-name": + name_capture = item_plan["name_capture"] + name_prefix = name_capture["start"] + if buffer.startswith(name_prefix): + remainder = buffer[len(name_prefix) :] + elif name_prefix.startswith(buffer): + item_state["pending"] = buffer + return True, deltas + else: + return False, deltas + name_end = 0 + while name_end < len(remainder) and ( + remainder[name_end].isalnum() or remainder[name_end] == "_" + ): + name_end += 1 + if name_end == 0: + if not remainder: + item_state["pending"] = buffer + return True, deltas + return False, deltas + if name_end == len(remainder): + item_state["pending"] = buffer + return True, deltas + function_name = remainder[:name_end] + item_state["tool_call"]["function"]["name"] = function_name + tool_call_index = cast(int, item_state["tool_call_index"]) + deltas.append( + { + "tool_calls": [ + { + "index": tool_call_index, + "id": ( + f"call_{self._choice_index}_{function_name}_" + f"{self._completion_id}_{tool_call_index}" + ), + "type": "function", + "function": {"name": function_name}, + } + ] + } + ) + buffer = remainder[name_end:] + item_state["mode"] = "seek-arguments" + continue + if mode == "seek-arguments": + arguments_capture = item_plan["arguments_capture"] + _, matched, remainder, pending = self._consume_until_literal( + buffer, + arguments_capture["start"], + ) + if not matched: + item_state["pending"] = pending + return True, deltas + buffer = remainder + item_state["mode"] = "arguments" + continue + if mode == "arguments": + if buffer: + item_state["arguments_text"] = item_state["arguments_text"] + buffer + deltas.append( + { + "tool_calls": [ + { + "index": cast(int, item_state["tool_call_index"]), + "function": {"arguments": buffer}, + } + ] + } + ) + item_state["pending"] = "" + arguments_schema = cast(Dict[str, Any], item_plan["arguments_schema"]) + schema_type = arguments_schema.get("type") + if not self._advance_json_scanner( + item_state, + buffer, + schema_type=schema_type if isinstance(schema_type, str) else None, + ): + return False, deltas + return True, deltas + + buffer = item_state["pending"] + text + item_state["pending"] = "" + while True: + mode = item_state["mode"] + if mode == "function-header": + function_start = item_plan["function_start"] + if buffer.startswith(function_start): + buffer = buffer[len(function_start) :] + elif function_start.startswith(buffer): + item_state["pending"] = buffer + return True, deltas + else: + return False, deltas + function_name_end = item_plan["function_name_end"] + delimiter_index = buffer.find(function_name_end) + if delimiter_index < 0: + item_state["pending"] = function_start + buffer + return True, deltas + function_name = buffer[:delimiter_index] + if not function_name: + return False, deltas + item_state["tool_call"]["function"]["name"] = function_name + deltas.append( + { + "tool_calls": [ + { + "index": item_state["tool_call_index"], + "id": ( + f"call_{self._choice_index}_{function_name}_" + f"{self._completion_id}_{item_state['tool_call_index']}" + ), + "type": "function", + "function": { + "name": function_name, + "arguments": "{", + }, + } + ] + } + ) + buffer = buffer[delimiter_index + len(function_name_end) :] + item_state["mode"] = "after-function-header" + continue + if mode == "after-function-header": + if buffer.startswith("\n"): + buffer = buffer[1:] + continue + function_end = item_plan["function_end"] + parameter_start = item_plan["parameter_start"] + if buffer.startswith(parameter_start): + item_state["mode"] = "parameter-name" + continue + if buffer.startswith(function_end): + buffer = buffer[len(function_end) :] + item_state["mode"] = "done" + deltas.append( + { + "tool_calls": [ + { + "index": item_state["tool_call_index"], + "function": {"arguments": "}"}, + } + ] + } + ) + continue + if parameter_start.startswith(buffer) or function_end.startswith(buffer): + item_state["pending"] = buffer + return True, deltas + if not buffer: + return True, deltas + return False, deltas + if mode == "parameter-name": + parameter_start = item_plan["parameter_start"] + if buffer.startswith(parameter_start): + buffer = buffer[len(parameter_start) :] + elif parameter_start.startswith(buffer): + item_state["pending"] = buffer + return True, deltas + else: + return False, deltas + parameter_name_end = item_plan["parameter_name_end"] + delimiter_index = buffer.find(parameter_name_end) + if delimiter_index < 0: + item_state["pending"] = parameter_start + buffer + return True, deltas + parameter_name = buffer[:delimiter_index] + if not parameter_name: + return False, deltas + function = item_state["tool_call"]["function"] + arguments = cast(ResponseParser.PartialJsonObject, function["arguments"]) + tool_name = function["name"] + parameter_schema = self._parameter_schema_for_tool(tool_name, parameter_name) + schema_type = ( + parameter_schema.get("type") + if isinstance(parameter_schema, dict) + else None + ) + arguments.value[parameter_name] = ResponseParser.PartialJsonValue( + text="", + schema_type=schema_type if isinstance(schema_type, str) else None, + complete=False, + ) + key_prefix = json.dumps( + parameter_name, + ensure_ascii=False, + separators=(",", ":"), + ) + ":" + if len(arguments.value) > 1: + key_prefix = "," + key_prefix + if schema_type in {None, "string"}: + key_prefix += '"' + deltas.append( + { + "tool_calls": [ + { + "index": item_state["tool_call_index"], + "function": {"arguments": key_prefix}, + } + ] + } + ) + item_state["current_parameter"] = parameter_name + item_state["current_schema_type"] = ( + schema_type if isinstance(schema_type, str) else None + ) + buffer = buffer[delimiter_index + len(parameter_name_end) :] + item_state["mode"] = "parameter-value" + continue + if mode == "parameter-value": + parameter_end = item_plan["parameter_end"] + value_delta, matched, remainder, pending = self._consume_until_literal( + buffer, + parameter_end, + ) + function = item_state["tool_call"]["function"] + arguments = cast(ResponseParser.PartialJsonObject, function["arguments"]) + parameter_name = cast(str, item_state["current_parameter"]) + current_value = arguments.value[parameter_name] + assert isinstance(current_value, ResponseParser.PartialJsonValue) + current_value.text = current_value.text + value_delta + if value_delta: + deltas.append( + { + "tool_calls": [ + { + "index": item_state["tool_call_index"], + "function": {"arguments": value_delta}, + } + ] + } + ) + if not matched: + item_state["pending"] = pending + return True, deltas + tool_name = function["name"] + parameter_schema = self._parameter_schema_for_tool(tool_name, parameter_name) + self._coerce_tool_argument( + current_value.text, + parameter_schema, + tool_name=tool_name, + argument_name=parameter_name, + ) + current_value.complete = True + if current_value.schema_type in {None, "string"}: + deltas.append( + { + "tool_calls": [ + { + "index": item_state["tool_call_index"], + "function": {"arguments": '"'}, + } + ] + } + ) + item_state["current_parameter"] = None + item_state["current_schema_type"] = None + buffer = remainder + item_state["mode"] = "after-parameter" + continue + if mode == "after-parameter": + if buffer.startswith("\n"): + buffer = buffer[1:] + continue + function_end = item_plan["function_end"] + parameter_start = item_plan["parameter_start"] + if buffer.startswith(parameter_start): + item_state["mode"] = "parameter-name" + continue + if buffer.startswith(function_end): + buffer = buffer[len(function_end) :] + cast( + ResponseParser.PartialJsonObject, + item_state["tool_call"]["function"]["arguments"], + ).complete = True + item_state["mode"] = "done" + deltas.append( + { + "tool_calls": [ + { + "index": item_state["tool_call_index"], + "function": {"arguments": "}"}, + } + ] + } + ) + continue + if parameter_start.startswith(buffer) or function_end.startswith(buffer): + item_state["pending"] = buffer + return True, deltas + if not buffer: + return True, deltas + return False, deltas + if mode == "done": + if buffer.strip(): + return False, deltas + item_state["pending"] = buffer + return True, deltas + + def _finish_tool_call_state( + self, + item_state: Dict[str, Any], + item_plan: Dict[str, Any], + ) -> Optional[Dict[str, Any]]: + if item_plan["kind"] == "buffered": + parsed_item = self._parse_response_value( + item_state["buffer"], + item_plan["schema"], + partial=False, + ) + return self._normalize_tool_call_item(parsed_item, partial=False) + if item_plan["kind"] == "json-message": + if item_state["mode"] != "arguments": + return None + if item_state["pending"] or not item_state["json_started"] or not item_state["json_complete"]: + return None + try: + arguments = self._parse_response_value( + item_state["arguments_text"], + item_plan["arguments_schema"], + partial=False, + ) + except CompletionResponseParsingError: + return None + if arguments is None: + return None + item_state["tool_call"]["function"]["arguments"] = arguments + return cast(Dict[str, Any], item_state["tool_call"]) + if item_state["pending"] or item_state["current_parameter"] is not None: + return None + if item_state["mode"] not in {"done", "after-function-header"}: + return None + return cast(Dict[str, Any], item_state["tool_call"]) + + def _tool_call_delta( + self, + *, + tool_call: Dict[str, Any], + tool_call_index: int, + partial: bool, + ) -> Dict[str, Any]: + function = cast(Dict[str, Any], tool_call["function"]) + return { + "tool_calls": [ + { + "index": tool_call_index, + "id": ( + f"call_{self._choice_index}_{function['name']}_" + f"{self._completion_id}_{tool_call_index}" + ), + "type": tool_call.get("type", "function"), + "function": { + "name": function["name"], + "arguments": self._serialize_tool_arguments( + function["arguments"], + partial=partial, + ), + }, + } + ] + } + + def _advance_stream_state(self, text: str) -> Tuple[bool, List[Dict[str, Any]]]: + deltas: List[Dict[str, Any]] = [] + if self._stream_plan is None: + return False, deltas + if self._direct.deltas: + return self._advance_direct_stream_state(text) + if self._stream_state is None: + return False, deltas + if self._stream_plan["kind"] == "segment-message": + state = self._stream_state + parsed = cast(Dict[str, Any], state.parsed) + buffer = state.pending + text + state.pending = "" + while True: + if state.mode == "segment-start": + _, matched_start, remainder, pending = self._consume_until_any_literal( + buffer, + cast(Tuple[str, ...], self._stream_plan["segment_starts"]), + ) + if matched_start is None: + state.pending = buffer if not pending else pending + return True, deltas + state.current_segment = cast( + Dict[str, Any], + self._stream_plan["segments_by_start"][matched_start], + ) + buffer = remainder + state.mode = ( + "segment-tool-item" + if state.current_segment["kind"] == "iterator" + else "segment-field" + ) + if state.current_segment["kind"] == "iterator": + item_state = self._new_tool_call_state(state.current_segment["item"]) + if item_state["kind"] in {"tagged-parameters", "json-message"}: + tool_calls = cast( + List[Dict[str, Any]], + parsed.setdefault("tool_calls", []), + ) + tool_calls.append(item_state["tool_call"]) + item_state["tool_call_index"] = len(tool_calls) - 1 + state.current_item = item_state + state.saw_tool_calls = True + continue + if state.mode == "segment-field": + current_segment = state.current_segment + if not isinstance(current_segment, dict): + return False, deltas + segment_text, matched, remainder, pending = self._consume_until_literal( + buffer, + cast(str, current_segment["end"]), + ) + field_name = cast(str, current_segment["field"]) + self._append_parsed_text(parsed, field_name, segment_text) + if segment_text: + deltas.append({field_name: segment_text}) + if not matched: + state.pending = pending + return True, deltas + buffer = remainder + state.current_segment = None + state.mode = "segment-start" + continue + if state.mode == "segment-tool-item": + current_segment = state.current_segment + if not isinstance(current_segment, dict): + return False, deltas + active_item_state = state.current_item + if not isinstance(active_item_state, dict): + return False, deltas + item_text, matched, remainder, pending = self._consume_until_literal( + buffer, + cast(str, current_segment["end"]), + ) + if item_text: + success, item_deltas = self._advance_tool_call_state( + active_item_state, + item_text, + cast(Dict[str, Any], current_segment["item"]), + ) + if not success: + return False, deltas + deltas.extend(item_deltas) + if not matched: + state.pending = pending + return True, deltas + tool_call = self._finish_tool_call_state( + active_item_state, + cast(Dict[str, Any], current_segment["item"]), + ) + if tool_call is None: + return False, deltas + if active_item_state["kind"] == "buffered": + tool_calls = cast(List[Dict[str, Any]], parsed.setdefault("tool_calls", [])) + tool_call_index = len(tool_calls) + tool_calls.append(tool_call) + deltas.append( + self._tool_call_delta( + tool_call=tool_call, + tool_call_index=tool_call_index, + partial=False, + ) + ) + state.current_item = None + state.current_segment = None + state.mode = "segment-start" + buffer = remainder + continue + return False, deltas + if self._stream_plan["kind"] == "json-root": + buffer = self._stream_state.buffer + text + try: + parsed = self.parse_chat_response(buffer, partial=True) + except CompletionResponseParsingError: + return False, deltas + self._stream_state.buffer = buffer + self._stream_state.parsed = parsed + return True, deltas + + state = self._stream_state + plan = self._stream_plan + buffer = state.pending + text + state.pending = "" + parsed = cast(Dict[str, Any], state.parsed) + iterator_start = plan["iterator"]["start"] + iterator_end = plan["iterator"]["end"] + while True: + mode = state.mode + if mode == "assistant-prefix": + assistant_prefix = plan.get("assistant_prefix") + if not assistant_prefix: + state.mode = "prelude" + continue + if buffer.startswith(assistant_prefix): + buffer = buffer[len(assistant_prefix) :] + state.mode = "prelude" + continue + if assistant_prefix.startswith(buffer): + state.pending = buffer + return True, deltas + state.mode = "prelude" + continue + if mode == "prelude": + leading_capture = plan.get("leading_capture") + if leading_capture is not None: + capture_start = leading_capture["start"] + if buffer.startswith(capture_start): + buffer = buffer[len(capture_start) :] + state.mode = "leading-capture" + continue + if capture_start.startswith(buffer): + state.pending = buffer + return True, deltas + if leading_capture.get("implicit_at_start"): + capture_end = leading_capture["end"] + tool_index = buffer.find(iterator_start) + end_index = buffer.find(capture_end) + overlap = self._literal_suffix_prefix_length(buffer, capture_end) + if ( + end_index >= 0 + and (tool_index < 0 or end_index < tool_index) + ) or overlap: + state.mode = "leading-capture" + continue + state.mode = "content" + continue + if mode == "leading-capture": + leading_capture = cast(Dict[str, Any], plan["leading_capture"]) + segment, matched, remainder, pending = self._consume_until_literal( + buffer, + leading_capture["end"], + ) + self._append_parsed_text(parsed, leading_capture["field"], segment) + if segment: + deltas.append({leading_capture["field"]: segment}) + if not matched: + state.pending = pending + return True, deltas + buffer = remainder.lstrip() if leading_capture.get("strip_after") else remainder + state.mode = "content" + continue + if mode == "content": + content_field = plan.get("content_field") + end_markers = cast(List[str], plan["content_end_markers"]) + segment, matched_marker, remainder, pending = self._consume_until_any_literal( + buffer, + end_markers, + ) + if content_field is not None: + content_segment = ( + segment.rstrip() + if matched_marker == iterator_start and plan.get("trim_before_iterator") + else segment + ) + self._append_parsed_text(parsed, content_field, content_segment) + if content_segment: + deltas.append({content_field: content_segment}) + if matched_marker is None: + state.pending = pending + return True, deltas + if matched_marker == iterator_start: + item_state = self._new_tool_call_state(plan["iterator"]["item"]) + state.saw_tool_calls = True + if item_state["kind"] == "tagged-parameters": + tool_calls = cast(List[Dict[str, Any]], parsed.setdefault("tool_calls", [])) + tool_calls.append(item_state["tool_call"]) + item_state["tool_call_index"] = len(tool_calls) - 1 + state.current_item = item_state + state.mode = "tool-item" + buffer = remainder + continue + state.done = True + state.pending = "" + return True, deltas + if mode == "tool-item": + current_item_state: Optional[Dict[str, Any]] = state.current_item + if not isinstance(current_item_state, dict): + return False, deltas + item_text, matched, remainder, pending = self._consume_until_literal( + buffer, + iterator_end, + ) + if item_text: + success, item_deltas = self._advance_tool_call_state( + current_item_state, + item_text, + plan["iterator"]["item"], + ) + if not success: + return False, deltas + deltas.extend(item_deltas) + if not matched: + state.pending = pending + return True, deltas + tool_call = self._finish_tool_call_state( + current_item_state, + plan["iterator"]["item"], + ) + if tool_call is None: + return False, deltas + if current_item_state["kind"] == "buffered": + parsed.setdefault("tool_calls", []).append(tool_call) + state.current_item = None + buffer = remainder.lstrip() + state.mode = "after-tool-item" + continue + if mode == "after-tool-item": + end_markers = [ + iterator_start, + *[ + marker + for marker in cast(List[str], plan["content_end_markers"]) + if marker != iterator_start + ], + ] + if not buffer: + state.pending = "" + return True, deltas + if buffer.startswith(iterator_start): + item_state = self._new_tool_call_state(plan["iterator"]["item"]) + state.saw_tool_calls = True + if item_state["kind"] == "tagged-parameters": + tool_calls = cast(List[Dict[str, Any]], parsed.setdefault("tool_calls", [])) + tool_calls.append(item_state["tool_call"]) + item_state["tool_call_index"] = len(tool_calls) - 1 + state.current_item = item_state + state.mode = "tool-item" + buffer = buffer[len(iterator_start) :] + continue + for marker in end_markers[1:]: + if buffer.startswith(marker): + state.done = True + state.pending = "" + return True, deltas + if any(marker.startswith(buffer) for marker in end_markers): + state.pending = buffer + return True, deltas + if buffer.isspace(): + state.pending = buffer + return True, deltas + return False, deltas + + @staticmethod + def _partial_regex_key_value_item( + pattern: str, + text: str, + *, + min_start: int, + ) -> Optional[Tuple[str, str]]: + value_group = "(?P<value>" + value_group_start = pattern.find(value_group) + if value_group_start < 0: + return None + partial_pattern = pattern[:value_group_start] + r"(?P<value>.*)\Z" + partial_match: Optional[re.Match[str]] = None + for match in re.finditer(partial_pattern, text, re.S): + if match.start() < min_start: + continue + partial_match = match + if partial_match is None: + return None + group_dict = partial_match.groupdict() + key = group_dict.get("key") + value = group_dict.get("value") + if key is None or value is None: + return None + value_pattern = "(?P<value>.*?)" + if value_pattern in pattern: + suffix_literal = ResponseParser._regex_literal_prefix( + pattern.split(value_pattern, 1)[1] + ) + for suffix_length in range(len(suffix_literal), 0, -1): + suffix_prefix = suffix_literal[:suffix_length] + if value.endswith(suffix_prefix): + value = value[:-suffix_length] + break + return key, value + + def _trim_partial_tool_call_prefix( + self, + *, + response_text: str, + parsed: Dict[str, Any], + ) -> None: + if not isinstance(parsed.get("content"), str): + return + tool_calls_schema = self._schema.get("properties", {}).get("tool_calls") + if not isinstance(tool_calls_schema, dict): + return + iterator_pattern = tool_calls_schema.get("x-regex-iterator") + if not isinstance(iterator_pattern, str) or "(.*?)" not in iterator_pattern: + return + prefix_pattern = iterator_pattern.split("(.*?)", 1)[0] + literal_prefix = self._regex_literal_prefix(prefix_pattern) + if not literal_prefix: + return + content = cast(str, parsed["content"]) + for prefix_length in range(len(literal_prefix) - 1, 0, -1): + prefix = literal_prefix[:prefix_length] + if content.endswith(prefix) and response_text.endswith(prefix): + trimmed = content[:-prefix_length] + parsed["content"] = trimmed if trimmed else None + break + + def _parse_response_value( + self, + text: Any, + schema: Dict[str, Any], + *, + partial: bool, + ) -> Any: + if "const" in schema: + return schema["const"] + if text is None: + return None + node_content: Any = text + node_regex = schema.get("x-regex") + if node_regex is not None: + if not isinstance(node_content, str): + raise CompletionResponseParsingError( + "response_schema x-regex requires string input" + ) + captured_content = self._regex_capture(node_content, node_regex) + if captured_content is None: + if ( + partial + and schema.get("type") == "object" + and "x-regex-key-value" in schema + ): + captured_content = node_content + else: + return None + node_content = captured_content + node_regex_iterator = schema.get("x-regex-iterator") + if node_regex_iterator is not None: + if schema.get("type") != "array": + raise CompletionResponseParsingError( + "response_schema x-regex-iterator requires array type" + ) + if not isinstance(node_content, str): + raise CompletionResponseParsingError( + "response_schema x-regex-iterator requires string input" + ) + array_values = [] + matches = list(re.finditer(node_regex_iterator, node_content, re.S)) + for match in matches: + item_text = self._regex_capture(match.group(0), node_regex_iterator) + if item_text is not None: + array_values.append(item_text) + if partial and "(.*?)" in node_regex_iterator: + prefix_pattern, suffix_pattern = node_regex_iterator.split("(.*?)", 1) + prefix_matches = list(re.finditer(prefix_pattern, node_content, re.S)) + if prefix_matches: + last_prefix_match = prefix_matches[-1] + if not matches or matches[-1].start() != last_prefix_match.start(): + tail = node_content[last_prefix_match.end() :] + if re.search(suffix_pattern, tail, re.S) is None: + array_values.append(tail) + if not array_values: + return None + node_content = array_values + node_regex_key_value = schema.get("x-regex-key-value") + if node_regex_key_value is not None: + if schema.get("type") != "object": + raise CompletionResponseParsingError( + "response_schema x-regex-key-value requires object type" + ) + if not isinstance(node_content, str): + raise CompletionResponseParsingError( + "response_schema x-regex-key-value requires string input" + ) + key_values: Dict[str, str] = {} + matches = list(re.finditer(node_regex_key_value, node_content, re.S)) + for match in matches: + group_dict = match.groupdict() + if "key" not in group_dict or "value" not in group_dict: + raise CompletionResponseParsingError( + "response_schema x-regex-key-value must define key and value groups" + ) + key = group_dict["key"] + value = group_dict["value"] + if key is None or value is None: + raise CompletionResponseParsingError( + "response_schema x-regex-key-value matched null key or value" + ) + key_values[key] = value + if partial: + min_start = matches[-1].end() if matches else 0 + partial_item = self._partial_regex_key_value_item( + node_regex_key_value, + node_content, + min_start=min_start, + ) + if partial_item is not None: + key_values[partial_item[0]] = partial_item[1] + if not key_values: + return None + node_content = key_values + parser_name = schema.get("x-parser") + if parser_name is not None: + if parser_name != "json": + if parser_name != "gemma4-tool-call": + raise CompletionResponseParsingError( + f"unsupported response_schema x-parser: {parser_name}" + ) + if not isinstance(node_content, str): + raise CompletionResponseParsingError( + "response_schema x-parser='gemma4-tool-call' requires string input" + ) + node_content = self._gemma4_tool_call_to_json(node_content) + parser_name = "json" + if parser_name == "json": + if not isinstance(node_content, str): + raise CompletionResponseParsingError( + "response_schema x-parser='json' requires string input" + ) + try: + parsed = from_json(node_content, allow_partial=partial) + except ValueError as exc: + if ( + self._has_text_tools() + and schema.get("type") == "object" + and schema.get("additionalProperties") is True + and not schema.get("properties") + ): + return node_content + if partial: + return None + raise CompletionResponseParsingError( + "response did not match response_schema JSON parser" + ) from exc + stripped_schema = { + key: value + for key, value in schema.items() + if key + not in { + "x-parser", + "x-parser-args", + "x-regex", + "x-regex-iterator", + "x-regex-key-value", + } + } + return self._parse_response_value( + parsed, stripped_schema, partial=partial + ) + schema_type = schema.get("type") + if schema_type == "string": + return node_content + if schema_type == "array": + if isinstance(node_content, list): + array_values = [] + item_schema = schema.get("items", {}) + for item in node_content: + parsed_item = self._parse_response_value( + item, + item_schema, + partial=partial, + ) + if parsed_item is not None: + array_values.append(parsed_item) + return array_values + return [] + if schema_type == "object": + properties = schema.get("properties", {}) + if isinstance(node_content, dict): + parsed_object: Dict[str, Any] = {} + for key, value_schema in properties.items(): + value = self._parse_response_value( + node_content.get(key), + value_schema, + partial=partial, + ) + if value is None: + continue + if isinstance(value, list) and not value: + continue + parsed_object[key] = value + additional_properties = schema.get("additionalProperties") + if additional_properties is True: + for key, value in node_content.items(): + if key not in parsed_object and key not in properties: + parsed_object[key] = value + elif isinstance(additional_properties, dict): + for key, value in node_content.items(): + if key in parsed_object or key in properties: + continue + parsed_value = self._parse_response_value( + value, + additional_properties, + partial=partial, + ) + if parsed_value is not None: + parsed_object[key] = parsed_value + if not partial: + missing = [ + key + for key in schema.get("required", []) + if key not in parsed_object + ] + if missing: + raise CompletionResponseParsingError( + f"response did not match response_schema: missing {', '.join(missing)}" + ) + return parsed_object + parsed_object_from_text: Dict[str, Any] = {} + for key, value_schema in properties.items(): + value = self._parse_response_value( + node_content, value_schema, partial=partial + ) + if value is None: + continue + if isinstance(value, list) and not value: + continue + parsed_object_from_text[key] = value + if not partial: + missing = [ + key + for key in schema.get("required", []) + if key not in parsed_object_from_text + ] + if missing: + raise CompletionResponseParsingError( + f"response did not match response_schema: missing {', '.join(missing)}" + ) + return parsed_object_from_text + if schema_type == "integer": + if isinstance(node_content, int) and not isinstance(node_content, bool): + return node_content + if partial and isinstance(node_content, str) and not node_content: + return None + try: + return int(node_content) + except (TypeError, ValueError): + return None + if schema_type == "number": + if isinstance(node_content, (int, float)) and not isinstance( + node_content, bool + ): + return node_content + if partial and isinstance(node_content, str) and not node_content: + return None + try: + return float(node_content) + except (TypeError, ValueError): + return None + if schema_type == "boolean": + if isinstance(node_content, bool): + return node_content + if node_content in {"true", "True", 1, "1"}: + return True + if node_content in {"false", "False", 0, "0"}: + return False + return None + one_of = schema.get("oneOf") + if isinstance(one_of, list): + for option in one_of: + value = self._parse_response_value( + node_content, option, partial=partial + ) + if value is not None: + return value + return None + if schema_type is None or schema_type == "any": + return node_content + return None + + def _coerce_tool_argument( + self, + raw_value: str, + schema: Dict[str, Any], + *, + tool_name: str, + argument_name: str, + ) -> Any: + if "oneOf" in schema: + last_error: Optional[BaseException] = None + for variant in schema["oneOf"]: + try: + return self._coerce_tool_argument( + raw_value, + variant, + tool_name=tool_name, + argument_name=argument_name, + ) + except BaseException as exc: + last_error = exc + raise CompletionResponseParsingError( + f"tool '{tool_name}' argument '{argument_name}' did not match any allowed schema" + ) from last_error + schema_type = schema.get("type") + if isinstance(schema_type, list): + last_type_error: Optional[BaseException] = None + for variant_type in schema_type: + try: + return self._coerce_tool_argument( + raw_value, + {**schema, "type": variant_type}, + tool_name=tool_name, + argument_name=argument_name, + ) + except BaseException as exc: + last_type_error = exc + raise CompletionResponseParsingError( + f"tool '{tool_name}' argument '{argument_name}' did not match any allowed type" + ) from last_type_error + if schema_type in {None, "string"}: + return raw_value + try: + decoded = json.loads(raw_value) + except json.JSONDecodeError as exc: + raise CompletionResponseParsingError( + f"tool '{tool_name}' argument '{argument_name}' is not valid JSON for type '{schema_type}'" + ) from exc + if schema_type == "integer": + if isinstance(decoded, bool) or not isinstance(decoded, int): + raise CompletionResponseParsingError( + f"tool '{tool_name}' argument '{argument_name}' must be an integer" + ) + return decoded + if schema_type == "number": + if isinstance(decoded, bool) or not isinstance(decoded, (int, float)): + raise CompletionResponseParsingError( + f"tool '{tool_name}' argument '{argument_name}' must be a number" + ) + return decoded + if schema_type == "boolean": + if not isinstance(decoded, bool): + raise CompletionResponseParsingError( + f"tool '{tool_name}' argument '{argument_name}' must be a boolean" + ) + return decoded + if schema_type == "object": + if not isinstance(decoded, dict): + raise CompletionResponseParsingError( + f"tool '{tool_name}' argument '{argument_name}' must be an object" + ) + return decoded + if schema_type == "array": + if not isinstance(decoded, list): + raise CompletionResponseParsingError( + f"tool '{tool_name}' argument '{argument_name}' must be an array" + ) + return decoded + return decoded + + def _coerce_tool_arguments( + self, + tool_name: str, + arguments: Dict[str, str], + *, + partial: bool, + ) -> Dict[str, Any]: + if self._tool_content_type(tool_name) == "text": + raw_input = arguments.get("input", "") + if not isinstance(raw_input, str): + raw_input = str(raw_input) + return {"input": raw_input} + if self._tools is None: + raise CompletionResponseParsingError( + f"response included tool call '{tool_name}' but the request declared no tools" + ) + tool = next( + ( + candidate + for candidate in self._tools + if candidate.get("type") == "function" + and candidate.get("function", {}).get("name") == tool_name + ), + None, + ) + if tool is None: + raise CompletionResponseParsingError( + f"response included undeclared tool call '{tool_name}'" + ) + parameters = tool.get("function", {}).get("parameters") or { + "type": "object", + "properties": {}, + "required": [], + } + properties = parameters.get("properties", {}) + coerced: Dict[str, Any] = {} + for argument_name, raw_value in arguments.items(): + if argument_name not in properties: + raise CompletionResponseParsingError( + f"tool '{tool_name}' returned unexpected argument '{argument_name}'" + ) + coerced[argument_name] = self._coerce_tool_argument( + raw_value, + properties[argument_name], + tool_name=tool_name, + argument_name=argument_name, + ) + if not partial: + missing = [ + argument_name + for argument_name in parameters.get("required", []) + if argument_name not in coerced + ] + if missing: + raise CompletionResponseParsingError( + f"tool '{tool_name}' response is missing required arguments: {', '.join(missing)}" + ) + return coerced + + def parse_chat_response( + self, + response_text: str, + *, + partial: bool, + ) -> Dict[str, Any]: + full_response_text = self._generation_prompt + response_text + parsed = self._parse_response_value( + full_response_text, + self._schema, + partial=partial, + ) + if not isinstance(parsed, dict): + raise CompletionResponseParsingError("response_schema must produce an object") + if partial: + self._trim_partial_tool_call_prefix( + response_text=full_response_text, + parsed=parsed, + ) + tool_calls = parsed.get("tool_calls") + if isinstance(tool_calls, list): + normalized_tool_calls: List[Dict[str, Any]] = [] + for tool_call in tool_calls: + normalized = self._normalize_tool_call_item(tool_call, partial=partial) + if normalized is None: + continue + normalized_tool_calls.append(normalized) + if normalized_tool_calls: + parsed["tool_calls"] = normalized_tool_calls + else: + parsed.pop("tool_calls", None) + for field in ("reasoning_content", "thinking"): + value = parsed.get(field) + if isinstance(value, str) and not value.strip(): + parsed.pop(field, None) + return parsed + + def _normalize_tool_call_item( + self, + tool_call: Any, + *, + partial: bool, + ) -> Optional[Dict[str, Any]]: + if not isinstance(tool_call, dict): + if partial: + return None + raise CompletionResponseParsingError("tool_calls items must be objects") + function = tool_call.get("function") + if not isinstance(function, dict): + if partial: + return None + raise CompletionResponseParsingError( + "tool_calls items must define a function object" + ) + tool_name = function.get("name") + if not isinstance(tool_name, str) or not tool_name: + if partial: + return None + raise CompletionResponseParsingError( + "tool_calls function name must be a non-empty string" + ) + if self._tool_content_type(tool_name) == "text": + arguments = function.get("arguments", "") + if not isinstance(arguments, str): + if partial: + return None + raise CompletionResponseParsingError( + "tool_calls function arguments must be a string for text tools" + ) + return { + "type": tool_call.get("type", "function"), + "function": { + "name": tool_name, + "arguments": arguments, + }, + } + arguments = function.get("arguments", {}) + if not isinstance(arguments, (dict, ResponseParser.PartialJsonObject)): + if partial: + return None + raise CompletionResponseParsingError( + "tool_calls function arguments must parse to an object" + ) + argument_values = ( + arguments.value + if isinstance(arguments, ResponseParser.PartialJsonObject) + else arguments + ) + raw_arguments: Dict[str, str] = {} + for argument_name, argument_value in argument_values.items(): + if isinstance(argument_value, ResponseParser.PartialJsonValue): + raw_arguments[argument_name] = argument_value.text + else: + raw_arguments[argument_name] = str(argument_value) + normalized_arguments = ( + arguments + if isinstance(arguments, ResponseParser.PartialJsonObject) + or any( + isinstance(value, ResponseParser.PartialJsonValue) + for value in argument_values.values() + ) + else self._coerce_tool_arguments( + tool_name, + raw_arguments, + partial=partial, + ) + ) + return { + "type": tool_call.get("type", "function"), + "function": { + "name": tool_name, + "arguments": normalized_arguments, + }, + } + + @classmethod + def _serialize_partial_json_prefix(cls, value: Any) -> str: + if isinstance(value, ResponseParser.PartialJsonValue): + if value.complete and value.schema_type in {None, "string"}: + return json.dumps(value.text, ensure_ascii=False, separators=(",", ":")) + if value.schema_type in {None, "string"}: + return json.dumps(value.text, ensure_ascii=False, separators=(",", ":"))[:-1] + return value.text + if isinstance(value, ResponseParser.PartialJsonObject): + return cls._serialize_partial_json_prefix(value.value) + if isinstance(value, dict): + items = list(value.items()) + if not items: + return "{" + rendered = ["{"] + last_index = len(items) - 1 + for index, (key, item_value) in enumerate(items): + if index > 0: + rendered.append(",") + rendered.append(json.dumps(key, ensure_ascii=False, separators=(",", ":"))) + rendered.append(":") + if index == last_index: + rendered.append(cls._serialize_partial_json_prefix(item_value)) + else: + rendered.append( + json.dumps(item_value, ensure_ascii=False, separators=(",", ":")) + ) + return "".join(rendered) + if isinstance(value, list): + if not value: + return "[" + rendered = ["["] + last_index = len(value) - 1 + for index, item_value in enumerate(value): + if index > 0: + rendered.append(",") + if index == last_index: + rendered.append(cls._serialize_partial_json_prefix(item_value)) + else: + rendered.append( + json.dumps(item_value, ensure_ascii=False, separators=(",", ":")) + ) + return "".join(rendered) + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False, separators=(",", ":"))[:-1] + return json.dumps(value, ensure_ascii=False, separators=(",", ":")) + + @classmethod + def _contains_partial_json_value(cls, value: Any) -> bool: + if isinstance(value, ResponseParser.PartialJsonValue): + return True + if isinstance(value, ResponseParser.PartialJsonObject): + return True + if isinstance(value, dict): + return any(cls._contains_partial_json_value(item) for item in value.values()) + if isinstance(value, list): + return any(cls._contains_partial_json_value(item) for item in value) + return False + + @classmethod + def _serialize_partial_json_state(cls, value: Any) -> Tuple[str, bool]: + if isinstance(value, ResponseParser.PartialJsonValue): + if value.schema_type in {None, "string"}: + rendered = json.dumps(value.text, ensure_ascii=False, separators=(",", ":")) + if value.complete: + return rendered, True + return rendered[:-1], False + return value.text, value.complete + if isinstance(value, ResponseParser.PartialJsonObject): + rendered, children_complete = cls._serialize_partial_json_state(value.value) + if value.complete and children_complete: + return rendered, True + if rendered.endswith("}"): + return rendered[:-1], False + return rendered, False + if isinstance(value, dict): + parts = ["{"] + items = list(value.items()) + for index, (key, item_value) in enumerate(items): + if index > 0: + parts.append(",") + parts.append(json.dumps(key, ensure_ascii=False, separators=(",", ":"))) + parts.append(":") + rendered_item, item_complete = cls._serialize_partial_json_state(item_value) + parts.append(rendered_item) + if index == len(items) - 1 and not item_complete: + return "".join(parts), False + parts.append("}") + return "".join(parts), True + if isinstance(value, list): + parts = ["["] + for index, item_value in enumerate(value): + if index > 0: + parts.append(",") + rendered_item, item_complete = cls._serialize_partial_json_state(item_value) + parts.append(rendered_item) + if index == len(value) - 1 and not item_complete: + return "".join(parts), False + parts.append("]") + return "".join(parts), True + return json.dumps(value, ensure_ascii=False, separators=(",", ":")), True + + @classmethod + def _serialize_tool_arguments(cls, arguments: Any, *, partial: bool = False) -> str: + if partial: + if cls._contains_partial_json_value(arguments): + return cls._serialize_partial_json_state(arguments)[0] + return cls._serialize_partial_json_prefix(arguments) + if isinstance(arguments, ResponseParser.PartialJsonObject): + arguments = arguments.value + return json.dumps(arguments, ensure_ascii=False, separators=(",", ":")) + + def _parsed_chat_message( + self, + *, + parsed: Dict[str, Any], + partial: bool = False, + ) -> Dict[str, Any]: + message: Dict[str, Any] = { + "role": parsed.get("role", "assistant"), + } + for key, value in parsed.items(): + if key in {"role", "content", "tool_calls"}: + continue + if value is None or value == "": + continue + message[key] = value + content = parsed.get("content") + tool_calls = parsed.get("tool_calls") + if isinstance(tool_calls, list) and tool_calls: + normalized_tool_calls = [] + for tool_call_index, tool_call in enumerate(tool_calls): + function = tool_call["function"] + if self._tool_content_type(function["name"]) == "text": + raw_arguments = function["arguments"] + arguments = ( + raw_arguments + if isinstance(raw_arguments, str) + else str(raw_arguments) + ) + else: + arguments = self._serialize_tool_arguments( + function["arguments"], + partial=partial, + ) + normalized_tool_calls.append( + { + "id": f"call_{self._choice_index}_{function['name']}_{self._completion_id}_{tool_call_index}", + "type": tool_call.get("type", "function"), + "function": { + "name": function["name"], + "arguments": arguments, + }, + } + ) + message["content"] = content if content not in {None, ""} else None + message["tool_calls"] = normalized_tool_calls + if len(normalized_tool_calls) == 1: + message["function_call"] = dict(normalized_tool_calls[0]["function"]) + return message + message["content"] = content if content is not None else "" + return message + + def _message_deltas( + self, + previous_message: Dict[str, Any], + message: Dict[str, Any], + ) -> List[Dict[str, Any]]: + deltas: List[Dict[str, Any]] = [] + for key, value in message.items(): + if key in {"role", "content", "tool_calls", "function_call"}: + continue + old_value = previous_message.get(key, "") + if not isinstance(value, str): + if key not in previous_message and value is not None: + deltas.append({key: value}) + continue + if not value: + continue + if isinstance(old_value, str) and value.startswith(old_value): + delta_value = value[len(old_value) :] + else: + delta_value = value + if delta_value: + deltas.append({key: delta_value}) + + new_content = message.get("content") + old_content = previous_message.get("content", "") + if isinstance(new_content, str) and new_content: + if isinstance(old_content, str) and new_content.startswith(old_content): + content_delta = new_content[len(old_content) :] + else: + content_delta = new_content + if content_delta: + deltas.append({"content": content_delta}) + + new_tool_calls = cast(List[Dict[str, Any]], message.get("tool_calls", [])) + old_tool_calls = cast(List[Dict[str, Any]], previous_message.get("tool_calls", [])) + for tool_call_index, tool_call in enumerate(new_tool_calls): + old_tool_call = old_tool_calls[tool_call_index] if tool_call_index < len(old_tool_calls) else None + delta_tool_call: Dict[str, Any] = {"index": tool_call_index} + if old_tool_call is None: + delta_tool_call["id"] = tool_call["id"] + delta_tool_call["type"] = tool_call["type"] + function = cast(Dict[str, Any], tool_call["function"]) + old_function = ( + cast(Dict[str, Any], old_tool_call["function"]) + if old_tool_call is not None + else {} + ) + function_delta: Dict[str, Any] = {} + if function.get("name") and function.get("name") != old_function.get("name"): + function_delta["name"] = function["name"] + arguments = cast(str, function.get("arguments", "")) + old_arguments = cast(str, old_function.get("arguments", "")) + if old_tool_call is None and arguments == "{}": + argument_delta = "" + elif arguments.startswith(old_arguments): + argument_delta = arguments[len(old_arguments) :] + else: + argument_delta = arguments + if argument_delta: + function_delta["arguments"] = argument_delta + if function_delta: + delta_tool_call["function"] = function_delta + if len(delta_tool_call) > 1: + deltas.append({"tool_calls": [delta_tool_call]}) + return deltas + + @staticmethod + def _apply_message_delta(message: Dict[str, Any], delta: Dict[str, Any]) -> None: + if "role" in delta: + message["role"] = delta["role"] + for key, value in delta.items(): + if key in {"role", "tool_calls", "function_call"}: + continue + if isinstance(value, str): + existing = message.get(key) + if isinstance(existing, str): + message[key] = existing + value + else: + message[key] = value + else: + message[key] = value + tool_call_deltas = delta.get("tool_calls") + if not isinstance(tool_call_deltas, list): + return + tool_calls = cast(List[Dict[str, Any]], message.setdefault("tool_calls", [])) + for tool_delta in tool_call_deltas: + if not isinstance(tool_delta, dict): + continue + index = tool_delta.get("index") + if not isinstance(index, int): + continue + while len(tool_calls) <= index: + tool_calls.append({"function": {"name": "", "arguments": ""}}) + tool_call = tool_calls[index] + if "id" in tool_delta: + tool_call["id"] = tool_delta["id"] + if "type" in tool_delta: + tool_call["type"] = tool_delta["type"] + function_delta = tool_delta.get("function") + if not isinstance(function_delta, dict): + continue + function = cast(Dict[str, Any], tool_call.setdefault("function", {})) + name_delta = function_delta.get("name") + if isinstance(name_delta, str): + function["name"] = cast(str, function.get("name", "")) + name_delta + arguments_delta = function_delta.get("arguments") + if isinstance(arguments_delta, str): + function["arguments"] = cast(str, function.get("arguments", "")) + arguments_delta + if tool_calls: + message["function_call"] = dict(cast(Dict[str, Any], tool_calls[0]["function"])) + + def parse_completion_message(self, response_text: str) -> Dict[str, Any]: + parsed = self.parse_chat_response(response_text, partial=False) + return self._parsed_chat_message(parsed=parsed) + + def _stream_state_message(self, *, partial: bool) -> Dict[str, Any]: + assert self._stream_state is not None + if self._stream_plan is not None and self._stream_plan.get("direct_deltas"): + copied = { + key: ( + [ + { + child_key: ( + dict(child_value) + if isinstance(child_value, dict) + else child_value + ) + for child_key, child_value in tool_call.items() + } + for tool_call in cast(List[Dict[str, Any]], value) + ] + if key == "tool_calls" and isinstance(value, list) + else value + ) + for key, value in self._message.items() + } + if copied.get("tool_calls"): + copied["function_call"] = dict( + cast(List[Dict[str, Any]], copied["tool_calls"])[0]["function"] + ) + return copied + parsed = cast(Dict[str, Any], self._stream_state.parsed) + return self._parsed_chat_message(parsed=parsed, partial=partial) + + def _stream_state_complete(self) -> bool: + if self._stream_plan is None: + return False + if self._direct.deltas: + return ( + not self._direct.pending + and self._direct.mode in {self.DIRECT_MODE_CONTENT, self.DIRECT_MODE_AFTER_TOOL_ITEM} + ) + if self._stream_state is None: + return False + if self._stream_plan["kind"] == "json-root": + return True + if self._stream_plan["kind"] == "segment-message": + return ( + not self._stream_state.pending + and self._stream_state.current_item is None + and self._stream_state.current_segment is None + and self._stream_state.mode == "segment-start" + ) + return ( + not self._stream_state.pending + and self._stream_state.current_item is None + and self._stream_state.mode in {"content", "after-tool-item"} + ) + + @staticmethod + def _chat_chunk_payload( + *, + chunk_id: str, + created: int, + model: str, + index: int, + delta: Dict[str, Any], + finish_reason: Optional[str], + logprobs: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + choice: Dict[str, Any] = { + "index": index, + "delta": delta, + "finish_reason": finish_reason, + } + if logprobs is not None: + choice["logprobs"] = logprobs + return { + "id": "chat" + chunk_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [choice], + } + + def _chunk_payloads( + self, + *, + chunk_id: str, + created: int, + model: str, + deltas: List[Dict[str, Any]], + finish_reason: Optional[str], + logprobs: Optional[Dict[str, Any]], + leading_delta: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + payloads: List[Dict[str, Any]] = [] + logprobs_sent = False + chat_chunk_id = "chat" + chunk_id + index = self._choice_index + if leading_delta is not None: + payloads.append( + { + "id": chat_chunk_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [ + { + "index": index, + "delta": leading_delta, + "finish_reason": None, + } + ], + } + ) + for delta in deltas: + payload_logprobs = None + if ( + logprobs is not None + and delta.get("role") != "assistant" + and not logprobs_sent + ): + payload_logprobs = logprobs + logprobs_sent = True + choice: Dict[str, Any] = { + "index": index, + "delta": delta, + "finish_reason": None, + } + if payload_logprobs is not None: + choice["logprobs"] = payload_logprobs + payloads.append( + { + "id": chat_chunk_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [choice], + } + ) + if finish_reason is not None: + payloads.append( + { + "id": chat_chunk_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [ + { + "index": index, + "delta": {}, + "finish_reason": finish_reason, + } + ], + } + ) + return payloads + + def consume_completion_chunk( + self, + text: str, + *, + chunk_id: str, + created: int, + model: str, + finish_reason: Optional[str], + logprobs: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + role_delta: Optional[Dict[str, Any]] = None + if not self._started: + self._started = True + self._message["role"] = "assistant" + role_delta = {"role": "assistant"} + self._text_parts.append(text) + + if self._stream_plan is not None and not self._stream_failed: + success, stream_deltas = self._advance_stream_state(text) + if success: + if self._direct.deltas: + if finish_reason is None: + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=stream_deltas, + finish_reason=None, + logprobs=logprobs, + leading_delta=role_delta, + ) + if self._stream_state_complete(): + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=stream_deltas, + finish_reason=( + "tool_calls" if self._direct.saw_tool_calls else finish_reason + ), + logprobs=logprobs, + leading_delta=role_delta, + ) + self._stream_failed = True + elif self._stream_plan["kind"] == "segment-message": + if role_delta is not None: + stream_deltas = [role_delta, *stream_deltas] + for delta in stream_deltas: + self._apply_message_delta(self._message, delta) + if finish_reason is None: + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=stream_deltas, + finish_reason=None, + logprobs=logprobs, + ) + if self._stream_state_complete(): + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=stream_deltas, + finish_reason=( + "tool_calls" if self._message.get("tool_calls") else finish_reason + ), + logprobs=logprobs, + ) + self._stream_failed = True + else: + previous_message = self._message + partial_deltas: List[Dict[str, Any]] = [] + assert self._stream_state is not None + parsed = cast(Dict[str, Any], self._stream_state.parsed) + message = self._parsed_chat_message( + parsed=parsed, + partial=finish_reason is None, + ) + if finish_reason is None: + if role_delta is not None: + partial_deltas.append(role_delta) + partial_deltas.extend(self._message_deltas(previous_message, message)) + self._message = message + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=partial_deltas, + finish_reason=None, + logprobs=logprobs, + ) + if self._stream_state_complete(): + if role_delta is not None: + partial_deltas.append(role_delta) + partial_deltas.extend(self._message_deltas(previous_message, message)) + self._message = message + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=partial_deltas, + finish_reason=( + "tool_calls" if message.get("tool_calls") else finish_reason + ), + logprobs=logprobs, + ) + self._stream_failed = True + else: + self._stream_failed = True + + response_text = "".join(self._text_parts) + parsed = self.parse_chat_response(response_text, partial=finish_reason is None) + message = self._parsed_chat_message(parsed=parsed, partial=finish_reason is None) + previous_message = self._message + deltas: List[Dict[str, Any]] = [] + if role_delta is not None: + deltas.append(role_delta) + deltas.extend(self._message_deltas(previous_message, message)) + self._message = message + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=deltas, + finish_reason=( + "tool_calls" + if finish_reason is not None and message.get("tool_calls") + else finish_reason + ), + logprobs=logprobs, + ) + + +class OpenAIFormatter: + @dataclass + class ReturnedToken: + index: int + text_bytes: bytes + token: Token + text_offset: int + + @dataclass + class ResponsesOutputItem: + output_index: int + item: Dict[str, Any] + content_index: Optional[int] = None + + @dataclass + class ResponsesStream: + body: CreateResponseRequest + response_id: str + created_at: float + model: str + output: List[Dict[str, Any]] = field(default_factory=list) + sequence_number: int = 0 + started: bool = False + assistant_meta: Dict[str, Any] = field(default_factory=dict) + reasoning_item: Optional["OpenAIFormatter.ResponsesOutputItem"] = None + message_item: Optional["OpenAIFormatter.ResponsesOutputItem"] = None + tool_items: Dict[int, "OpenAIFormatter.ResponsesOutputItem"] = field( + default_factory=dict + ) + final_status: Optional[str] = None + incomplete_details: Optional[Dict[str, Any]] = None + + def __init__(self, model: Model) -> None: + self.model = model + + @staticmethod + def decode_text(data: bytes) -> str: + return data.decode("utf-8", errors="ignore") + + @staticmethod + def encode_sse_payload(payload: BaseModel | Dict[str, Any]) -> bytes: + data = ( + payload.model_dump(mode="json", exclude_none=True) + if isinstance(payload, BaseModel) + else payload + ) + return ( + "data: " + f"{json.dumps(data, ensure_ascii=False, separators=(',', ':'))}\n\n" + ).encode("utf-8") + + @staticmethod + def next_stream_chunk( + stream: Iterator[CompletionChunk], + ) -> Tuple[bool, Optional[CompletionChunk]]: + try: + return False, next(stream) + except StopIteration: + return True, None + + @staticmethod + def next_stream_output( + stream: Iterator[CompletionChunk], + ) -> Tuple[bool, Optional[CompletionChunk], Optional[OpenAICompletion]]: + try: + return False, next(stream), None + except StopIteration as stop: + return True, None, cast(Optional[OpenAICompletion], stop.value) + + @staticmethod + def collect_completion(stream: Iterator[Any]) -> OpenAICompletion: + iterator = iter(stream) + while True: + try: + next(iterator) + except StopIteration as stop: + result = stop.value + if result is None: + raise RuntimeError("missing completion result") + return cast(OpenAICompletion, result) + + @staticmethod + def _tools_for_response_parser( + *, + functions: Optional[List[ChatTemplateFunctionDefinition]] = None, + tools: Optional[List[ChatTemplateTool]] = None, + ) -> Optional[List[ChatTemplateTool]]: + if functions is not None: + return [ + { + "type": "function", + "function": function, + } + for function in functions + ] + return tools + + def _chat_template_text(self) -> str: + if self.model.chat_formatter is None: + return "" + return cast(str, self.model.chat_formatter._template_text) + + def _uses_harmony_channels(self) -> bool: + template = self._chat_template_text() + return "<|channel|>" in template or "<|recipient|>" in template + + def _uses_reasoning_content(self) -> bool: + template = self._chat_template_text() + return "reasoning_content" in template or "<think>" in template + + @staticmethod + def _chat_message(data: Dict[str, Any]) -> ChatCompletionRequestMessage: + return ChatCompletionRequestMessage.model_validate(data) + + def _instructions_role(self) -> Literal["developer", "system"]: + return "developer" if "developer" in self._chat_template_text() else "system" + + @staticmethod + def _response_reasoning_effort( + body: CreateResponseRequest, + ) -> Optional[str]: + reasoning = body.reasoning + if reasoning is None: + return None + effort = reasoning.effort + if effort in {"low", "medium", "high"}: + return cast(str, effort) + return None + + @staticmethod + def _response_text_from_content(content: Any) -> str: + if isinstance(content, str): + return content + if not isinstance(content, list): + raise CompletionRequestValidationError("responses input content must be a string or list") + parts: List[str] = [] + for part in content: + if not isinstance(part, dict): + raise CompletionRequestValidationError("responses content parts must be objects") + part_type = part.get("type") + if part_type in {"input_text", "output_text", "reasoning_text", "summary_text", "text"}: + text = part.get("text") + if not isinstance(text, str): + raise CompletionRequestValidationError( + f"responses content part {part_type!r} requires string text" + ) + parts.append(text) + continue + raise CompletionRequestValidationError( + f"unsupported responses content part type: {part_type!r}" + ) + return "".join(parts) + + @staticmethod + def _response_chat_content_from_content( + content: Any, + ) -> Union[str, List[Dict[str, Any]]]: + if isinstance(content, str): + return content + if not isinstance(content, list): + raise CompletionRequestValidationError("responses input content must be a string or list") + parts: List[Dict[str, Any]] = [] + for part in content: + if not isinstance(part, dict): + raise CompletionRequestValidationError("responses content parts must be objects") + part_type = part.get("type") + if part_type in {"input_text", "output_text", "reasoning_text", "summary_text", "text"}: + text = part.get("text") + if not isinstance(text, str): + raise CompletionRequestValidationError( + f"responses content part {part_type!r} requires string text" + ) + parts.append({"type": "text", "text": text}) + continue + if part_type == "input_image": + image_url = part.get("image_url") + if not isinstance(image_url, str): + raise CompletionRequestValidationError( + "responses input_image content part requires image_url" + ) + parts.append({"type": "image_url", "image_url": {"url": image_url}}) + continue + if part_type == "input_audio": + input_audio = part.get("input_audio") + if isinstance(input_audio, dict): + data = input_audio.get("data") + audio_format = input_audio.get("format") + else: + data = part.get("data") + audio_format = part.get("format") + if not isinstance(data, str): + raise CompletionRequestValidationError( + "responses input_audio content part requires base64 data" + ) + if audio_format is not None and not isinstance(audio_format, str): + raise CompletionRequestValidationError( + "responses input_audio format must be a string" + ) + audio_part: Dict[str, Any] = {"data": data} + if audio_format is not None: + audio_part["format"] = audio_format + parts.append({"type": "input_audio", "input_audio": audio_part}) + continue + if part_type == "audio_url": + audio_url = part.get("audio_url") + if not isinstance(audio_url, str): + raise CompletionRequestValidationError( + "responses audio_url content part requires audio_url" + ) + parts.append({"type": "audio_url", "audio_url": {"url": audio_url}}) + continue + raise CompletionRequestValidationError( + f"unsupported responses content part type: {part_type!r}" + ) + if all(part.get("type") == "text" for part in parts): + return "".join(cast(str, part.get("text", "")) for part in parts) + return parts + + @staticmethod + def _response_reasoning_text(item: Dict[str, Any]) -> str: + content = item.get("content") + if isinstance(content, list) and content: + return OpenAIFormatter._response_text_from_content(content) + summary = item.get("summary") + if isinstance(summary, list) and summary: + return OpenAIFormatter._response_text_from_content(summary) + return "" + + def _response_reasoning_input_message( + self, + *, + text: str, + ) -> ChatCompletionRequestMessage: + if self._uses_harmony_channels(): + return self._chat_message( + { + "role": "assistant", + "content": text, + "channel": "analysis", + } + ) + if self._uses_reasoning_content(): + return self._chat_message( + { + "role": "assistant", + "content": "", + "reasoning_content": text, + } + ) + return self._chat_message( + { + "role": "assistant", + "content": text, + } + ) + + @staticmethod + def _response_tool_call_input_message( + *, + name: str, + arguments: str, + call_id: str, + content_type: Optional[str] = None, + ) -> ChatCompletionRequestMessage: + function: Dict[str, Any] = { + "name": name, + "arguments": arguments, + } + if isinstance(content_type, str) and content_type: + function["content_type"] = content_type + tool_call = { + "id": call_id, + "type": "function", + "function": function, + } + message: Dict[str, Any] = { + "role": "assistant", + "content": None, + "tool_calls": [tool_call], + "function_call": dict(function), + } + return OpenAIFormatter._chat_message(message) + + def _response_function_call_input_message( + self, + item: Dict[str, Any], + ) -> ChatCompletionRequestMessage: + name = item.get("name") + if not isinstance(name, str) or not name: + raise CompletionRequestValidationError("function_call input requires name") + arguments = item.get("arguments", "") + if not isinstance(arguments, str): + raise CompletionRequestValidationError("function_call input requires string arguments") + call_id = item.get("call_id") or item.get("id") or f"call_{uuid.uuid4().hex}" + return self._response_tool_call_input_message( + name=name, + arguments=arguments, + call_id=call_id, + content_type="json", + ) + + def _response_custom_tool_call_input_message( + self, + item: Dict[str, Any], + ) -> ChatCompletionRequestMessage: + name = item.get("name") + if not isinstance(name, str) or not name: + raise CompletionRequestValidationError("custom_tool_call input requires name") + input_text = item.get("input", "") + if not isinstance(input_text, str): + raise CompletionRequestValidationError("custom_tool_call input requires string input") + call_id = item.get("call_id") or item.get("id") or f"call_{uuid.uuid4().hex}" + content_type = item.get("content_type") + if not isinstance(content_type, str) or not content_type: + content_type = "text" + return self._response_tool_call_input_message( + name=name, + arguments=input_text, + call_id=call_id, + content_type=content_type, + ) + + def responses_input_to_chat_messages( + self, + body: CreateResponseRequest, + ) -> List[ChatCompletionRequestMessage]: + if body.conversation is not None: + raise CompletionRequestValidationError( + "conversation is not supported in stateless /v1/responses" + ) + if body.store: + raise CompletionRequestValidationError( + "store=true is not supported in stateless /v1/responses" + ) + if body.truncation not in {None, "disabled"}: + raise CompletionRequestValidationError( + "only truncation='disabled' is supported" + ) + + messages: List[ChatCompletionRequestMessage] = [] + if body.instructions is not None: + messages.append( + self._chat_message( + { + "role": self._instructions_role(), + "content": body.instructions, + } + ) + ) + + if isinstance(body.input, str): + messages.append( + self._chat_message( + { + "role": "user", + "content": body.input, + } + ) + ) + return messages + + items = body.input + if isinstance(items, dict): + items = [items] + if not isinstance(items, list): + raise CompletionRequestValidationError( + "responses input must be a string, object, or list" + ) + + function_names_by_call_id: Dict[str, str] = {} + for item in items: + if not isinstance(item, dict): + raise CompletionRequestValidationError( + "responses input items must be objects" + ) + item_type = item.get("type") + if item_type is None and "role" in item: + item_type = "message" + if item_type == "message": + role = item.get("role", "user") + if role not in { + "user", + "assistant", + "system", + "developer", + "tool", + "function", + }: + raise CompletionRequestValidationError( + f"unsupported responses message role: {role!r}" + ) + if role == "function": + role = "tool" + data: Dict[str, Any] = { + "role": role, + "content": self._response_chat_content_from_content( + item.get("content", "") + ), + } + phase = item.get("phase") + if isinstance(phase, str): + data["phase"] = phase + if role == "assistant" and self._uses_harmony_channels(): + if phase == "commentary": + data["channel"] = "commentary" + elif phase == "final_answer": + data["channel"] = "final" + messages.append(self._chat_message(data)) + continue + if item_type == "reasoning": + text = self._response_reasoning_text(item) + if text: + messages.append(self._response_reasoning_input_message(text=text)) + continue + if item_type == "function_call": + message = self._response_function_call_input_message(item) + call_id = item.get("call_id") or item.get("id") + name = item.get("name") + if isinstance(call_id, str) and isinstance(name, str): + function_names_by_call_id[call_id] = name + messages.append(message) + continue + if item_type == "custom_tool_call": + message = self._response_custom_tool_call_input_message(item) + call_id = item.get("call_id") or item.get("id") + name = item.get("name") + if isinstance(call_id, str) and isinstance(name, str): + function_names_by_call_id[call_id] = name + messages.append(message) + continue + if item_type == "function_call_output": + call_id = item.get("call_id") + if not isinstance(call_id, str) or not call_id: + raise CompletionRequestValidationError( + "function_call_output input requires call_id" + ) + tool_output_data: Dict[str, Any] = { + "role": "tool", + "tool_call_id": call_id, + "content": self._response_text_from_content(item.get("output", "")), + } + name = function_names_by_call_id.get(call_id) + if name is not None: + tool_output_data["name"] = name + messages.append(self._chat_message(tool_output_data)) + continue + if item_type == "custom_tool_call_output": + call_id = item.get("call_id") + if not isinstance(call_id, str) or not call_id: + raise CompletionRequestValidationError( + "custom_tool_call_output input requires call_id" + ) + tool_output_data = { + "role": "tool", + "tool_call_id": call_id, + "content": self._response_text_from_content(item.get("output", "")), + } + name = function_names_by_call_id.get(call_id) + if name is not None: + tool_output_data["name"] = name + messages.append(self._chat_message(tool_output_data)) + continue + raise CompletionRequestValidationError( + f"unsupported responses input item type: {item_type!r}" + ) + return messages + + @staticmethod + def _clone_response_input_items(input_items: Any) -> List[Any]: + if isinstance(input_items, list): + return copy.deepcopy(input_items) + if isinstance(input_items, dict): + return [copy.deepcopy(input_items)] + return [copy.deepcopy(input_items)] + + @staticmethod + def _normalize_response_output_item_for_input( + item: Dict[str, Any], + ) -> Optional[Dict[str, Any]]: + item_type = item.get("type") + if item_type == "message": + role = item.get("role") + if not isinstance(role, str): + return None + normalized: Dict[str, Any] = { + "type": "message", + "role": role, + "content": copy.deepcopy(item.get("content", [])), + } + for key in ("id", "phase", "status"): + value = item.get(key) + if value is not None: + normalized[key] = copy.deepcopy(value) + return normalized + if item_type == "reasoning": + normalized = { + "type": "reasoning", + "content": copy.deepcopy(item.get("content", [])), + } + for key in ("id", "summary", "status"): + value = item.get(key) + if value is not None: + normalized[key] = copy.deepcopy(value) + return normalized + if item_type == "function_call": + normalized = { + "type": "function_call", + "call_id": copy.deepcopy(item.get("call_id")), + "name": copy.deepcopy(item.get("name")), + "arguments": copy.deepcopy(item.get("arguments", "")), + } + namespace = item.get("namespace") + item_id = item.get("id") + if item_id is not None: + normalized["id"] = copy.deepcopy(item_id) + if namespace is not None: + normalized["namespace"] = copy.deepcopy(namespace) + return normalized + if item_type == "custom_tool_call": + normalized = { + "type": "custom_tool_call", + "call_id": copy.deepcopy(item.get("call_id")), + "name": copy.deepcopy(item.get("name")), + "input": copy.deepcopy(item.get("input", "")), + } + namespace = item.get("namespace") + content_type = item.get("content_type") + item_id = item.get("id") + if item_id is not None: + normalized["id"] = copy.deepcopy(item_id) + if namespace is not None: + normalized["namespace"] = copy.deepcopy(namespace) + if content_type is not None: + normalized["content_type"] = copy.deepcopy(content_type) + return normalized + return None + + def _responses_tools_to_chat_tools( + self, + tools: Optional[List[ResponsesToolDefinition]], + ) -> Optional[List[ChatTemplateTool]]: + if tools is None: + return None + chat_tools: List[ChatTemplateTool] = [] + for tool in tools: + if isinstance(tool, ResponsesWebSearchTool): + continue + if isinstance(tool, ResponsesFunctionTool): + chat_tools.append(tool.to_chat_template_tool()) + continue + if isinstance(tool, ResponsesCustomTool): + chat_tools.append(tool.to_chat_template_tool()) + continue + raise CompletionRequestValidationError( + f"unsupported responses tool type: {tool.type!r}" + ) + return chat_tools + + @staticmethod + def _responses_tool_choice_to_chat_tool_choice( + tool_choice: Optional[ResponsesToolChoice], + ) -> Optional[Union[Literal["auto", "none", "required"], ChatTemplateToolChoice]]: + if tool_choice is None or isinstance(tool_choice, str): + return tool_choice + if tool_choice.type in {"function", "custom"}: + return tool_choice.to_chat_template_tool_choice() + raise CompletionRequestValidationError( + f"unsupported responses tool_choice type: {tool_choice.type!r}" + ) + + @staticmethod + def _response_format_type(response_format: Optional[ChatTemplateResponseFormat]) -> Optional[str]: + if response_format is None: + return None + format_type = response_format.get("type") + if isinstance(format_type, str): + return format_type + return None + + @staticmethod + def _grammar_for_response_format( + response_format: Optional[ChatTemplateResponseFormat], + ) -> Optional[str]: + format_type = OpenAIFormatter._response_format_type(response_format) + if format_type is None or format_type == "text": + return None + if format_type == "json_object": + return JSON_GBNF + if format_type == "json_schema": + assert response_format is not None + schema = response_format.get("schema") + if schema is None and isinstance(response_format.get("json_schema"), dict): + schema = cast(Dict[str, Any], response_format["json_schema"]).get("schema") + if not isinstance(schema, dict): + raise CompletionRequestValidationError( + "json_schema response format requires a schema object" + ) + return JsonSchemaConverter.to_gbnf( + json.dumps(schema, ensure_ascii=False, separators=(",", ":")) + ) + raise CompletionRequestValidationError( + f"unsupported response format type: {format_type!r}" + ) + + def response_request_to_chat_parts( + self, + body: CreateResponseRequest, + ) -> ResponsesChatRequestParts: + chat_tools = self._responses_tools_to_chat_tools(body.tools) + response_format = ( + None + if body.text is None or body.text.format is None + else body.text.format.to_chat_response_format() + ) + return ResponsesChatRequestParts( + messages=self.responses_input_to_chat_messages(body), + max_tokens=body.max_output_tokens, + temperature=0.8 if body.temperature is None else body.temperature, + top_p=0.95 if body.top_p is None else body.top_p, + stream=body.stream, + logprobs=body.top_logprobs is not None, + top_logprobs=body.top_logprobs, + model=body.model, + user=body.user, + tools=chat_tools, + tool_choice=self._responses_tool_choice_to_chat_tool_choice(body.tool_choice), + response_format=response_format, + reasoning_effort=self._response_reasoning_effort(body), + ) + + def _response_parser( + self, + *, + tools: Optional[List[ChatTemplateTool]] = None, + completion_id: str = "", + choice_index: int = 0, + generation_prompt: str = "", + ) -> ResponseParser: + if self.model.response_schema is None: + raise CompletionResponseParsingError("model does not define response_schema") + return ResponseParser( + self.model.response_schema, + tools=tools, + completion_id=completion_id, + choice_index=choice_index, + generation_prompt=generation_prompt, + ) + + def parse_chat_response( + self, + response_text: str, + *, + tools: Optional[List[ChatTemplateTool]] = None, + partial: bool, + generation_prompt: str = "", + ) -> Dict[str, Any]: + return self._response_parser( + tools=tools, + generation_prompt=generation_prompt, + ).parse_chat_response( + response_text, + partial=partial, + ) + + def _chat_tool_name_and_grammar( + self, + *, + tools: Optional[List[ChatTemplateTool]], + function_call: Optional[Union[Literal["none", "auto"], ChatTemplateFunctionCall]], + tool_choice: Optional[Union[Literal["none", "auto", "required"], ChatTemplateToolChoice]], + response_format: Optional[ChatTemplateResponseFormat], + ) -> Tuple[Optional[str], Optional[str]]: + selected_tool_choice: Optional[Union[str, ChatTemplateToolChoice]] = tool_choice + if function_call is not None: + if isinstance(function_call, str): + selected_tool_choice = function_call + else: + selected_tool_choice = { + "type": "function", + "function": { + "name": function_call["name"], + }, + } + grammar_text = self._grammar_for_response_format(response_format) + if not isinstance(selected_tool_choice, dict): + return None, grammar_text + if tools is None: + raise CompletionRequestValidationError("tool choice requires tools") + tool_name = selected_tool_choice["function"]["name"] + tool = next((tool for tool in tools if tool["function"]["name"] == tool_name), None) + if tool is None: + raise CompletionRequestValidationError( + f"Tool choice '{tool_name}' not found in tools." + ) + if self.model.response_schema is None: + return tool_name, JSON_GBNF + return tool_name, grammar_text + + def completion_request_from_chat_request( + self, + body: CreateChatCompletionRequest, + ) -> PreparedCompletionParts: + functions = ( + [function.to_template_function() for function in body.functions] + if body.functions is not None + else None + ) + tools = ( + [tool.to_template_tool() for tool in body.tools] + if body.tools is not None + else None + ) + function_call = ( + body.function_call + if body.function_call is None or isinstance(body.function_call, str) + else body.function_call.to_template_function_call() + ) + tool_choice = ( + body.tool_choice + if body.tool_choice is None or isinstance(body.tool_choice, str) + else body.tool_choice.to_template_tool_choice() + ) + response_format = ( + body.response_format.to_template_response_format() + if body.response_format is not None + else None + ) + parser_tools = self._tools_for_response_parser( + functions=functions, + tools=tools, + ) + try: + prompt_text, generation_prompt, prompt_plan, formatter_stop = self.model.build_chat_prompt( + body.messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + reasoning_effort=body.reasoning_effort, + ) + tool_name, grammar_text = self._chat_tool_name_and_grammar( + tools=parser_tools, + function_call=function_call, + tool_choice=tool_choice, + response_format=response_format, + ) + except ValueError as exc: + raise CompletionRequestValidationError(str(exc)) from exc + request_stop: List[str] = [] + if body.stop is None: + request_stop = [] + elif isinstance(body.stop, str): + request_stop = [body.stop] + else: + request_stop = list(body.stop) + stop_sequences = [stop for stop in [*request_stop, *formatter_stop] if stop] + deduped_stop: List[str] = [] + seen_stop: set[str] = set() + for stop in stop_sequences: + if stop not in seen_stop: + deduped_stop.append(stop) + seen_stop.add(stop) + payload = CreateCompletionRequest( + prompt=prompt_text, + max_tokens=body.max_tokens, + temperature=body.temperature, + top_p=body.top_p, + echo=False, + stop=deduped_stop or None, + stream=body.stream, + logprobs=( + 0 if body.logprobs and body.top_logprobs is None else body.top_logprobs + ), + presence_penalty=body.presence_penalty, + frequency_penalty=body.frequency_penalty, + logit_bias=body.logit_bias, + seed=body.seed, + model=body.model, + n=body.n, + user=body.user, + ) + return PreparedCompletionParts( + payload=payload, + prompt_text=prompt_text, + generation_prompt=generation_prompt, + prompt_plan=prompt_plan, + grammar_text=grammar_text, + tool_name=tool_name, + ) + + def completion_request_from_response_chat_parts( + self, + body: ResponsesChatRequestParts, + ) -> PreparedCompletionParts: + try: + prompt_text, generation_prompt, prompt_plan, formatter_stop = self.model.build_chat_prompt( + body.messages, + tools=body.tools, + tool_choice=body.tool_choice, + reasoning_effort=body.reasoning_effort, + ) + tool_name, grammar_text = self._chat_tool_name_and_grammar( + tools=body.tools, + function_call=None, + tool_choice=body.tool_choice, + response_format=body.response_format, + ) + except ValueError as exc: + raise CompletionRequestValidationError(str(exc)) from exc + deduped_stop: List[str] = [] + seen_stop: set[str] = set() + for stop in formatter_stop: + if stop and stop not in seen_stop: + deduped_stop.append(stop) + seen_stop.add(stop) + payload = CreateCompletionRequest( + prompt=prompt_text, + max_tokens=body.max_tokens, + temperature=body.temperature, + top_p=body.top_p, + echo=False, + stop=deduped_stop or None, + stream=body.stream, + logprobs=( + 0 if body.logprobs and body.top_logprobs is None else body.top_logprobs + ), + seed=None, + model=body.model, + n=1, + user=body.user, + ) + return PreparedCompletionParts( + payload=payload, + prompt_text=prompt_text, + generation_prompt=generation_prompt, + prompt_plan=prompt_plan, + grammar_text=grammar_text, + tool_name=tool_name, + ) + + @staticmethod + def _response_phase_from_message(message: Dict[str, Any]) -> Optional[str]: + phase = message.get("phase") + if phase in {"commentary", "final_answer"}: + return cast(str, phase) + channel = message.get("channel") + if channel == "commentary": + return "commentary" + if channel == "final": + return "final_answer" + return None + + @staticmethod + def _response_reasoning_text_from_message(message: Dict[str, Any]) -> str: + thinking = message.get("thinking") + if isinstance(thinking, str) and thinking: + return thinking + reasoning_content = message.get("reasoning_content") + if isinstance(reasoning_content, str) and reasoning_content: + return reasoning_content + if message.get("channel") == "analysis": + content = message.get("content") + if isinstance(content, str): + return content + return "" + + @staticmethod + def _response_output_text_from_message(message: Dict[str, Any]) -> str: + if message.get("channel") == "analysis": + return "" + content = message.get("content") + if isinstance(content, str): + return content + return "" + + @staticmethod + def _response_logprobs_from_chat_logprobs( + logprobs: Optional[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + if logprobs is None: + return [] + content = logprobs.get("content") + if not isinstance(content, list): + return [] + response_logprobs: List[Dict[str, Any]] = [] + for entry in content: + if not isinstance(entry, dict): + continue + token = entry.get("token") + logprob = entry.get("logprob") + if not isinstance(token, str) or not isinstance(logprob, (int, float)): + continue + top_logprobs = entry.get("top_logprobs") + converted_top: List[Dict[str, Any]] = [] + if isinstance(top_logprobs, list): + for top in top_logprobs: + if not isinstance(top, dict): + continue + converted_top.append( + { + "token": top.get("token"), + "logprob": top.get("logprob"), + } + ) + response_logprobs.append( + { + "token": token, + "logprob": float(logprob), + "top_logprobs": converted_top or None, + } + ) + return response_logprobs + + def _response_logprobs_from_completion( + self, + logprobs: Optional[CompletionLogprobs], + ) -> List[Dict[str, Any]]: + if logprobs is None: + return [] + response_logprobs: List[Dict[str, Any]] = [] + tokens = logprobs.tokens or [] + token_logprobs_list = logprobs.token_logprobs or [] + top_logprobs_list = logprobs.top_logprobs or [] + for token, token_logprob, top_logprobs in zip( + tokens, + token_logprobs_list, + top_logprobs_list, + ): + if token_logprob is None: + continue + converted_top: List[Dict[str, Any]] = [] + if top_logprobs is not None: + for top_token, top_logprob in top_logprobs.items(): + converted_top.append( + { + "token": top_token, + "logprob": float(top_logprob), + } + ) + response_logprobs.append( + { + "token": token, + "logprob": float(token_logprob), + "top_logprobs": converted_top or None, + } + ) + return response_logprobs + + @staticmethod + def _response_reasoning_item( + *, + item_id: str, + text: str, + status: str, + ) -> Dict[str, Any]: + return { + "id": item_id, + "type": "reasoning", + "summary": [], + "content": [ + { + "type": "reasoning_text", + "text": text, + } + ], + "status": status, + } + + @staticmethod + def _response_message_item( + *, + item_id: str, + text: str, + status: str, + phase: Optional[str], + logprobs: Optional[List[Dict[str, Any]]] = None, + ) -> Dict[str, Any]: + content_item: Dict[str, Any] = { + "type": "output_text", + "text": text, + "annotations": [], + } + if logprobs: + content_item["logprobs"] = logprobs + item: Dict[str, Any] = { + "id": item_id, + "type": "message", + "role": "assistant", + "content": [content_item], + "status": status, + } + if phase is not None: + item["phase"] = phase + return item + + @staticmethod + def _response_function_call_item( + *, + item_id: str, + call_id: str, + name: str, + arguments: str, + status: str, + ) -> Dict[str, Any]: + item: Dict[str, Any] = { + "id": item_id, + "type": "function_call", + "call_id": call_id, + "name": name, + "arguments": arguments, + "status": status, + } + if "." in name: + namespace, bare_name = name.split(".", 1) + item["namespace"] = namespace + item["name"] = bare_name + return item + + @staticmethod + def _response_custom_tool_call_item( + *, + item_id: str, + call_id: str, + name: str, + input_text: str, + ) -> Dict[str, Any]: + item: Dict[str, Any] = { + "id": item_id, + "type": "custom_tool_call", + "call_id": call_id, + "name": name, + "input": input_text, + } + if "." in name: + namespace, bare_name = name.split(".", 1) + item["namespace"] = namespace + item["name"] = bare_name + return item + + @staticmethod + def _responses_tool_type_by_name( + tools: Optional[List[Any]], + ) -> Dict[str, str]: + if tools is None: + return {} + tool_types: Dict[str, str] = {} + for tool in tools: + if isinstance(tool, BaseModel): + tool = tool.model_dump(mode="python", exclude_none=True) + if not isinstance(tool, dict): + continue + tool_type = tool.get("original_type") or tool.get("type") + if not isinstance(tool_type, str): + continue + function = tool.get("function") + if isinstance(function, dict): + name = function.get("name") + else: + name = tool.get("name") + if isinstance(name, str) and name: + tool_types[name] = tool_type + return tool_types + + def _response_output_items_from_message( + self, + *, + response_id: str, + message: Dict[str, Any], + logprobs: Optional[List[Dict[str, Any]]] = None, + tools: Optional[List[ChatTemplateTool]] = None, + ) -> List[Dict[str, Any]]: + items: List[Dict[str, Any]] = [] + tool_types_by_name = self._responses_tool_type_by_name(tools) + reasoning_text = self._response_reasoning_text_from_message(message) + if reasoning_text: + items.append( + self._response_reasoning_item( + item_id=f"rs_{response_id}_0", + text=reasoning_text, + status="completed", + ) + ) + tool_calls = message.get("tool_calls") + if isinstance(tool_calls, list): + for tool_call_index, tool_call in enumerate(tool_calls): + if not isinstance(tool_call, dict): + continue + function = tool_call.get("function") + if not isinstance(function, dict): + continue + name = function.get("name") + arguments = function.get("arguments") + if not isinstance(name, str) or not isinstance(arguments, str): + continue + call_id = tool_call.get("id") + if not isinstance(call_id, str) or not call_id: + call_id = f"call_{response_id}_{tool_call_index}" + tool_type = tool_types_by_name.get(name, "function") + if tool_type == "custom": + items.append( + self._response_custom_tool_call_item( + item_id=f"fc_{response_id}_{tool_call_index}", + call_id=call_id, + name=name, + input_text=arguments, + ) + ) + continue + items.append( + self._response_function_call_item( + item_id=f"fc_{response_id}_{tool_call_index}", + call_id=call_id, + name=name, + arguments=arguments, + status="completed", + ) + ) + output_text = self._response_output_text_from_message(message) + if output_text: + items.append( + self._response_message_item( + item_id=f"msg_{response_id}_0", + text=output_text, + status="completed", + phase=self._response_phase_from_message(message), + logprobs=logprobs, + ) + ) + return items + + def _responses_usage_from_completion( + self, + *, + usage: Optional[CompletionUsage], + output_items: Sequence[Dict[str, Any]], + ) -> Optional[Dict[str, Any]]: + if usage is None: + return None + reasoning_tokens = 0 + for item in output_items: + if item.get("type") != "reasoning": + continue + content = item.get("content") + if not isinstance(content, list): + continue + for part in content: + if isinstance(part, dict) and part.get("type") == "reasoning_text": + text = part.get("text") + if isinstance(text, str) and text: + reasoning_tokens += len( + self.model.tokenize(text, add_bos=False, special=True) + ) + return { + "input_tokens": usage.prompt_tokens, + "input_tokens_details": { + "cached_tokens": 0, + }, + "output_tokens": usage.completion_tokens, + "output_tokens_details": { + "reasoning_tokens": min(reasoning_tokens, usage.completion_tokens), + }, + "total_tokens": usage.total_tokens, + } + + def _response_status_and_incomplete_details( + self, + *, + finish_reason: Optional[str], + ) -> Tuple[str, Optional[Dict[str, str]]]: + if finish_reason == "length": + return "incomplete", {"reason": "max_output_tokens"} + return "completed", None + + def _response_object( + self, + *, + body: CreateResponseRequest, + response_id: str, + created_at: float, + model: str, + output_items: Sequence[Dict[str, Any]], + usage: Optional[Dict[str, Any]], + status: str, + incomplete_details: Optional[Dict[str, str]], + completed_at: Optional[float], + ) -> Dict[str, Any]: + return { + "id": response_id, + "object": "response", + "created_at": created_at, + "completed_at": completed_at, + "error": None, + "incomplete_details": incomplete_details, + "instructions": body.instructions, + "metadata": body.metadata, + "model": model, + "output": list(output_items), + "parallel_tool_calls": body.parallel_tool_calls, + "reasoning": { + "effort": self._response_reasoning_effort(body), + "summary": None, + }, + "store": False, + "temperature": body.temperature, + "tool_choice": ( + body.tool_choice.model_dump(mode="python", exclude_none=True) + if isinstance(body.tool_choice, BaseModel) + else body.tool_choice or "auto" + ), + "tools": ( + [ + tool.model_dump(mode="python", exclude_none=True) + if isinstance(tool, BaseModel) + else tool + for tool in body.tools + ] + if body.tools + else [] + ), + "top_p": body.top_p, + "max_output_tokens": body.max_output_tokens, + "previous_response_id": None, + "status": status, + "text": ( + body.text.model_dump(mode="python", exclude_none=True, by_alias=True) + if body.text is not None + else {"format": {"type": "text"}} + ), + "top_logprobs": body.top_logprobs, + "truncation": body.truncation, + "usage": usage, + "user": body.user, + } + + def _response_from_chat_message( + self, + *, + body: CreateResponseRequest, + response_id: str, + created_at: float, + model: str, + message: Dict[str, Any], + finish_reason: Optional[str], + usage: Optional[CompletionUsage], + logprobs: Optional[List[Dict[str, Any]]] = None, + ) -> Dict[str, Any]: + status, incomplete_details = self._response_status_and_incomplete_details( + finish_reason=finish_reason, + ) + output_items = self._response_output_items_from_message( + response_id=response_id, + message=message, + logprobs=logprobs, + tools=cast(Optional[List[Any]], body.tools), + ) + return self._response_object( + body=body, + response_id=response_id, + created_at=created_at, + model=model, + output_items=output_items, + usage=self._responses_usage_from_completion( + usage=usage, + output_items=output_items, + ), + status=status, + incomplete_details=incomplete_details, + completed_at=time.time() if status in {"completed", "incomplete"} else None, + ) + + def convert_completion_response_to_response( + self, + completion: OpenAICompletion, + body: CreateResponseRequest, + tool_name: Optional[str] = None, + *, + tools: Optional[List[ChatTemplateTool]] = None, + generation_prompt: str = "", + ) -> Dict[str, Any]: + chat_response = self.convert_completion_response_to_chat( + completion, + tool_name, + tools=tools, + generation_prompt=generation_prompt, + ) + if isinstance(chat_response, BaseModel): + chat_payload = chat_response.model_dump(mode="json", exclude_none=True) + else: + chat_payload = chat_response + choice = cast(Dict[str, Any], chat_payload["choices"][0]) + message = cast(Dict[str, Any], choice["message"]) + response_id = "resp_" + completion.id + logprobs = self._response_logprobs_from_completion(completion.choices[0].logprobs) + return self._response_from_chat_message( + body=body, + response_id=response_id, + created_at=float(completion.created), + model=completion.model, + message=message, + finish_reason=cast(Optional[str], choice.get("finish_reason")), + usage=completion.usage, + logprobs=logprobs or None, + ) + + @staticmethod + def _response_event( + state: "OpenAIFormatter.ResponsesStream", + event_type: str, + **payload: Any, + ) -> Dict[str, Any]: + state.sequence_number += 1 + event = { + "type": event_type, + "sequence_number": state.sequence_number, + } + event.update(payload) + return event + + def _response_stream_response( + self, + state: "OpenAIFormatter.ResponsesStream", + *, + status: str, + usage: Optional[Dict[str, Any]], + incomplete_details: Optional[Dict[str, str]], + completed_at: Optional[float], + ) -> Dict[str, Any]: + return self._response_object( + body=state.body, + response_id=state.response_id, + created_at=state.created_at, + model=state.model, + output_items=state.output, + usage=usage, + status=status, + incomplete_details=incomplete_details, + completed_at=completed_at, + ) + + def start_response_stream( + self, + state: "OpenAIFormatter.ResponsesStream", + ) -> List[Dict[str, Any]]: + if state.started: + return [] + state.started = True + response = self._response_stream_response( + state, + status="in_progress", + usage=None, + incomplete_details=None, + completed_at=None, + ) + return [ + self._response_event(state, "response.created", response=response), + self._response_event(state, "response.in_progress", response=response), + ] + + @staticmethod + def _response_item_status( + finish_reason: Optional[str], + ) -> str: + if finish_reason == "length": + return "incomplete" + return "completed" + + def _add_response_stream_item( + self, + state: "OpenAIFormatter.ResponsesStream", + item: Dict[str, Any], + *, + content_index: Optional[int] = None, + ) -> "OpenAIFormatter.ResponsesOutputItem": + item_state = OpenAIFormatter.ResponsesOutputItem( + output_index=len(state.output), + item=item, + content_index=content_index, + ) + state.output.append(item) + return item_state + + def _ensure_reasoning_stream_item( + self, + state: "OpenAIFormatter.ResponsesStream", + ) -> Tuple[List[Dict[str, Any]], "OpenAIFormatter.ResponsesOutputItem"]: + if state.reasoning_item is not None: + return [], state.reasoning_item + item = self._response_reasoning_item( + item_id=f"rs_{state.response_id}_{len(state.output)}", + text="", + status="in_progress", + ) + item_state = self._add_response_stream_item(state, item, content_index=0) + state.reasoning_item = item_state + part = cast(List[Dict[str, Any]], item["content"])[0] + return [ + self._response_event( + state, + "response.output_item.added", + output_index=item_state.output_index, + item=item, + ), + self._response_event( + state, + "response.content_part.added", + item_id=cast(str, item["id"]), + output_index=item_state.output_index, + content_index=0, + part=part, + ), + ], item_state + + def _ensure_message_stream_item( + self, + state: "OpenAIFormatter.ResponsesStream", + ) -> Tuple[List[Dict[str, Any]], "OpenAIFormatter.ResponsesOutputItem"]: + if state.message_item is not None: + return [], state.message_item + item = self._response_message_item( + item_id=f"msg_{state.response_id}_{len(state.output)}", + text="", + status="in_progress", + phase=cast(Optional[str], state.assistant_meta.get("phase")), + ) + item_state = self._add_response_stream_item(state, item, content_index=0) + state.message_item = item_state + part = cast(List[Dict[str, Any]], item["content"])[0] + return [ + self._response_event( + state, + "response.output_item.added", + output_index=item_state.output_index, + item=item, + ), + self._response_event( + state, + "response.content_part.added", + item_id=cast(str, item["id"]), + output_index=item_state.output_index, + content_index=0, + part=part, + ), + ], item_state + + def _ensure_tool_stream_item( + self, + state: "OpenAIFormatter.ResponsesStream", + *, + tool_call_index: int, + call_id: Optional[str], + name: Optional[str], + ) -> Tuple[List[Dict[str, Any]], "OpenAIFormatter.ResponsesOutputItem"]: + existing = state.tool_items.get(tool_call_index) + if existing is not None: + return [], existing + tool_types_by_name = self._responses_tool_type_by_name(state.body.tools) + tool_type = tool_types_by_name.get(name or "", "function") + item_id = f"fc_{state.response_id}_{tool_call_index}" + resolved_call_id = call_id or f"call_{state.response_id}_{tool_call_index}" + resolved_name = name or "" + if tool_type == "custom": + item = self._response_custom_tool_call_item( + item_id=item_id, + call_id=resolved_call_id, + name=resolved_name, + input_text="", + ) + else: + item = self._response_function_call_item( + item_id=item_id, + call_id=resolved_call_id, + name=resolved_name, + arguments="", + status="in_progress", + ) + item_state = self._add_response_stream_item(state, item) + state.tool_items[tool_call_index] = item_state + return [ + self._response_event( + state, + "response.output_item.added", + output_index=item_state.output_index, + item=item, + ) + ], item_state + + def _update_tool_stream_item( + self, + item: Dict[str, Any], + *, + call_id: Optional[str], + name_delta: Optional[str], + arguments_delta: Optional[str], + ) -> None: + if isinstance(call_id, str) and call_id: + item["call_id"] = call_id + if isinstance(name_delta, str) and name_delta: + current_name = cast(str, item.get("name", "")) + if not current_name: + item["name"] = name_delta + elif name_delta == current_name or name_delta.startswith(current_name): + item["name"] = name_delta + elif not current_name.endswith(name_delta): + item["name"] = current_name + name_delta + if isinstance(arguments_delta, str) and arguments_delta: + if item.get("type") == "custom_tool_call": + raw_arguments = cast(str, item.get("_raw_arguments", "")) + arguments_delta + item["_raw_arguments"] = raw_arguments + normalized_input = self._normalize_text_tool_payload(raw_arguments) + if normalized_input is not None: + item["input"] = normalized_input + else: + item["arguments"] = ( + cast(str, item.get("arguments", "")) + arguments_delta + ) + + @staticmethod + def _normalize_text_tool_payload(payload: str) -> Optional[str]: + if payload == "": + return "" + stripped = payload.lstrip() + if not stripped: + return "" + if stripped[0] not in '{["': + return payload + try: + decoded = json.loads(payload) + except Exception: + return None + if isinstance(decoded, str): + return decoded + if isinstance(decoded, dict): + input_value = decoded.get("input") + if isinstance(input_value, str): + return input_value + if len(decoded) == 1: + sole_value = next(iter(decoded.values())) + if isinstance(sole_value, str): + return sole_value + return payload + + def _finalize_response_stream_items( + self, + state: "OpenAIFormatter.ResponsesStream", + *, + finish_reason: Optional[str], + ) -> List[Dict[str, Any]]: + events: List[Dict[str, Any]] = [] + item_status = self._response_item_status(finish_reason) + + if ( + state.reasoning_item is not None + and state.reasoning_item.item["status"] == "in_progress" + ): + item = state.reasoning_item.item + item["status"] = item_status + part = cast(List[Dict[str, Any]], item["content"])[0] + events.append( + self._response_event( + state, + "response.reasoning_text.done", + item_id=cast(str, item["id"]), + output_index=state.reasoning_item.output_index, + content_index=cast(int, state.reasoning_item.content_index), + text=part["text"], + ) + ) + events.append( + self._response_event( + state, + "response.content_part.done", + item_id=cast(str, item["id"]), + output_index=state.reasoning_item.output_index, + content_index=cast(int, state.reasoning_item.content_index), + part=part, + ) + ) + events.append( + self._response_event( + state, + "response.output_item.done", + output_index=state.reasoning_item.output_index, + item=item, + ) + ) + + if ( + state.message_item is not None + and state.message_item.item["status"] == "in_progress" + ): + item = state.message_item.item + item["status"] = item_status + part = cast(List[Dict[str, Any]], item["content"])[0] + events.append( + self._response_event( + state, + "response.output_text.done", + item_id=cast(str, item["id"]), + output_index=state.message_item.output_index, + content_index=cast(int, state.message_item.content_index), + text=part["text"], + logprobs=part.get("logprobs", []), + ) + ) + events.append( + self._response_event( + state, + "response.content_part.done", + item_id=cast(str, item["id"]), + output_index=state.message_item.output_index, + content_index=cast(int, state.message_item.content_index), + part=part, + ) + ) + events.append( + self._response_event( + state, + "response.output_item.done", + output_index=state.message_item.output_index, + item=item, + ) + ) + + for tool_call_index in sorted(state.tool_items): + item_state = state.tool_items[tool_call_index] + item = item_state.item + item.pop("_raw_arguments", None) + if ( + item.get("status") != "in_progress" + and item.get("type") != "custom_tool_call" + ): + continue + if item.get("type") == "custom_tool_call": + events.append( + self._response_event( + state, + "response.custom_tool_call_input.done", + item_id=cast(str, item["id"]), + output_index=item_state.output_index, + input=cast(str, item.get("input", "")), + ) + ) + else: + item["status"] = item_status + events.append( + self._response_event( + state, + "response.function_call_arguments.done", + item_id=cast(str, item["id"]), + output_index=item_state.output_index, + name=cast(str, item["name"]), + arguments=cast(str, item["arguments"]), + ) + ) + events.append( + self._response_event( + state, + "response.output_item.done", + output_index=item_state.output_index, + item=item, + ) + ) + + state.final_status, state.incomplete_details = ( + self._response_status_and_incomplete_details( + finish_reason=finish_reason, + ) + ) + return events + + def convert_chat_chunk_to_response_events( + self, + chunk: ChatCompletionChunk | Dict[str, Any], + state: "OpenAIFormatter.ResponsesStream", + ) -> List[Dict[str, Any]]: + payload = ( + chunk.model_dump(mode="json", exclude_none=True) + if isinstance(chunk, BaseModel) + else chunk + ) + events = self.start_response_stream(state) + choice = cast(Dict[str, Any], payload["choices"][0]) + delta = cast(Dict[str, Any], choice.get("delta", {})) + finish_reason = cast(Optional[str], choice.get("finish_reason")) + logprobs = self._response_logprobs_from_chat_logprobs( + cast(Optional[Dict[str, Any]], choice.get("logprobs")) + ) + + phase = delta.get("phase") + if isinstance(phase, str): + state.assistant_meta["phase"] = phase + if state.message_item is not None: + state.message_item.item["phase"] = phase + + reasoning_delta = delta.get("reasoning_content") + if not isinstance(reasoning_delta, str) or not reasoning_delta: + candidate = delta.get("thinking") + reasoning_delta = candidate if isinstance(candidate, str) else None + if isinstance(reasoning_delta, str) and reasoning_delta: + added, item_state = self._ensure_reasoning_stream_item(state) + events.extend(added) + part = cast(List[Dict[str, Any]], item_state.item["content"])[0] + part["text"] += reasoning_delta + events.append( + self._response_event( + state, + "response.reasoning_text.delta", + item_id=cast(str, item_state.item["id"]), + output_index=item_state.output_index, + content_index=cast(int, item_state.content_index), + delta=reasoning_delta, + ) + ) + + content_delta = delta.get("content") + if isinstance(content_delta, str) and content_delta: + added, item_state = self._ensure_message_stream_item(state) + events.extend(added) + part = cast(List[Dict[str, Any]], item_state.item["content"])[0] + part["text"] += content_delta + if logprobs: + cast(List[Dict[str, Any]], part.setdefault("logprobs", [])).extend( + logprobs + ) + events.append( + self._response_event( + state, + "response.output_text.delta", + item_id=cast(str, item_state.item["id"]), + output_index=item_state.output_index, + content_index=cast(int, item_state.content_index), + delta=content_delta, + logprobs=logprobs, + ) + ) + + tool_calls = delta.get("tool_calls") + if isinstance(tool_calls, list): + for tool_call in tool_calls: + if not isinstance(tool_call, dict): + continue + tool_call_index = tool_call.get("index", 0) + if not isinstance(tool_call_index, int): + continue + function = tool_call.get("function") + if not isinstance(function, dict): + continue + added, item_state = self._ensure_tool_stream_item( + state, + tool_call_index=tool_call_index, + call_id=cast(Optional[str], tool_call.get("id")), + name=cast(Optional[str], function.get("name")), + ) + events.extend(added) + previous_input = cast(str, item_state.item.get("input", "")) + self._update_tool_stream_item( + item_state.item, + call_id=cast(Optional[str], tool_call.get("id")), + name_delta=cast(Optional[str], function.get("name")), + arguments_delta=cast(Optional[str], function.get("arguments")), + ) + arguments_delta = function.get("arguments") + if isinstance(arguments_delta, str) and arguments_delta: + if item_state.item.get("type") == "custom_tool_call": + current_input = cast(str, item_state.item.get("input", "")) + if not current_input or current_input == previous_input: + continue + delta_text = current_input + if current_input.startswith(previous_input): + delta_text = current_input[len(previous_input) :] + events.append( + self._response_event( + state, + "response.custom_tool_call_input.delta", + item_id=cast(str, item_state.item["id"]), + output_index=item_state.output_index, + delta=delta_text, + ) + ) + continue + events.append( + self._response_event( + state, + "response.function_call_arguments.delta", + item_id=cast(str, item_state.item["id"]), + output_index=item_state.output_index, + delta=arguments_delta, + ) + ) + + if finish_reason is not None: + events.extend( + self._finalize_response_stream_items( + state, + finish_reason=finish_reason, + ) + ) + return events + + def response_stream_terminal_events( + self, + state: "OpenAIFormatter.ResponsesStream", + completion: Optional[OpenAICompletion], + ) -> List[Dict[str, Any]]: + if not state.started: + state.started = True + if completion is not None and state.final_status is None: + finish_reason = None + if completion.choices: + finish_reason = completion.choices[0].finish_reason + self._finalize_response_stream_items(state, finish_reason=finish_reason) + status = state.final_status or "completed" + response = self._response_stream_response( + state, + status=status, + usage=( + self._responses_usage_from_completion( + usage=completion.usage if completion is not None else None, + output_items=state.output, + ) + ), + incomplete_details=state.incomplete_details, + completed_at=time.time() if status in {"completed", "incomplete"} else None, + ) + event_type = "response.incomplete" if status == "incomplete" else "response.completed" + return [self._response_event(state, event_type, response=response)] + + def aggregate_completion_results( + self, + results: Sequence[OpenAICompletion], + ) -> OpenAICompletion: + choices: List[CompletionChoice] = [] + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + for result in results: + for choice in result.choices: + choices.append(choice.model_copy(update={"index": len(choices)})) + assert result.usage is not None + prompt_tokens += result.usage.prompt_tokens + completion_tokens += result.usage.completion_tokens + total_tokens += result.usage.total_tokens + return OpenAICompletion( + id=f"cmpl-{uuid.uuid4().hex}", + object="text_completion", + created=int(time.time()), + model=self.model.model_path, + choices=choices, + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ), + ) + + @staticmethod + def _convert_completion_logprobs_to_chat_choice( + logprobs: Optional[CompletionLogprobs], + ) -> Optional[ChatCompletionChoiceLogprobs]: + if logprobs is None: + return None + tokens = logprobs.tokens or [] + token_logprobs_list = logprobs.token_logprobs or [] + top_logprobs_list = logprobs.top_logprobs or [] + return ChatCompletionChoiceLogprobs( + content=[ + ChatCompletionTokenLogprob( + token=token, + bytes=None, + logprob=token_logprob if token_logprob is not None else 0.0, + top_logprobs=( + [ + TopLogprob( + token=top_token, + logprob=top_logprob, + bytes=None, + ) + for top_token, top_logprob in top_logprobs.items() + ] + if top_logprobs is not None + else [] + ), + ) + for token, token_logprob, top_logprobs in zip( + tokens, + token_logprobs_list, + top_logprobs_list, + ) + ], + refusal=None, + ) + + @staticmethod + def _convert_completion_logprobs_to_chat_chunk( + logprobs: Optional[CompletionLogprobs], + ) -> Optional[ChatCompletionChunkChoiceLogprobs]: + choice_logprobs = OpenAIFormatter._convert_completion_logprobs_to_chat_choice(logprobs) + if choice_logprobs is None: + return None + return ChatCompletionChunkChoiceLogprobs.model_validate( + choice_logprobs.model_dump(mode="python", exclude_none=True) + ) + + @staticmethod + def _chat_message_from_completion_choice( + completion_id: str, + choice: CompletionChoice, + tool_name: Optional[str], + ) -> ChatCompletionMessage: + if tool_name is None: + return ChatCompletionMessage( + role="assistant", + content=choice.text, + ) + tool_id = f"call_{choice.index}_{tool_name}_{completion_id}" + arguments = choice.text + return ChatCompletionMessage( + role="assistant", + content=None, + function_call=ChatCompletionMessageFunctionCall( + name=tool_name, + arguments=arguments, + ), + tool_calls=[ + ChatCompletionMessageToolCall( + id=tool_id, + type="function", + function=ChatCompletionMessageToolCallFunction( + name=tool_name, + arguments=arguments, + ), + ) + ], + ) + + @staticmethod + def _chat_chunk_payload( + *, + chunk_id: str, + created: int, + model: str, + index: int, + delta: Dict[str, Any], + finish_reason: Optional[str], + logprobs: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + choice: Dict[str, Any] = { + "index": index, + "delta": delta, + "finish_reason": finish_reason, + } + if logprobs is not None: + choice["logprobs"] = logprobs + return { + "id": "chat" + chunk_id, + "object": "chat.completion.chunk", + "created": created, + "model": model, + "choices": [choice], + } + + def convert_completion_response_to_chat( + self, + completion: OpenAICompletion, + tool_name: Optional[str] = None, + *, + functions: Optional[List[ChatTemplateFunctionDefinition]] = None, + tools: Optional[List[ChatTemplateTool]] = None, + generation_prompt: str = "", + ) -> ChatCompletion | Dict[str, Any]: + parser_tools = self._tools_for_response_parser( + functions=functions, + tools=tools, + ) + if self.model.response_schema is not None: + choices: List[Dict[str, Any]] = [] + for choice in completion.choices: + parser = self._response_parser( + tools=parser_tools, + completion_id=completion.id, + choice_index=choice.index, + generation_prompt=generation_prompt, + ) + message = parser.parse_completion_message(choice.text) + logprobs = self._convert_completion_logprobs_to_chat_choice(choice.logprobs) + choices.append( + { + "index": choice.index, + "message": message, + "logprobs": ( + logprobs.model_dump(mode="json", exclude_none=True) + if logprobs is not None + else None + ), + "finish_reason": ( + "tool_calls" + if message.get("tool_calls") + else choice.finish_reason + ), + } + ) + return { + "id": "chat" + completion.id, + "object": "chat.completion", + "created": completion.created, + "model": completion.model, + "choices": choices, + "usage": ( + completion.usage.model_dump(mode="json", exclude_none=True) + if completion.usage is not None + else None + ), + } + return ChatCompletion( + id="chat" + completion.id, + object="chat.completion", + created=completion.created, + model=completion.model, + choices=[ + ChatCompletionChoice( + index=choice.index, + message=self._chat_message_from_completion_choice( + completion.id, + choice, + tool_name, + ), + logprobs=self._convert_completion_logprobs_to_chat_choice(choice.logprobs), + finish_reason=cast(Any, ( + "tool_calls" if tool_name is not None else choice.finish_reason + )), + ) + for choice in completion.choices + ], + usage=completion.usage, + ) + + def convert_completion_chunk_to_chat_chunks( + self, + chunk: CompletionChunk, + started_indices: set[int], + tool_name: Optional[str] = None, + *, + functions: Optional[List[ChatTemplateFunctionDefinition]] = None, + tools: Optional[List[ChatTemplateTool]] = None, + parsed_states: Optional[Dict[int, Any]] = None, + generation_prompt: str = "", + ) -> List[ChatCompletionChunk | Dict[str, Any]]: + parser_tools = self._tools_for_response_parser( + functions=functions, + tools=tools, + ) + if self.model.response_schema is not None: + parsed_chunks: List[Dict[str, Any]] = [] + if parsed_states is None: + parsed_states = {} + for choice in chunk["choices"]: + index = choice["index"] + parser = parsed_states.get(index) + if not isinstance(parser, ResponseParser): + parser = self._response_parser( + tools=parser_tools, + completion_id=chunk["id"], + choice_index=index, + generation_prompt=generation_prompt, + ) + parsed_states[index] = parser + logprobs = self._convert_completion_logprobs_to_chat_chunk( + CompletionLogprobs.model_validate(choice["logprobs"]) + if choice["logprobs"] is not None + else None + ) + parsed_chunks.extend( + parser.consume_completion_chunk( + choice["text"], + chunk_id=chunk["id"], + created=chunk["created"], + model=chunk["model"], + finish_reason=choice["finish_reason"], + logprobs=( + logprobs.model_dump(mode="json", exclude_none=True) + if logprobs is not None + else None + ), + ) + ) + if parser.started: + started_indices.add(index) + return cast(List[ChatCompletionChunk | Dict[str, Any]], parsed_chunks) + chat_chunks: List[ChatCompletionChunk] = [] + for choice in chunk["choices"]: + index = choice["index"] + if index not in started_indices: + started_indices.add(index) + chat_chunks.append( + ChatCompletionChunk( + id="chat" + chunk["id"], + object="chat.completion.chunk", + created=chunk["created"], + model=chunk["model"], + choices=[ + ChatCompletionChunkChoice( + index=index, + delta=( + ChoiceDelta( + role="assistant", + content=None, + function_call=None, + tool_calls=None, + ) + if tool_name is not None + else ChoiceDelta(role="assistant") + ), + logprobs=None, + finish_reason=None, + ) + ], + ) + ) + if tool_name is not None: + delta: ChoiceDelta + if choice["finish_reason"] is None: + tool_id = f"call_{index}_{tool_name}_{chunk['id']}" + delta = ChoiceDelta( + role=None, + content=None, + function_call=ChoiceDeltaFunctionCall( + name=tool_name, + arguments=choice["text"], + ), + tool_calls=[ + ChoiceDeltaToolCall( + index=0, + id=tool_id, + type="function", + function=ChoiceDeltaToolCallFunction( + name=tool_name, + arguments=choice["text"], + ), + ) + ], + ) + else: + delta = ChoiceDelta( + role=None, + content=None, + function_call=None, + tool_calls=None, + ) + chat_chunks.append( + ChatCompletionChunk( + id="chat" + chunk["id"], + object="chat.completion.chunk", + created=chunk["created"], + model=chunk["model"], + choices=[ + ChatCompletionChunkChoice( + index=index, + delta=delta, + logprobs=self._convert_completion_logprobs_to_chat_chunk( + CompletionLogprobs.model_validate(choice["logprobs"]) + if choice["logprobs"] is not None + else None + ), + finish_reason=( + "tool_calls" + if choice["finish_reason"] is not None + else None + ), + ) + ], + ) + ) + continue + chat_chunks.append( + ChatCompletionChunk( + id="chat" + chunk["id"], + object="chat.completion.chunk", + created=chunk["created"], + model=chunk["model"], + choices=[ + ChatCompletionChunkChoice( + index=index, + delta=( + ChoiceDelta(content=choice["text"]) + if choice["finish_reason"] is None + else ChoiceDelta() + ), + logprobs=self._convert_completion_logprobs_to_chat_chunk( + CompletionLogprobs.model_validate(choice["logprobs"]) + if choice["logprobs"] is not None + else None + ), + finish_reason=cast(Any, choice["finish_reason"]), + ) + ], + ) + ) + return cast(List[ChatCompletionChunk | Dict[str, Any]], chat_chunks) + + def returned_output_end( + self, + completion: Completion, + finish_reason: Optional[str], + ) -> int: + completion_bytes: bytes | bytearray + if completion.rendered_bytes: + completion_bytes = completion.rendered_bytes + else: + completion_bytes = b"".join(record.text_bytes for record in completion.token_records) + returned_end = len(completion_bytes) + if finish_reason == "stop": + stops = [stop for stop in completion.stop_sequences if stop in completion_bytes] + if stops: + returned_end = min(completion_bytes.index(stop) for stop in stops) + elif finish_reason is None: + holdback = 0 + for stop in completion.stop_sequences: + for size in range(min(len(stop), returned_end), 0, -1): + if completion_bytes.endswith(stop[:size]): + holdback = max(holdback, size) + break + returned_end -= holdback + return returned_end + + def returned_tokens( + self, + completion: Completion, + finish_reason: Optional[str], + *, + start_index: int = 0, + ) -> List["OpenAIFormatter.ReturnedToken"]: + returned_end = self.returned_output_end(completion, finish_reason) + returned_tokens: List[OpenAIFormatter.ReturnedToken] = [] + prefix_bytes = b"" + for index, record in enumerate(completion.token_records): + token_start = len(prefix_bytes) + if token_start >= returned_end: + break + token_end = token_start + len(record.text_bytes) + text_bytes = record.text_bytes + if token_end > returned_end: + if finish_reason is None: + break + text_bytes = text_bytes[: returned_end - token_start] + if index >= start_index: + returned_tokens.append( + OpenAIFormatter.ReturnedToken( + index=index, + text_bytes=text_bytes, + token=record, + text_offset=len(self.decode_text(prefix_bytes)), + ) + ) + prefix_bytes += record.text_bytes + return returned_tokens + + def stream_completion_chunks( + self, + request: CompletionRequest, + completion: Completion, + finish_reason: Optional[str], + ) -> List[CompletionChunk]: + returned_tokens = self.returned_tokens( + completion, + finish_reason, + start_index=completion.returned_token_count, + ) + chunks: List[CompletionChunk] = [] + if completion.logprobs is not None: + for returned_token in returned_tokens: + token = returned_token.token + chunks.append( + { + "id": request.id, + "object": "text_completion", + "created": request.created, + "model": self.model.model_path, + "choices": [ + { + "text": self.decode_text(returned_token.text_bytes), + "index": completion.index, + "logprobs": { + "tokens": [self.decode_text(token.text_bytes)], + "text_offset": [ + len(completion.prompt_text) + returned_token.text_offset + ], + "token_logprobs": [token.token_logprob], + "top_logprobs": [token.top_logprobs], + }, + "finish_reason": None, + } + ], + } + ) + completion.returned_token_count = returned_token.index + 1 + return chunks + + chunk_tokens: List[OpenAIFormatter.ReturnedToken] = [] + for returned_token in returned_tokens: + chunk_tokens.append(returned_token) + chunk_bytes = b"".join(token.text_bytes for token in chunk_tokens) + if returned_token.text_bytes != returned_token.token.text_bytes: + chunks.append( + { + "id": request.id, + "object": "text_completion", + "created": request.created, + "model": self.model.model_path, + "choices": [ + { + "text": self.decode_text(returned_token.text_bytes), + "index": completion.index, + "logprobs": None, + "finish_reason": None, + } + ], + } + ) + completion.returned_token_count = returned_token.index + 1 + chunk_tokens = [] + continue + try: + text = chunk_bytes.decode("utf-8") + except UnicodeError: + continue + chunks.append( + { + "id": request.id, + "object": "text_completion", + "created": request.created, + "model": self.model.model_path, + "choices": [ + { + "text": text, + "index": completion.index, + "logprobs": None, + "finish_reason": None, + } + ], + } + ) + completion.returned_token_count = returned_token.index + 1 + chunk_tokens = [] + return chunks + + def completion_finish_chunk( + self, + request: CompletionRequest, + completion: Completion, + finish_reason: str, + ) -> CompletionChunk: + return { + "id": request.id, + "object": "text_completion", + "created": request.created, + "model": self.model.model_path, + "choices": [ + { + "text": "", + "index": completion.index, + "logprobs": None, + "finish_reason": finish_reason, + } + ], + } + + def _build_completion_choice( + self, + request: CompletionRequest, + completion: Completion, + ) -> CompletionChoice: + returned_tokens = self.returned_tokens(completion, completion.finish_reason) + text_bytes = b"".join(returned_token.text_bytes for returned_token in returned_tokens) + text = self.decode_text(text_bytes) + if request.payload.echo: + text = completion.prompt_text + text + logprobs: Optional[CompletionLogprobs] = None + if completion.logprobs is not None: + offsets: List[int] = [] + token_texts: List[str] = [] + token_logprobs: List[Optional[float]] = [] + top_logprobs: List[Optional[Dict[str, float]]] = [] + text_cursor = request.prompt_text if not request.payload.echo else "" + if request.payload.echo: + for record in request.prompt_records: + offsets.append(len(text_cursor)) + token_texts.append(self.decode_text(record.text_bytes)) + token_logprobs.append(record.token_logprob) + top_logprobs.append(record.top_logprobs) + text_cursor += self.decode_text(record.text_bytes) + text_cursor = request.prompt_text + for returned_token in returned_tokens: + token = returned_token.token + offsets.append(len(completion.prompt_text) + returned_token.text_offset) + if request.payload.echo: + offsets[-1] = len(text_cursor) + token_texts.append(self.decode_text(token.text_bytes)) + token_logprobs.append(token.token_logprob) + top_logprobs.append(token.top_logprobs) + text_cursor += self.decode_text(token.text_bytes) + logprobs = CompletionLogprobs.model_construct( + text_offset=offsets, + token_logprobs=token_logprobs, + tokens=token_texts, + top_logprobs=top_logprobs, + ) + return CompletionChoice( + text=text, + index=completion.index, + logprobs=logprobs, + finish_reason=cast(Any, completion.finish_reason), + ) + + def build_completion_response( + self, + request: CompletionRequest, + completions: Sequence[Completion], + ) -> OpenAICompletion: + completion_tokens = sum(completion.completion_token_count for completion in completions) + prompt_tokens = request.prompt_plan.eval_token_count + return OpenAICompletion( + id=request.id, + object="text_completion", + created=request.created, + model=self.model.model_path, + choices=[ + self._build_completion_choice(request, completion) + for completion in completions + ], + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +class Sampler: + TOKEN_DATA_DTYPE = np.dtype( + [("id", np.intc), ("logit", np.single), ("p", np.single)], + align=True, + ) + + def __init__( + self, + *, + seed: int, + vocab: llama_cpp.llama_vocab_p, + n_vocab: int, + top_p: float, + temperature: float, + frequency_penalty: float, + presence_penalty: float, + logit_bias: Optional[Dict[int, float]], + grammar_text: Optional[str] = None, + grammar_root: str = "root", + ) -> None: + params = llama_cpp.llama_sampler_chain_default_params() + self._sampler = llama_cpp.llama_sampler_chain_init(params) + self._closed = False + self._sample_logits_size = 0 + self._sample_logits_token_data: Optional[Any] = None + self._sample_logits_token_array: Optional[Any] = None + self._sample_logits_recarray: Optional[np.recarray] = None + if logit_bias: + bias_array = (llama_cpp.llama_logit_bias * len(logit_bias))() + for index, (token, bias) in enumerate(logit_bias.items()): + bias_array[index].token = ctypes.c_int32(token) + bias_array[index].bias = float(bias) + llama_cpp.llama_sampler_chain_add( + self._sampler, + llama_cpp.llama_sampler_init_logit_bias( + n_vocab, len(logit_bias), bias_array + ), + ) + self.bias_array = bias_array + if frequency_penalty != 0.0 or presence_penalty != 0.0: + llama_cpp.llama_sampler_chain_add( + self._sampler, + llama_cpp.llama_sampler_init_penalties( + 64, + 1.0, + frequency_penalty, + presence_penalty, + ), + ) + if grammar_text is not None: + grammar_sampler = llama_cpp.llama_sampler_init_grammar( + vocab, + grammar_text.encode("utf-8"), + grammar_root.encode("utf-8"), + ) + if grammar_sampler is None: + raise RuntimeError("failed to initialize grammar sampler") + llama_cpp.llama_sampler_chain_add(self._sampler, grammar_sampler) + if temperature < 0.0: + llama_cpp.llama_sampler_chain_add( + self._sampler, llama_cpp.llama_sampler_init_dist(seed) + ) + return + if temperature == 0.0: + llama_cpp.llama_sampler_chain_add( + self._sampler, llama_cpp.llama_sampler_init_greedy() + ) + return + min_keep = 1 + llama_cpp.llama_sampler_chain_add( + self._sampler, llama_cpp.llama_sampler_init_top_p(top_p, min_keep) + ) + llama_cpp.llama_sampler_chain_add( + self._sampler, llama_cpp.llama_sampler_init_temp(temperature) + ) + llama_cpp.llama_sampler_chain_add( + self._sampler, llama_cpp.llama_sampler_init_dist(seed) + ) + + def sample(self, ctx: llama_cpp.llama_context_p, output_index: int) -> int: + return int(llama_cpp.llama_sampler_sample(self._sampler, ctx, output_index)) + + def _ensure_sample_logits_buffer(self, size: int) -> None: + if size == self._sample_logits_size and self._sample_logits_recarray is not None: + return + token_data = (llama_cpp.llama_token_data * size)() + token_data_address = ctypes.addressof(token_data) + recarray = np.recarray( + shape=(size,), + dtype=self.TOKEN_DATA_DTYPE, + buf=cast( + Any, + (llama_cpp.llama_token_data * size).from_address(token_data_address), + ), + ) + recarray.id[:] = np.arange(size, dtype=np.intc) + token_array = llama_cpp.llama_token_data_array( + data=token_data, + size=size, + selected=-1, + sorted=False, + ) + self._sample_logits_size = size + self._sample_logits_token_data = token_data + self._sample_logits_token_array = token_array + self._sample_logits_recarray = recarray + + def sample_logits(self, logits: np.ndarray) -> int: + self._ensure_sample_logits_buffer(len(logits)) + assert self._sample_logits_recarray is not None + assert self._sample_logits_token_array is not None + self._sample_logits_recarray.logit[:] = logits + self._sample_logits_recarray.p.fill(0.0) + self._sample_logits_token_array.selected = -1 + self._sample_logits_token_array.sorted = False + llama_cpp.llama_sampler_apply( + self._sampler, + cast(Any, ctypes.byref(self._sample_logits_token_array)), + ) + token = int(self._sample_logits_recarray.id[self._sample_logits_token_array.selected]) + llama_cpp.llama_sampler_accept(self._sampler, token) + return token + + def close(self) -> None: + if not self._closed: + llama_cpp.llama_sampler_free(self._sampler) + self._closed = True + + +@dataclass +class MTMDEmbedding: + key: str + embeddings: np.ndarray + + +class MTMDEmbeddingCache: + _metadata_version = "1" + + def __init__( + self, + *, + path: str, + max_bytes: int, + model_fingerprint: str, + mmproj_fingerprint: str, + ) -> None: + self.path = Path(path) + self.max_bytes = max_bytes + self.model_fingerprint = model_fingerprint + self.mmproj_fingerprint = mmproj_fingerprint + self.path.mkdir(parents=True, exist_ok=True) + + @staticmethod + def _safe_open(path: Path) -> Any: + try: + from safetensors import safe_open + except ImportError as exc: + raise RuntimeError( + "model.mtmd.embedding_cache requires safetensors. " + "Install it with `pip install safetensors`." + ) from exc + return safe_open(str(path), framework="numpy") + + @staticmethod + def _save_file( + tensors: Dict[str, np.ndarray], + path: Path, + metadata: Dict[str, str], + ) -> None: + from safetensors.numpy import save_file + + save_file(tensors, str(path), metadata=metadata) + + @staticmethod + def fingerprint_file(path: str) -> str: + stat = os.stat(path) + payload = f"{Path(path).resolve()}:{stat.st_size}:{stat.st_mtime_ns}" + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + @staticmethod + def _build_key( + *, + model_fingerprint: str, + mmproj_fingerprint: str, + kind: Literal["image", "audio"], + media_bytes: bytes, + ) -> str: + digest = hashlib.sha256(f"{kind}:".encode("utf-8") + media_bytes).hexdigest() + payload = ":".join( + [ + MTMDEmbeddingCache._metadata_version, + model_fingerprint, + mmproj_fingerprint, + kind, + digest, + ] + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + def key_for_media(self, kind: Literal["image", "audio"], media_bytes: bytes) -> str: + return self._build_key( + model_fingerprint=self.model_fingerprint, + mmproj_fingerprint=self.mmproj_fingerprint, + kind=kind, + media_bytes=media_bytes, + ) + + def _path_for_key(self, key: str) -> Path: + return self.path / f"{key}.safetensors" + + def load(self, key: str) -> Optional[MTMDEmbedding]: + entry_path = self._path_for_key(key) + if not entry_path.exists(): + return None + with self._safe_open(entry_path) as tensors: + metadata = tensors.metadata() + if metadata.get("version") != self._metadata_version: + return None + if metadata.get("model") != self.model_fingerprint: + return None + if metadata.get("mmproj") != self.mmproj_fingerprint: + return None + embeddings = np.array(tensors.get_tensor("embeddings"), copy=True) + return MTMDEmbedding(key=key, embeddings=embeddings.astype(np.float32, copy=False)) + + def save(self, key: str, embeddings: np.ndarray) -> None: + if self.max_bytes == 0: + return + tmp_path = self.path / f".{key}.{uuid.uuid4().hex}.tmp" + final_path = self._path_for_key(key) + metadata = { + "version": self._metadata_version, + "model": self.model_fingerprint, + "mmproj": self.mmproj_fingerprint, + "key": key, + } + self._save_file( + {"embeddings": np.ascontiguousarray(embeddings, dtype=np.float32)}, + tmp_path, + metadata, + ) + os.replace(tmp_path, final_path) + self.evict_if_needed() + + def evict_if_needed(self) -> None: + if self.max_bytes <= 0: + return + entries = [path for path in self.path.glob("*.safetensors") if path.is_file()] + total = sum(path.stat().st_size for path in entries) + if total <= self.max_bytes: + return + for path in sorted(entries, key=lambda item: item.stat().st_mtime_ns): + if total <= self.max_bytes: + break + try: + size = path.stat().st_size + path.unlink() + total -= size + except OSError: + continue + + +class MTMDProcessor: + def __init__( + self, + *, + model_path: str, + llama_model: Any, + chat_formatter: Jinja2ChatFormatter, + tokenize: Callable[..., List[int]], + n_embd_inp: int, + n_batch: int, + n_ubatch: int, + n_threads_batch: int, + mmproj_path: str, + embedding_cache: Optional[MTMDEmbeddingCache], + allowed_media_domains: Optional[List[str]], + allowed_local_media_path: Optional[str], + image_max_bytes: int, + audio_max_bytes: int, + image_timeout_seconds: float, + ) -> None: + self.chat_formatter = chat_formatter + self.tokenize = tokenize + self.n_embd_inp = n_embd_inp + self.n_batch = n_batch + self.n_ubatch = n_ubatch + self.mmproj_path = mmproj_path + self.embedding_cache = embedding_cache + self.model_fingerprint = MTMDEmbeddingCache.fingerprint_file(model_path) + self.mmproj_fingerprint = MTMDEmbeddingCache.fingerprint_file(mmproj_path) + self.allowed_media_domains = ( + {domain.lower() for domain in allowed_media_domains} + if allowed_media_domains is not None + else set() + ) + self.allowed_local_media_path = ( + Path(allowed_local_media_path).expanduser().resolve() + if allowed_local_media_path is not None + else None + ) + self.image_max_bytes = image_max_bytes + self.audio_max_bytes = audio_max_bytes + self.image_timeout_seconds = image_timeout_seconds + self.lock = threading.Lock() + params = mtmd_cpp.mtmd_context_params_default() + params.n_threads = max(1, n_threads_batch) + self.ctx = mtmd_cpp.mtmd_init_from_file( + mmproj_path.encode("utf-8"), + llama_model, + params, + ) + if self.ctx is None: + raise RuntimeError(f"failed to load MTMD context: {mmproj_path}") + self.supports_vision = bool(mtmd_cpp.mtmd_support_vision(self.ctx)) + self.supports_audio = bool(mtmd_cpp.mtmd_support_audio(self.ctx)) + if not self.supports_vision and not self.supports_audio: + mtmd_cpp.mtmd_free(self.ctx) + self.ctx = None + raise RuntimeError(f"MTMD projector does not support image or audio input: {mmproj_path}") + self.media_marker = mtmd_cpp.mtmd_default_marker().decode("utf-8") + + def close(self) -> None: + if self.ctx is not None: + mtmd_cpp.mtmd_free(self.ctx) + self.ctx = None + + def _max_bytes_for_media(self, kind: Literal["image", "audio"]) -> int: + if kind == "image": + return self.image_max_bytes + return self.audio_max_bytes + + def _load_media_file(self, kind: Literal["image", "audio"], media_url: str) -> bytes: + if self.allowed_local_media_path is None: + raise CompletionRequestValidationError("local media path is not allowed") + parsed = urllib.parse.urlsplit(media_url) + if parsed.netloc not in {"", "localhost"}: + raise CompletionRequestValidationError("local media path is not allowed") + path = Path(urllib.parse.unquote(parsed.path)).expanduser().resolve() + try: + path.relative_to(self.allowed_local_media_path) + except ValueError as exc: + raise CompletionRequestValidationError("local media path is not allowed") from exc + max_bytes = self._max_bytes_for_media(kind) + try: + if path.stat().st_size > max_bytes: + raise CompletionRequestValidationError(f"{kind} exceeds model.mtmd.{kind}_max_bytes") + data = path.read_bytes() + except OSError as exc: + raise CompletionRequestValidationError(f"failed to read local {kind}: {exc}") from exc + if len(data) > max_bytes: + raise CompletionRequestValidationError(f"{kind} exceeds model.mtmd.{kind}_max_bytes") + return data + + def _validate_remote_media_url(self, media_url: str) -> str: + parsed = urllib.parse.urlsplit(media_url) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise CompletionRequestValidationError( + "only data:, file:, http:, and https: media URLs are supported" + ) + if parsed.username is not None or parsed.password is not None: + raise CompletionRequestValidationError("remote media domain is not allowed") + hostname = parsed.hostname.lower() + if self.allowed_media_domains and hostname not in self.allowed_media_domains: + raise CompletionRequestValidationError("remote media domain is not allowed") + return urllib.parse.urlunsplit(parsed) + + @staticmethod + def _urlopen_without_redirects( + request: urllib.request.Request, + *, + timeout: float, + ) -> Any: + class NoRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore[no-untyped-def] + return None + + opener = urllib.request.build_opener(NoRedirectHandler) + return opener.open(request, timeout=timeout) + + def _load_media_url(self, kind: Literal["image", "audio"], media_url: str) -> bytes: + max_bytes = self._max_bytes_for_media(kind) + if media_url.startswith("data:"): + try: + _, encoded = media_url.split(",", 1) + except ValueError as exc: + raise CompletionRequestValidationError(f"invalid data {kind} URL") from exc + data = base64.b64decode(encoded, validate=False) + if len(data) > max_bytes: + raise CompletionRequestValidationError(f"{kind} exceeds model.mtmd.{kind}_max_bytes") + return data + if media_url.startswith("file:"): + return self._load_media_file(kind, media_url) + media_url = self._validate_remote_media_url(media_url) + request = urllib.request.Request( + media_url, + headers={"User-Agent": "llama-cpp-python-batch-mtmd/0"}, + ) + try: + with self._urlopen_without_redirects(request, timeout=self.image_timeout_seconds) as response: + data = response.read(max_bytes + 1) + except (OSError, TimeoutError, urllib.error.URLError) as exc: + raise CompletionRequestValidationError( + f"failed to fetch {kind} URL: {exc}" + ) from exc + if len(data) > max_bytes: + raise CompletionRequestValidationError(f"{kind} exceeds model.mtmd.{kind}_max_bytes") + return data + + def load_media(self, media: MediaInput) -> bytes: + if media.url is not None: + return self._load_media_url(media.kind, media.url) + if media.kind != "audio" or media.data is None: + raise CompletionRequestValidationError(f"{media.kind} input requires a URL") + try: + data = base64.b64decode(media.data, validate=False) + except (ValueError, binascii.Error) as exc: + raise CompletionRequestValidationError("input_audio data must be valid base64") from exc + if len(data) > self.audio_max_bytes: + raise CompletionRequestValidationError("audio exceeds model.mtmd.audio_max_bytes") + return data + + def _create_bitmap(self, media_bytes: bytes, kind: Literal["image", "audio"]) -> Any: + buffer = (ctypes.c_uint8 * len(media_bytes)).from_buffer_copy(media_bytes) + bitmap = mtmd_cpp.mtmd_helper_bitmap_init_from_buf( + self.ctx, + buffer, + len(media_bytes), + ) + if bitmap is None: + raise CompletionRequestValidationError(f"failed to create MTMD {kind} bitmap") + return bitmap + + def _media_identity_tokens(self, kind: Literal["image", "audio"], key: str, n_pos: int) -> List[int]: + tokens: List[int] = [] + for index in range(n_pos): + digest = hashlib.sha256(f"{kind}:{key}:{index}".encode("utf-8")).digest() + tokens.append(-1 - (int.from_bytes(digest[:4], "little") & 0x3FFFFFFF)) + return tokens + + def _encode_media_chunk( + self, + *, + kind: Literal["image", "audio"], + key: str, + chunk: Any, + ) -> np.ndarray: + n_tokens = int(mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk)) + if self.embedding_cache is not None: + cached = self.embedding_cache.load(key) + if ( + cached is not None + and cached.embeddings.shape == (n_tokens, self.n_embd_inp) + ): + return cached.embeddings + result = int(mtmd_cpp.mtmd_encode_chunk(self.ctx, chunk)) + if result != 0: + raise CompletionRequestValidationError( + f"failed to encode {kind} chunk: error code {result}" + ) + output = mtmd_cpp.mtmd_get_output_embd(self.ctx) + if output is None: + raise CompletionRequestValidationError(f"MTMD {kind} encoder returned no embeddings") + flat = np.ctypeslib.as_array(output, shape=(n_tokens * self.n_embd_inp,)) + embeddings = np.array(flat, dtype=np.float32, copy=True).reshape( + n_tokens, + self.n_embd_inp, + ) + if self.embedding_cache is not None: + self.embedding_cache.save(key, embeddings) + return embeddings + + def _positions_for_chunk(self, chunk: Any, start_pos: int) -> np.ndarray: + n_tokens = int(mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk)) + if not mtmd_cpp.mtmd_decode_use_mrope(self.ctx): + return np.arange(start_pos, start_pos + n_tokens, dtype=np.int32) + chunk_type = int(mtmd_cpp.mtmd_input_chunk_get_type(chunk)) + if chunk_type == mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_AUDIO: + positions = np.empty((4, n_tokens), dtype=np.int32) + positions[:] = np.arange(start_pos, start_pos + n_tokens, dtype=np.int32) + return positions.reshape(-1) + image_tokens = mtmd_cpp.mtmd_input_chunk_get_tokens_image(chunk) + if image_tokens is None: + raise CompletionRequestValidationError("MTMD image chunk has no image tokens") + positions = np.empty((4, n_tokens), dtype=np.int32) + for index in range(n_tokens): + pos = mtmd_cpp.mtmd_image_tokens_get_decoder_pos( + image_tokens, + llama_cpp.llama_pos(start_pos), + index, + ) + positions[0, index] = int(pos.t) + positions[1, index] = int(pos.y) + positions[2, index] = int(pos.x) + positions[3, index] = int(pos.z) + return positions.reshape(-1) + + def build_prompt_plan( + self, + *, + messages: List[ChatCompletionRequestMessage], + functions: Optional[List[ChatTemplateFunctionDefinition]] = None, + function_call: Optional[Union[str, ChatTemplateFunctionCall]] = None, + tools: Optional[List[ChatTemplateTool]] = None, + tool_choice: Optional[Union[str, ChatTemplateToolChoice]] = None, + reasoning_effort: Optional[str] = None, + ) -> PromptPlan: + media_inputs = Jinja2ChatFormatter.media_inputs_from_messages(messages) + if not media_inputs: + prompt, generation_prompt, _ = self.chat_formatter.format( + messages=messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + reasoning_effort=reasoning_effort, + ) + tokens = self.tokenize(prompt, add_bos=False, special=True) + return PromptPlan.from_tokens(prompt, tokens, generation_prompt=generation_prompt) + if any(media.kind == "image" for media in media_inputs) and not self.supports_vision: + raise CompletionRequestValidationError("MTMD projector does not support images") + if any(media.kind == "audio" for media in media_inputs) and not self.supports_audio: + raise CompletionRequestValidationError("MTMD projector does not support audio") + with self.lock: + return self._build_prompt_plan_locked( + messages=messages, + media_inputs=media_inputs, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + reasoning_effort=reasoning_effort, + ) + + def _build_prompt_plan_locked( + self, + *, + messages: List[ChatCompletionRequestMessage], + media_inputs: List[MediaInput], + functions: Optional[List[ChatTemplateFunctionDefinition]], + function_call: Optional[Union[str, ChatTemplateFunctionCall]], + tools: Optional[List[ChatTemplateTool]], + tool_choice: Optional[Union[str, ChatTemplateToolChoice]], + reasoning_effort: Optional[str], + ) -> PromptPlan: + prompt, generation_prompt, _ = self.chat_formatter.format( + messages=messages, + media_marker=self.media_marker, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + reasoning_effort=reasoning_effort, + ) + media_bytes_by_index = [self.load_media(media) for media in media_inputs] + bitmaps: List[Any] = [] + chunks: Optional[Any] = None + try: + bitmaps = [ + self._create_bitmap(media_bytes, media.kind) + for media, media_bytes in zip(media_inputs, media_bytes_by_index) + ] + input_text = mtmd_cpp.mtmd_input_text() + input_text.text = prompt.encode("utf-8") + input_text.add_special = False + input_text.parse_special = True + chunks = mtmd_cpp.mtmd_input_chunks_init() + if chunks is None: + raise CompletionRequestValidationError("failed to create MTMD input chunks") + bitmap_array = (mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps) + result = int( + mtmd_cpp.mtmd_tokenize( + self.ctx, + chunks, + ctypes.byref(input_text), + bitmap_array, + len(bitmaps), + ) + ) + if result != 0: + raise CompletionRequestValidationError( + f"failed to tokenize MTMD prompt: error code {result}" + ) + segments: List[PromptSegment] = [] + identity_tokens: List[int] = [] + text_tokens: List[int] = [] + text_token_index_by_pos: Dict[int, int] = {} + identity_pos = 0 + decode_pos = 0 + media_index = 0 + n_chunks = int(mtmd_cpp.mtmd_input_chunks_size(chunks)) + for chunk_index in range(n_chunks): + chunk = mtmd_cpp.mtmd_input_chunks_get(chunks, chunk_index) + if chunk is None: + continue + chunk_type = int(mtmd_cpp.mtmd_input_chunk_get_type(chunk)) + if chunk_type == mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_TEXT: + n_tokens_out = ctypes.c_size_t() + tokens_ptr = mtmd_cpp.mtmd_input_chunk_get_tokens_text( + chunk, + ctypes.byref(n_tokens_out), + ) + tokens = ( + [int(tokens_ptr[index]) for index in range(int(n_tokens_out.value))] + if tokens_ptr + else [] + ) + if tokens: + start_pos = identity_pos + segments.append( + PromptSegment( + kind="text", + start_pos=start_pos, + n_pos=len(tokens), + identity_tokens=list(tokens), + decode_start_pos=decode_pos, + decode_n_pos=len(tokens), + text_tokens=list(tokens), + ) + ) + for offset, token in enumerate(tokens): + text_token_index_by_pos[start_pos + offset] = len(text_tokens) + text_tokens.append(token) + identity_tokens.extend(tokens) + identity_pos += len(tokens) + decode_pos += len(tokens) + continue + if chunk_type == mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_IMAGE: + kind: Literal["image", "audio"] = "image" + if not self.supports_vision: + raise CompletionRequestValidationError("MTMD projector does not support images") + elif chunk_type == mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_AUDIO: + kind = "audio" + if not self.supports_audio: + raise CompletionRequestValidationError("MTMD projector does not support audio") + else: + raise CompletionRequestValidationError("unsupported MTMD input chunk type") + if media_index >= len(media_bytes_by_index): + raise CompletionRequestValidationError("MTMD media chunk count mismatch") + media = media_inputs[media_index] + media_bytes = media_bytes_by_index[media_index] + media_index += 1 + if media.kind != kind: + raise CompletionRequestValidationError("MTMD media chunk modality mismatch") + key = ( + self.embedding_cache.key_for_media(kind, media_bytes) + if self.embedding_cache is not None + else MTMDEmbeddingCache._build_key( + model_fingerprint=self.model_fingerprint, + mmproj_fingerprint=self.mmproj_fingerprint, + kind=kind, + media_bytes=media_bytes, + ) + ) + decode_n_pos = int(mtmd_cpp.mtmd_input_chunk_get_n_pos(chunk)) + if decode_n_pos <= 0: + raise CompletionRequestValidationError("MTMD media chunk has no decoder positions") + embeddings = self._encode_media_chunk(kind=kind, key=key, chunk=chunk) + n_tokens = int(embeddings.shape[0]) + if n_tokens <= 0: + raise CompletionRequestValidationError("MTMD media chunk has no embeddings") + non_causal = bool(mtmd_cpp.mtmd_decode_use_non_causal(self.ctx, chunk)) + segment_identity = self._media_identity_tokens(kind, key, n_tokens) + positions = self._positions_for_chunk(chunk, decode_pos) + segment = PromptSegment( + kind=kind, + start_pos=identity_pos, + n_pos=n_tokens, + identity_tokens=segment_identity, + decode_start_pos=decode_pos, + decode_n_pos=decode_n_pos, + media=PromptSegment.Media( + embeddings=embeddings, + positions=positions, + non_causal=non_causal, + ), + ) + if non_causal and embeddings.shape[0] > min(self.n_batch, self.n_ubatch): + raise CompletionRequestValidationError( + f"non-causal {kind} embedding chunk exceeds model batch limits; " + "increase n_batch and n_ubatch" + ) + segments.append(segment) + identity_tokens.extend(segment_identity) + identity_pos += n_tokens + decode_pos += decode_n_pos + if media_index != len(media_bytes_by_index): + raise CompletionRequestValidationError("not all media inputs were consumed by MTMD") + return PromptPlan( + text=prompt, + generation_prompt=generation_prompt, + text_tokens=text_tokens, + identity_tokens=identity_tokens, + segments=segments, + text_token_index_by_pos=text_token_index_by_pos, + ) + finally: + if chunks is not None: + mtmd_cpp.mtmd_input_chunks_free(chunks) + for bitmap in bitmaps: + mtmd_cpp.mtmd_bitmap_free(bitmap) + + +class Model: + @dataclass(frozen=True) + class LoraAdapter: + path: str + scale: float = 1.0 + + def __init__( + self, + *, + model_path: str, + model_alias: Optional[str] = None, + chat_template: Optional[str] = None, + loras: Optional[List["Model.LoraAdapter"]] = None, + n_gpu_layers: Optional[int] = None, + split_mode: Optional[int] = None, + main_gpu: Optional[int] = None, + tensor_split: Optional[List[float]] = None, + vocab_only: Optional[bool] = None, + use_mmap: Optional[bool] = None, + use_mlock: Optional[bool] = None, + kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, + n_ctx: Optional[int], + n_batch: Optional[int], + n_ubatch: Optional[int] = None, + n_seq_max: Optional[int], + n_threads: Optional[int], + n_threads_batch: Optional[int], + rope_scaling_type: Optional[int] = None, + pooling_type: Optional[int] = None, + attention_type: Optional[int] = None, + rope_freq_base: Optional[float] = None, + rope_freq_scale: Optional[float] = None, + yarn_ext_factor: Optional[float] = None, + yarn_attn_factor: Optional[float] = None, + yarn_beta_fast: Optional[float] = None, + yarn_beta_slow: Optional[float] = None, + yarn_orig_ctx: Optional[int] = None, + offload_kqv: Optional[bool] = None, + flash_attn: Optional[bool] = None, + op_offload: Optional[bool] = None, + swa_full: Optional[bool] = None, + no_perf: Optional[bool] = None, + type_k: Optional[int] = None, + type_v: Optional[int] = None, + kv_unified: bool = True, + max_seq_len: Optional[int] = None, + max_output_tokens: Optional[int] = None, + draft_model: Optional[str] = None, + draft_model_num_pred_tokens: int = 16, + draft_model_max_ngram_size: int = 2, + draft_model_top_k: int = 1, + draft_model_p_min: float = 0.0, + draft_model_max_batch_size: Optional[int] = None, + draft_model_threads: Optional[int] = None, + draft_model_threads_batch: Optional[int] = None, + response_schema: Optional[Dict[str, Any]] = None, + store_logits: bool = True, + ) -> None: + llama_cpp.llama_backend_init() + self.backend_initialized = True + self.model_path = model_path + self.model_alias = model_alias + self.chat_template_override = chat_template + self.response_schema = response_schema + self.store_logits = store_logits + self.max_output_tokens = max_output_tokens + self.draft_model_max_batch_size = draft_model_max_batch_size + self.draft_provider: Optional[DraftProvider] = None + self.loras = list(loras or []) + self._lora_adapters: List[Any] = [] + self._lora_adapter_array: Optional[Any] = None + self._lora_scales_array: Optional[Any] = None + model_params, self._c_tensor_split, self._kv_overrides_array = ( + self.build_model_params( + n_gpu_layers=n_gpu_layers, + split_mode=split_mode, + main_gpu=main_gpu, + tensor_split=tensor_split, + vocab_only=vocab_only, + use_mmap=use_mmap, + use_mlock=use_mlock, + kv_overrides=kv_overrides, + ) + ) + llama_model = llama_cpp.llama_model_load_from_file( + model_path.encode("utf-8"), + model_params, + ) + if llama_model is None: + raise RuntimeError(f"failed to load model: {model_path}") + self.llama_model = llama_model + vocab = llama_cpp.llama_model_get_vocab(llama_model) + if vocab is None: + raise RuntimeError("failed to access model vocabulary") + self.vocab = vocab + if llama_cpp.llama_model_has_encoder(llama_model): + raise RuntimeError("encoder models are not supported") + if not llama_cpp.llama_model_has_decoder(llama_model): + raise RuntimeError("decoder is required") + if llama_cpp.llama_model_is_recurrent(llama_model): + self.memory_model = "recurrent" + elif llama_cpp.llama_model_is_hybrid(llama_model): + self.memory_model = "hybrid" + else: + self.memory_model = ( + "attention-unified" if kv_unified else "attention-partitioned" + ) + normalized_draft_model = draft_model + required_mtp_batch = max(1, draft_model_num_pred_tokens + 1) + if normalized_draft_model == "draft-mtp": + if n_batch is not None and n_batch < required_mtp_batch: + raise RuntimeError( + "MTP requires n_batch to fit the pending token plus draft tokens " + f"(required {required_mtp_batch}, got {n_batch})" + ) + self.draft_target_batching = normalized_draft_model is not None + if ( + normalized_draft_model is not None + and normalized_draft_model != "draft-mtp" + and not self.memory_model.startswith("attention") + ): + raise RuntimeError( + "speculative decoding is only supported for attention models" + ) + n_ctx_train = int(llama_cpp.llama_model_n_ctx_train(llama_model)) + target_n_rs_seq = ( + max(1, draft_model_num_pred_tokens) + if normalized_draft_model == "draft-mtp" + else None + ) + + context_params = self.build_context_params( + n_ctx=n_ctx if n_ctx is not None else n_ctx_train, + n_batch=n_batch, + n_ubatch=n_ubatch, + n_seq_max=n_seq_max, + n_threads=n_threads, + n_threads_batch=n_threads_batch, + rope_scaling_type=rope_scaling_type, + pooling_type=pooling_type, + attention_type=attention_type, + rope_freq_base=rope_freq_base, + rope_freq_scale=rope_freq_scale, + yarn_ext_factor=yarn_ext_factor, + yarn_attn_factor=yarn_attn_factor, + yarn_beta_fast=yarn_beta_fast, + yarn_beta_slow=yarn_beta_slow, + yarn_orig_ctx=yarn_orig_ctx, + offload_kqv=offload_kqv, + flash_attn=flash_attn, + op_offload=op_offload, + swa_full=swa_full, + no_perf=no_perf, + type_k=type_k, + type_v=type_v, + kv_unified=kv_unified, + n_rs_seq=target_n_rs_seq, + ctx_type=None, + ) + ctx = llama_cpp.llama_init_from_model(llama_model, context_params) + if ctx is None: + raise RuntimeError("failed to create context") + self.ctx = ctx + mem = llama_cpp.llama_get_memory(ctx) + if mem is None: + raise RuntimeError("failed to access model memory") + self.mem = mem + self.n_ctx = int(llama_cpp.llama_n_ctx(ctx)) + self.n_ctx_seq = int(llama_cpp.llama_n_ctx_seq(ctx)) + self.n_seq_max = int(llama_cpp.llama_n_seq_max(ctx)) + self.n_rs_seq = int(llama_cpp.llama_n_rs_seq(ctx)) + self.n_batch = int(llama_cpp.llama_n_batch(ctx)) + self.n_ubatch = int(llama_cpp.llama_n_ubatch(ctx)) + self.n_threads_batch = ( + n_threads_batch + if n_threads_batch is not None + else max(multiprocessing.cpu_count(), 1) + ) + self.mtmd_processor: Optional[MTMDProcessor] = None + self._embedding_batch: Optional[llama_cpp.llama_batch] = None + self._embedding_batch_refs: List[Any] = [] + if normalized_draft_model == "draft-mtp" and self.n_batch < required_mtp_batch: + raise RuntimeError( + "MTP requires runtime n_batch to fit the pending token plus draft tokens " + f"(required {required_mtp_batch}, got {self.n_batch})" + ) + if target_n_rs_seq is not None and self.n_rs_seq < target_n_rs_seq: + raise RuntimeError( + "MTP requires retained recurrent-state slots for rollback " + f"(required {target_n_rs_seq}, got {self.n_rs_seq})" + ) + self.n_ctx_train = n_ctx_train + self.n_vocab = int(llama_cpp.llama_vocab_n_tokens(self.vocab)) + self.n_embd_inp = int(llama_cpp.llama_model_n_embd_inp(self.llama_model)) + self.kv_unified = kv_unified + self.max_seq_len_limit = min(self.request_context_limit, self.n_ctx_train) + if max_seq_len is None: + self.max_seq_len = self.max_seq_len_limit + else: + if max_seq_len <= 0: + llama_cpp.llama_free(self.ctx) + llama_cpp.llama_model_free(self.llama_model) + if self.backend_initialized: + llama_cpp.llama_backend_free() + self.backend_initialized = False + raise RuntimeError("max_seq_len must be greater than 0") + if max_seq_len > self.max_seq_len_limit: + llama_cpp.llama_free(self.ctx) + llama_cpp.llama_model_free(self.llama_model) + if self.backend_initialized: + llama_cpp.llama_backend_free() + self.backend_initialized = False + raise RuntimeError( + f"max_seq_len ({max_seq_len}) exceeds runtime limit ({self.max_seq_len_limit})" + ) + self.max_seq_len = max_seq_len + self.batch = llama_cpp.llama_batch_init(self.n_batch, 0, self.n_seq_max) + self.bos_token = int(llama_cpp.llama_vocab_bos(self.vocab)) + self.eos_token = int(llama_cpp.llama_vocab_eos(self.vocab)) + self.cls_token = self.bos_token + self.sep_token = int(llama_cpp.llama_vocab_sep(self.vocab)) + self.fim_pre_token = int(llama_cpp.llama_vocab_fim_pre(self.vocab)) + self.fim_mid_token = int(llama_cpp.llama_vocab_fim_mid(self.vocab)) + self.fim_suf_token = int(llama_cpp.llama_vocab_fim_suf(self.vocab)) + self.add_bos_token = bool(llama_cpp.llama_vocab_get_add_bos(self.vocab)) + self.add_eos_token = bool(llama_cpp.llama_vocab_get_add_eos(self.vocab)) + self.add_space_prefix = ( + self._meta_value("tokenizer.ggml.add_space_prefix") != "false" + ) + if normalized_draft_model is None: + self.draft_provider = None + elif normalized_draft_model == "prompt-lookup-decoding": + self.draft_provider = PromptLookupDecoding( + max_ngram_size=draft_model_max_ngram_size, + num_pred_tokens=draft_model_num_pred_tokens, + ) + elif normalized_draft_model == "draft-mtp": + if self.n_ubatch < self.n_seq_max: + mtp_n_batch = self.n_batch + else: + mtp_n_batch = min( + self.n_batch, + max(self.n_ubatch, self.n_seq_max, required_mtp_batch), + ) + mtp_context_params = self.build_context_params( + n_ctx=self.n_ctx, + n_batch=mtp_n_batch, + n_ubatch=min(self.n_ubatch, mtp_n_batch), + n_seq_max=self.n_seq_max, + n_threads=( + draft_model_threads + if draft_model_threads is not None + else n_threads + ), + n_threads_batch=( + draft_model_threads_batch + if draft_model_threads_batch is not None + else ( + draft_model_threads + if draft_model_threads is not None + else n_threads_batch + ) + ), + rope_scaling_type=rope_scaling_type, + pooling_type=pooling_type, + attention_type=attention_type, + rope_freq_base=rope_freq_base, + rope_freq_scale=rope_freq_scale, + yarn_ext_factor=yarn_ext_factor, + yarn_attn_factor=yarn_attn_factor, + yarn_beta_fast=yarn_beta_fast, + yarn_beta_slow=yarn_beta_slow, + yarn_orig_ctx=yarn_orig_ctx, + offload_kqv=offload_kqv, + flash_attn=flash_attn, + op_offload=op_offload, + swa_full=swa_full, + no_perf=no_perf, + type_k=type_k, + type_v=type_v, + kv_unified=kv_unified, + n_rs_seq=target_n_rs_seq, + ctx_type=llama_cpp.LLAMA_CONTEXT_TYPE_MTP, + n_outputs_max=min(mtp_n_batch, self.n_seq_max), + ) + try: + self.draft_provider = MTPDraftProvider( + model=self, + context_params=mtp_context_params, + num_pred_tokens=draft_model_num_pred_tokens, + top_k=draft_model_top_k, + p_min=draft_model_p_min, + ) + except BaseException: + llama_cpp.llama_batch_free(self.batch) + llama_cpp.llama_free(self.ctx) + self._free_lora_adapters() + llama_cpp.llama_model_free(self.llama_model) + if self.backend_initialized: + llama_cpp.llama_backend_free() + self.backend_initialized = False + raise + else: + raise RuntimeError(f"unsupported draft model: {draft_model}") + try: + self._load_lora_adapters(self.loras) + self._apply_lora_adapters(self.ctx, "target") + if isinstance(self.draft_provider, MTPDraftProvider): + self._apply_lora_adapters(self.draft_provider.ctx, "MTP draft") + except BaseException: + if self.draft_provider is not None: + self.draft_provider.close() + llama_cpp.llama_batch_free(self.batch) + llama_cpp.llama_free(self.ctx) + self._free_lora_adapters() + llama_cpp.llama_model_free(self.llama_model) + if self.backend_initialized: + llama_cpp.llama_backend_free() + self.backend_initialized = False + raise + self.chat_formatter = self._build_chat_formatter() + + @staticmethod + def build_model_params( + *, + n_gpu_layers: Optional[int], + split_mode: Optional[int], + main_gpu: Optional[int], + tensor_split: Optional[List[float]], + vocab_only: Optional[bool], + use_mmap: Optional[bool], + use_mlock: Optional[bool], + kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]], + ) -> Tuple[Any, Optional[Any], Optional[Any]]: + model_params = llama_cpp.llama_model_default_params() + if n_gpu_layers is not None: + model_params.n_gpu_layers = 0x7FFFFFFF if n_gpu_layers == -1 else n_gpu_layers + if split_mode is not None: + model_params.split_mode = split_mode + if main_gpu is not None: + model_params.main_gpu = main_gpu + tensor_split_ref = None + if tensor_split is not None: + if len(tensor_split) > llama_cpp.LLAMA_MAX_DEVICES: + raise ValueError( + "tensor_split exceeds maximum supported devices " + f"({llama_cpp.LLAMA_MAX_DEVICES})" + ) + float_array = ctypes.c_float * llama_cpp.LLAMA_MAX_DEVICES + tensor_split_ref = float_array(*tensor_split) + model_params.tensor_split = tensor_split_ref + if vocab_only is not None: + model_params.vocab_only = vocab_only + if use_mmap is not None: + model_params.use_mmap = use_mmap + if use_mlock is not None: + model_params.use_mlock = use_mlock + + kv_overrides_ref = None + if kv_overrides is not None: + kv_overrides_ref = ( + llama_cpp.llama_model_kv_override * (len(kv_overrides) + 1) + )() + for index, (key, value) in enumerate(kv_overrides.items()): + kv_overrides_ref[index].key = key.encode("utf-8") + if isinstance(value, bool): + kv_overrides_ref[index].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL + kv_overrides_ref[index].value.val_bool = value + elif isinstance(value, int): + kv_overrides_ref[index].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT + kv_overrides_ref[index].value.val_i64 = value + elif isinstance(value, float): + kv_overrides_ref[index].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT + kv_overrides_ref[index].value.val_f64 = value + elif isinstance(value, str): + encoded = value.encode("utf-8") + if len(encoded) > 128: + raise ValueError(f"kv_overrides value for {key} is too long") + encoded = encoded.ljust(128, b"\0") + kv_overrides_ref[index].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR + address = cast( + int, + ctypes.addressof(kv_overrides_ref[index].value) + + cast( + Any, + llama_cpp.llama_model_kv_override_value.val_str, + ).offset, + ) + buffer_start = ctypes.cast(address, ctypes.POINTER(ctypes.c_char)) + ctypes.memmove(buffer_start, encoded, 128) + else: + raise ValueError(f"unsupported kv_override value for {key}: {value!r}") + kv_overrides_ref[-1].key = b"\0" + model_params.kv_overrides = kv_overrides_ref + return model_params, tensor_split_ref, kv_overrides_ref + + @staticmethod + def build_context_params( + *, + n_ctx: Optional[int], + n_batch: Optional[int], + n_ubatch: Optional[int], + n_seq_max: Optional[int], + n_threads: Optional[int], + n_threads_batch: Optional[int], + rope_scaling_type: Optional[int], + pooling_type: Optional[int], + attention_type: Optional[int], + rope_freq_base: Optional[float], + rope_freq_scale: Optional[float], + yarn_ext_factor: Optional[float], + yarn_attn_factor: Optional[float], + yarn_beta_fast: Optional[float], + yarn_beta_slow: Optional[float], + yarn_orig_ctx: Optional[int], + offload_kqv: Optional[bool], + flash_attn: Optional[bool], + op_offload: Optional[bool], + swa_full: Optional[bool], + no_perf: Optional[bool], + type_k: Optional[int], + type_v: Optional[int], + kv_unified: bool, + n_rs_seq: Optional[int] = None, + ctx_type: Optional[int] = None, + n_outputs_max: Optional[int] = None, + ) -> Any: + context_params = llama_cpp.llama_context_default_params() + if n_ctx is not None: + context_params.n_ctx = n_ctx + if n_batch is not None: + context_params.n_batch = min(int(context_params.n_ctx), n_batch) + if n_ubatch is not None: + context_params.n_ubatch = min(int(context_params.n_batch), n_ubatch) + if n_seq_max is not None: + context_params.n_seq_max = n_seq_max + if n_rs_seq is not None: + context_params.n_rs_seq = n_rs_seq + if n_threads is not None: + context_params.n_threads = n_threads + if n_threads_batch is not None: + context_params.n_threads_batch = n_threads_batch + if ctx_type is not None: + context_params.ctx_type = ctx_type + if n_outputs_max is not None: + context_params.n_outputs_max = n_outputs_max + if rope_scaling_type is not None: + context_params.rope_scaling_type = rope_scaling_type + if pooling_type is not None: + context_params.pooling_type = pooling_type + if attention_type is not None: + context_params.attention_type = attention_type + if rope_freq_base is not None: + context_params.rope_freq_base = rope_freq_base + if rope_freq_scale is not None: + context_params.rope_freq_scale = rope_freq_scale + if yarn_ext_factor is not None: + context_params.yarn_ext_factor = yarn_ext_factor + if yarn_attn_factor is not None: + context_params.yarn_attn_factor = yarn_attn_factor + if yarn_beta_fast is not None: + context_params.yarn_beta_fast = yarn_beta_fast + if yarn_beta_slow is not None: + context_params.yarn_beta_slow = yarn_beta_slow + if yarn_orig_ctx is not None: + context_params.yarn_orig_ctx = yarn_orig_ctx + if offload_kqv is not None: + context_params.offload_kqv = offload_kqv + if flash_attn is not None: + context_params.flash_attn_type = ( + llama_cpp.LLAMA_FLASH_ATTN_TYPE_ENABLED + if flash_attn + else llama_cpp.LLAMA_FLASH_ATTN_TYPE_DISABLED + ) + if op_offload is not None: + context_params.op_offload = op_offload + if swa_full is not None: + context_params.swa_full = swa_full + if no_perf is not None: + context_params.no_perf = no_perf + if type_k is not None: + context_params.type_k = type_k + if type_v is not None: + context_params.type_v = type_v + context_params.kv_unified = kv_unified + return context_params + + @property + def exact_checkpoints_only(self) -> bool: + return self.memory_model in {"recurrent", "hybrid"} + + @property + def has_attention_budget(self) -> bool: + return self.memory_model != "recurrent" + + @property + def attention_partitioned(self) -> bool: + return self.memory_model == "attention-partitioned" + + @property + def request_context_limit(self) -> int: + if self.attention_partitioned: + return self.n_ctx_seq + return self.n_ctx + + def _load_lora_adapters(self, loras: List["Model.LoraAdapter"]) -> None: + for lora in loras: + adapter = llama_cpp.llama_adapter_lora_init( + self.llama_model, + lora.path.encode("utf-8"), + ) + if adapter is None: + raise RuntimeError(f"failed to load LoRA adapter: {lora.path}") + self._lora_adapters.append(adapter) + + if not self._lora_adapters: + return + + adapter_array_type = llama_cpp.llama_adapter_lora_p_ctypes * len( + self._lora_adapters + ) + scale_array_type = ctypes.c_float * len(self._lora_adapters) + self._lora_adapter_array = adapter_array_type(*self._lora_adapters) + self._lora_scales_array = scale_array_type( + *(float(lora.scale) for lora in loras) + ) + + def _apply_lora_adapters(self, ctx: Any, context_name: str) -> None: + if not self._lora_adapters: + return + if self._lora_adapter_array is None or self._lora_scales_array is None: + raise RuntimeError("LoRA adapter arrays are not initialized") + result = llama_cpp.llama_set_adapters_lora( + ctx, + self._lora_adapter_array, + len(self._lora_adapters), + self._lora_scales_array, + ) + if result: + raise RuntimeError(f"failed to apply LoRA adapters to {context_name} context") + + def _free_lora_adapters(self) -> None: + while self._lora_adapters: + adapter = self._lora_adapters.pop() + llama_cpp.llama_adapter_lora_free(adapter) + self._lora_adapter_array = None + self._lora_scales_array = None + + def close(self) -> None: + if self.mtmd_processor is not None: + self.mtmd_processor.close() + self.mtmd_processor = None + if self.draft_provider is not None: + self.draft_provider.close() + llama_cpp.llama_batch_free(self.batch) + llama_cpp.llama_free(self.ctx) + self._free_lora_adapters() + llama_cpp.llama_model_free(self.llama_model) + if self.backend_initialized: + llama_cpp.llama_backend_free() + self.backend_initialized = False + + def _meta_value(self, key: str) -> Optional[str]: + encoded = key.encode("utf-8") + capacity = 256 + while True: + buffer = ctypes.create_string_buffer(capacity) + count = int( + llama_cpp.llama_model_meta_val_str( + self.llama_model, + encoded, + cast(Any, buffer), + capacity, + ) + ) + if count < 0: + return None + if count < capacity: + return buffer.value.decode("utf-8", errors="ignore") + capacity = count + 1 + + def _build_chat_formatter(self) -> Optional[Jinja2ChatFormatter]: + template_text = self.chat_template_override + if template_text is None: + template = llama_cpp.llama_model_chat_template(self.llama_model, None) + if template: + template_text = template.decode("utf-8", errors="ignore") + if not template_text: + return None + bos_token = "" + eos_token = "" + if self.bos_token != -1: + bos_text = llama_cpp.llama_vocab_get_text(self.vocab, self.bos_token) + bos_token = bos_text.decode("utf-8", errors="ignore") if bos_text else "" + if self.eos_token != -1: + eos_text = llama_cpp.llama_vocab_get_text(self.vocab, self.eos_token) + eos_token = eos_text.decode("utf-8", errors="ignore") if eos_text else "" + return Jinja2ChatFormatter( + template=template_text, + bos_token=bos_token, + eos_token=eos_token, + ) + + def tokenize( + self, + text: str, + *, + add_bos: bool = True, + special: bool = True, + ) -> List[int]: + encoded = text.encode("utf-8") + capacity = max(32, len(encoded) + 32) + while True: + tokens = (llama_cpp.llama_token * capacity)() + count = int( + llama_cpp.llama_tokenize( + self.vocab, + encoded, + len(encoded), + tokens, + capacity, + add_bos, + special, + ) + ) + if count >= 0: + return [int(tokens[index]) for index in range(count)] + capacity = max(capacity * 2, -count) + + def build_prompt_tokens(self, prompt: str, suffix: Optional[str]) -> List[int]: + if suffix is None: + return self.tokenize(prompt) + if min(self.fim_pre_token, self.fim_mid_token, self.fim_suf_token) < 0: + raise ValueError("suffix is not supported by this model") + bos_tokens = [self.cls_token if self.cls_token != -1 else self.bos_token] + eos_tokens = [self.sep_token if self.sep_token != -1 else self.eos_token] + if not self.add_bos_token or bos_tokens[:1] == [-1]: + bos_tokens = [] + if not self.add_eos_token and self.sep_token == -1: + eos_tokens = [] + suffix_text = suffix + suffix_space_prefix = 0 + if self.add_space_prefix and suffix_text: + suffix_text = "☺" + suffix_text + suffix_space_prefix = 2 + prefix_tokens = [self.fim_pre_token] + self.tokenize( + prompt, + add_bos=False, + special=False, + ) + suffix_tokens = [self.fim_suf_token] + if suffix_text: + suffix_tokens.extend( + self.tokenize( + suffix_text, + add_bos=False, + special=False, + )[suffix_space_prefix:] + ) + return bos_tokens + prefix_tokens + suffix_tokens + [self.fim_mid_token] + eos_tokens + + def build_chat_prompt( + self, + messages: List[ChatCompletionRequestMessage], + *, + functions: Optional[List[ChatTemplateFunctionDefinition]] = None, + function_call: Optional[Union[str, ChatTemplateFunctionCall]] = None, + tools: Optional[List[ChatTemplateTool]] = None, + tool_choice: Optional[Union[str, ChatTemplateToolChoice]] = None, + reasoning_effort: Optional[str] = None, + ) -> Tuple[str, str, PromptPlan, List[str]]: + if self.chat_formatter is None: + raise ValueError("model does not provide a GGUF chat template") + if self.mtmd_processor is not None: + prompt_plan = self.mtmd_processor.build_prompt_plan( + messages=messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + reasoning_effort=reasoning_effort, + ) + formatter_stop = [self.chat_formatter._eos_token] if self.chat_formatter._eos_token else [] + return ( + prompt_plan.text, + prompt_plan.generation_prompt, + prompt_plan, + formatter_stop, + ) + prompt, generation_prompt, formatter_stop = self.chat_formatter.format( + messages=messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + reasoning_effort=reasoning_effort, + ) + prompt_tokens = self.tokenize(prompt, add_bos=False, special=True) + return ( + prompt, + generation_prompt, + PromptPlan.from_tokens( + prompt, + prompt_tokens, + generation_prompt=generation_prompt, + ), + formatter_stop, + ) + + def detokenize(self, tokens: Sequence[int]) -> bytes: + if not tokens: + return b"" + array = (llama_cpp.llama_token * len(tokens))(*tokens) + capacity = max(64, len(tokens) * 16) + while True: + buffer = ctypes.create_string_buffer(capacity) + count = int( + llama_cpp.llama_detokenize( + self.vocab, + array, + len(tokens), + cast(Any, buffer), + capacity, + True, + True, + ) + ) + if count >= 0: + return bytes(buffer.raw[:count]) + capacity = max(capacity * 2, -count) + + def token_bytes(self, token: int) -> bytes: + return self.detokenize([token]) + + def token_bytes_with_prev(self, prev_tokens: Sequence[int], token: int) -> bytes: + current = self.detokenize([*prev_tokens, token]) + previous = self.detokenize(prev_tokens) + return current[len(previous) :] + + def token_bytes_with_prev_bytes( + self, + prev_tokens: Sequence[int], + prev_text_bytes: Union[bytes, bytearray], + token: int, + ) -> bytes: + current = self.detokenize([*prev_tokens, token]) + return current[len(prev_text_bytes) :] + + def clear_batch(self) -> None: + self.batch.n_tokens = 0 + self._embedding_batch = None + self._embedding_batch_refs = [] + + def add_batch_tokens( + self, + *, + seq_id: int, + start_pos: int, + tokens: Sequence[int], + output_indices: Sequence[Optional[int]], + ) -> None: + if not tokens: + return + for index, token in enumerate(tokens): + slot = self.batch.n_tokens + self.batch.token[slot] = token + self.batch.pos[slot] = start_pos + index + self.batch.seq_id[slot][0] = seq_id + self.batch.n_seq_id[slot] = 1 + self.batch.logits[slot] = int(output_indices[index] is not None) + self.batch.n_tokens += 1 + + def add_batch_embeddings( + self, + *, + seq_id: int, + embeddings: np.ndarray, + positions: np.ndarray, + output_indices: Sequence[Optional[int]], + ) -> None: + if self.batch.n_tokens: + raise RuntimeError("cannot mix token and embedding batches") + if self._embedding_batch is not None: + raise RuntimeError("only one embedding batch is supported per scheduler step") + embeddings = np.ascontiguousarray(embeddings, dtype=np.float32) + positions = np.ascontiguousarray(positions, dtype=np.int32).reshape(-1) + n_tokens = int(embeddings.shape[0]) + if n_tokens == 0: + return + if embeddings.ndim != 2 or embeddings.shape[1] != self.n_embd_inp: + raise RuntimeError("embedding batch shape does not match model input embedding size") + if len(positions) not in {n_tokens, n_tokens * 4}: + raise RuntimeError("embedding position length mismatch") + if len(output_indices) != n_tokens: + raise RuntimeError("embedding output index length mismatch") + pos_array = (llama_cpp.llama_pos * len(positions))( + *[int(pos) for pos in positions] + ) + n_seq_id_array = (ctypes.c_int32 * n_tokens)(*[1] * n_tokens) + seq_id_array = (llama_cpp.llama_seq_id * 1)(llama_cpp.llama_seq_id(seq_id)) + seq_ids_array = (ctypes.POINTER(llama_cpp.llama_seq_id) * (n_tokens + 1))() + for index in range(n_tokens): + seq_ids_array[index] = seq_id_array + logits_array = (ctypes.c_int8 * n_tokens)( + *[int(output_index is not None) for output_index in output_indices] + ) + batch = llama_cpp.llama_batch( + n_tokens=n_tokens, + token=None, + embd=embeddings.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + pos=pos_array, + n_seq_id=n_seq_id_array, + seq_id=seq_ids_array, + logits=logits_array, + ) + self._embedding_batch = batch + self._embedding_batch_refs = [ + embeddings, + positions, + pos_array, + n_seq_id_array, + seq_id_array, + seq_ids_array, + logits_array, + ] + + def decode(self) -> None: + batch = self._embedding_batch if self._embedding_batch is not None else self.batch + result = int(llama_cpp.llama_decode(self.ctx, batch)) + if result != 0: + raise RuntimeError(f"llama_decode failed with code {result}") + + def process_draft_batch(self) -> None: + if self.draft_provider is not None: + self.draft_provider.process(self.batch) + + def set_draft_processing_enabled(self, enabled: bool) -> None: + if self.draft_provider is not None: + self.draft_provider.set_target_processing_enabled(enabled) + + def accept_draft_tokens(self, seq_id: int, accepted_draft_tokens: int) -> None: + if self.draft_provider is not None: + self.draft_provider.accept(seq_id, accepted_draft_tokens) + + def truncate_draft_sequence(self, seq_id: int, keep_len: int) -> None: + if self.draft_provider is not None: + self.draft_provider.truncate(seq_id, keep_len) + + def copy_draft_sequence( + self, + source_seq_id: int, + dest_seq_id: int, + p0: int, + p1: int, + ) -> None: + if self.draft_provider is not None: + self.draft_provider.copy_sequence(source_seq_id, dest_seq_id, p0, p1) + + def logits(self, output_index: int) -> np.ndarray: + ptr = llama_cpp.llama_get_logits_ith(self.ctx, output_index) + if not ptr: + raise RuntimeError(f"missing logits output {output_index}") + return np.ctypeslib.as_array(ptr, shape=(self.n_vocab,)).copy() + + +class SequenceDiskCache(SequenceCache): + """Directory-backed cache for serialized llama.cpp sequence state.""" + + @dataclass + class Entry: + entry_id: int + path: Path + tokens: Tuple[int, ...] + size_bytes: int + has_prompt_logits: bool + last_accessed: float + + @dataclass(frozen=True) + class Header: + tokens: Tuple[int, ...] + has_prompt_logits: bool + + @dataclass + class Payload: + tokens: List[int] + state_bytes: np.ndarray + prompt_logits: Optional[np.ndarray] + + FORMAT_VERSION = "1" + + TENSOR_TOKENS = "tokens" + TENSOR_STATE = "state" + TENSOR_PROMPT_LOGITS = "prompt_logits" + + METADATA_FORMAT = "sequence_cache_format" + METADATA_COMPATIBILITY_KEY = "compatibility_key" + + def __init__( + self, + *, + path: Union[str, Path], + max_bytes: int, + compatibility_key: str, + min_tokens: int = 128, + ) -> None: + self.path = Path(path) + self.path.mkdir(parents=True, exist_ok=True) + self.max_bytes = max(0, int(max_bytes)) + self.min_tokens = max(1, int(min_tokens)) + self.compatibility_key = compatibility_key + self._metadata = { + self.METADATA_FORMAT: self.FORMAT_VERSION, + self.METADATA_COMPATIBILITY_KEY: compatibility_key, + } + self._trie = RadixTrie() + self._entries_by_id: Dict[int, SequenceDiskCache.Entry] = {} + self._entries_by_tokens: Dict[Tuple[int, ...], SequenceDiskCache.Entry] = {} + self._next_entry_id = 0 + self._size_bytes = 0 + self._load_entries() + self._evict_if_needed() + + @staticmethod + def fingerprint_file(path: str) -> str: + stat = os.stat(path) + payload = f"{Path(path).resolve()}:{stat.st_size}:{stat.st_mtime_ns}" + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + @classmethod + def compatibility_key_for_model(cls, model: "Model") -> str: + payload: Dict[str, Any] = { + "model": cls.fingerprint_file(model.model_path), + "memory_model": model.memory_model, + "loras": [ + { + "adapter": cls.fingerprint_file(lora.path), + "scale": float(lora.scale), + } + for lora in model.loras + ], + } + if model.memory_model == "attention-partitioned": + payload["attention_streams"] = model.n_seq_max + encoded = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + @staticmethod + def _safe_open(path: Path) -> Any: + try: + from safetensors import safe_open + except ImportError as exc: + raise ImportError( + "disk_cache requires safetensors. Install it with `pip install safetensors`." + ) from exc + + return safe_open(str(path), framework="numpy") + + @staticmethod + def _save_file( + tensors: Dict[str, np.ndarray], + path: Path, + metadata: Dict[str, str], + ) -> None: + from safetensors.numpy import save_file + + save_file(tensors, str(path), metadata=metadata) + + def _metadata_compatible(self, metadata: Optional[Dict[str, str]]) -> bool: + if metadata is None: + return False + return all(metadata.get(key) == value for key, value in self._metadata.items()) + + def _read_entry_header(self, path: Path) -> Optional["SequenceDiskCache.Header"]: + with self._safe_open(path) as tensors: + if not self._metadata_compatible(tensors.metadata()): + return None + tokens_array = tensors.get_tensor(self.TENSOR_TOKENS) + has_prompt_logits = self.TENSOR_PROMPT_LOGITS in tensors.keys() + tokens = tuple(int(token) for token in tokens_array.tolist()) + if len(tokens) < self.min_tokens: + return None + return SequenceDiskCache.Header( + tokens=tokens, + has_prompt_logits=has_prompt_logits, + ) + + def _read_entry_payload( + self, + entry: "SequenceDiskCache.Entry", + ) -> "SequenceDiskCache.Payload": + with self._safe_open(entry.path) as tensors: + tokens = [ + int(token) + for token in tensors.get_tensor(self.TENSOR_TOKENS).tolist() + ] + state_bytes = np.ascontiguousarray( + tensors.get_tensor(self.TENSOR_STATE), + dtype=np.uint8, + ).copy() + prompt_logits = ( + tensors.get_tensor(self.TENSOR_PROMPT_LOGITS) + if entry.has_prompt_logits + and self.TENSOR_PROMPT_LOGITS in tensors.keys() + else None + ) + return SequenceDiskCache.Payload( + tokens=tokens, + state_bytes=state_bytes, + prompt_logits=prompt_logits, + ) + + def _write_entry( + self, + path: Path, + tensors: Dict[str, np.ndarray], + ) -> None: + tmp_path = path.with_name(f"{path.name}.tmp") + try: + self._save_file(tensors, tmp_path, self._metadata) + os.replace(tmp_path, path) + except Exception: + try: + tmp_path.unlink() + except FileNotFoundError: + pass + raise + + def _load_entries(self) -> None: + for path in sorted(self.path.glob("*.safetensors")): + try: + header = self._read_entry_header(path) + if header is None: + continue + self._add_entry( + path=path, + tokens=header.tokens, + has_prompt_logits=header.has_prompt_logits, + last_accessed=path.stat().st_mtime, + ) + except Exception: # noqa: BLE001 + continue + + def _add_entry( + self, + *, + path: Path, + tokens: Tuple[int, ...], + has_prompt_logits: bool, + last_accessed: Optional[float] = None, + ) -> None: + existing = self._entries_by_tokens.get(tokens) + if existing is not None: + self._remove_entry(existing) + entry_id = self._next_entry_id + self._next_entry_id += 1 + size_bytes = path.stat().st_size + entry = SequenceDiskCache.Entry( + entry_id=entry_id, + path=path, + tokens=tokens, + size_bytes=size_bytes, + has_prompt_logits=has_prompt_logits, + last_accessed=last_accessed if last_accessed is not None else time.time(), + ) + self._entries_by_id[entry_id] = entry + self._entries_by_tokens[tokens] = entry + self._trie.extend(entry_id, tokens) + self._size_bytes += size_bytes + + def _remove_entry(self, entry: "SequenceDiskCache.Entry") -> None: + if entry.entry_id in self._trie.sequence_lengths: + self._trie.truncate(entry.entry_id, 0) + self._entries_by_id.pop(entry.entry_id, None) + if self._entries_by_tokens.get(entry.tokens) is entry: + self._entries_by_tokens.pop(entry.tokens, None) + self._size_bytes = max(0, self._size_bytes - entry.size_bytes) + try: + entry.path.unlink() + except FileNotFoundError: + pass + + @staticmethod + def _touch(entry: "SequenceDiskCache.Entry") -> None: + entry.last_accessed = time.time() + try: + os.utime(entry.path, None) + except OSError: + pass + + def _oldest_entry(self) -> Optional["SequenceDiskCache.Entry"]: + if not self._entries_by_id: + return None + return min( + self._entries_by_id.values(), + key=lambda item: (item.last_accessed, item.size_bytes), + ) + + def _evict_until(self, target_bytes: int) -> None: + while self._size_bytes > target_bytes: + entry = self._oldest_entry() + if entry is None: + return + self._remove_entry(entry) + + def _evict_if_needed(self) -> None: + if self.max_bytes <= 0: + for entry in list(self._entries_by_id.values()): + self._remove_entry(entry) + return + if self._size_bytes > self.max_bytes: + self._evict_until(self.max_bytes) + self._evict_until(int(self.max_bytes * 0.9)) + + def lookup(self, tokens: Sequence[int]) -> Optional[SequenceCache.Match]: + if not tokens: + return None + entry_id, _ = self._trie.longest_prefix(tokens) + entry = self._entries_by_id.get(entry_id) + if entry is None: + return None + self._touch(entry) + return SequenceCache.Match( + tokens=entry.tokens, + has_prompt_logits=entry.has_prompt_logits, + ) + + def load( + self, + match: SequenceCache.Match, + ) -> Optional[SequenceCache.Load]: + entry = self._entries_by_tokens.get(match.tokens) + if entry is None: + return None + + payload = self._read_entry_payload(entry) + self._touch(entry) + return SequenceCache.Load( + tokens=payload.tokens, + state_bytes=payload.state_bytes, + prompt_logits=( + np.asarray(payload.prompt_logits, dtype=np.float32) + if payload.prompt_logits is not None + else None + ), + ) + + def save( + self, + tokens: Sequence[int], + state_bytes: np.ndarray, + prompt_logits: Optional[np.ndarray], + ) -> None: + if len(tokens) < self.min_tokens or self.max_bytes <= 0: + return + state = np.asarray(state_bytes, dtype=np.uint8) + if state.size <= 0: + return + state = np.ascontiguousarray(state).copy() + entry_tokens = tuple(int(token) for token in tokens) + tensors: Dict[str, np.ndarray] = { + self.TENSOR_TOKENS: np.asarray(entry_tokens, dtype=np.int32), + self.TENSOR_STATE: state, + } + if prompt_logits is not None: + tensors[self.TENSOR_PROMPT_LOGITS] = np.asarray(prompt_logits, dtype=np.float32) + path = self.path / f"{uuid.uuid4().hex}.safetensors" + self._write_entry(path, tensors) + self._add_entry( + path=path, + tokens=entry_tokens, + has_prompt_logits=prompt_logits is not None, + ) + self._evict_if_needed() + + +class MemoryPolicy(abc.ABC): + def __init__(self, scheduler: CompletionScheduler) -> None: + self.scheduler = scheduler + + def reclaim_order(self, best_free: Optional[int]) -> List[int]: + reclaim_order = [seq_id for seq_id in self.scheduler.free_sequences if seq_id != best_free] + if best_free is not None: + reclaim_order.append(best_free) + return reclaim_order + + @staticmethod + def generation_kv_for_request( + request: CompletionRequest, + prompt_length: int, + ) -> int: + return request.internal_completion_count * ( + request.effective_max_len - prompt_length + ) + + @staticmethod + def attention_kv_required( + prompt_kv: int, + reused_kv: int, + generation_kv: int, + ) -> int: + return prompt_kv - reused_kv + generation_kv + + def try_set_sequence_cache_match( + self, + request: CompletionRequest, + resident_reuse_len: int, + required_sequence_ids: int, + required_attn_kv: Optional[int] = None, + skip_attention_budget_when_unbounded: bool = False, + ) -> bool: + cache_match_length = self.scheduler.find_sequence_cache_match( + request, + resident_reuse_len, + ) + if cache_match_length <= resident_reuse_len: + return False + has_sequence_budget = len(self.scheduler.unused_sequences) >= required_sequence_ids + has_attention_budget = required_attn_kv is None or ( + skip_attention_budget_when_unbounded + and not self.scheduler.model.has_attention_budget + ) or self.scheduler.sequence_history.size + required_attn_kv <= self.scheduler.model.n_ctx + if has_sequence_budget and has_attention_budget: + return True + self.scheduler.clear_sequence_cache_match(request) + return False + + @abc.abstractmethod + def can_admit(self, request: CompletionRequest) -> bool: + raise NotImplementedError + + @abc.abstractmethod + def admit_request(self, request: CompletionRequest) -> None: + raise NotImplementedError + + @abc.abstractmethod + def copy_prompt_state( + self, + source_sequence_id: int, + dest_sequence_id: int, + keep_len: int, + ) -> None: + raise NotImplementedError + + +class AttentionMemoryPolicy(MemoryPolicy): + def match_prefix(self, tokens: Sequence[int]) -> Tuple[int, int]: + return self.scheduler.radix_trie.longest_prefix( + tokens, + preferred_sequences=self.scheduler.free_sequences, + ) + + @staticmethod + def reuse_len_for_request(request: CompletionRequest, match_length: int) -> int: + needs_generation = ( + request.payload.max_tokens != 0 + and request.effective_max_len > len(request.prompt_tokens) + ) + reuse_len = match_length + if needs_generation and request.prompt_tokens: + reuse_len = min(reuse_len, len(request.prompt_tokens) - 1) + return request.prompt_plan.clamp_to_reusable_boundary(reuse_len) + + def admit_request(self, request: CompletionRequest) -> None: + match_seq_id = request.match_sequence_id + match_length = request.match_length + reuse_len = self.reuse_len_for_request(request, match_length) + claimable = match_seq_id in self.scheduler.free_sequences + if request.sequence_cache_match is not None: + base_seq_id = self.scheduler.claim_unused_sequence() + reuse_len, request.prompt_logits = self.scheduler.hydrate_sequence_cache_match( + request, + base_seq_id, + ) + elif claimable: + base_seq_id = self.scheduler.claim_free_sequence(match_seq_id) + if self.scheduler.radix_trie.length(base_seq_id) > reuse_len: + self.scheduler.truncate_sequence(base_seq_id, reuse_len) + else: + base_seq_id = self.scheduler.claim_unused_sequence() + if reuse_len > 0 and match_seq_id >= 0: + self.copy_prompt_state(match_seq_id, base_seq_id, reuse_len) + sibling_count = request.internal_completion_count - 1 + sibling_seq_ids: List[int] = [] + for _ in range(sibling_count): + seq_id = self.scheduler.claim_unused_sequence() + sibling_seq_ids.append(seq_id) + self.scheduler.activate_request( + request, + base_seq_id=base_seq_id, + sibling_seq_ids=sibling_seq_ids, + ) + request.prompt_cursor = reuse_len + if request.prompt_cursor == len(request.prompt_tokens): + request.prompt_done = True + self.scheduler.maybe_save_sequence_cache(request) + self.scheduler.start_completions( + request, + prompt_output_index=None, + prompt_logits=request.prompt_logits, + ) + + +class UnifiedAttentionMemoryPolicy(AttentionMemoryPolicy): + def can_admit(self, request: CompletionRequest) -> bool: + match_seq_id, match_length = self.match_prefix(request.prompt_tokens) + match_length = request.prompt_plan.clamp_to_reusable_boundary(match_length) + request.match_sequence_id = match_seq_id + request.match_length = match_length + claimable = match_seq_id in self.scheduler.free_sequences + required_sequence_ids = request.internal_completion_count - int(claimable) + prompt_length = len(request.prompt_tokens) + prompt_kv = request.prompt_plan.eval_token_count + reuse_len = self.reuse_len_for_request(request, match_length) + prefix_credit = match_length if claimable else reuse_len + prefix_credit_kv = max(0, min(prefix_credit, request.prompt_plan.length)) + generation_kv = self.generation_kv_for_request(request, prompt_length) + if self.try_set_sequence_cache_match( + request, + resident_reuse_len=reuse_len, + required_sequence_ids=request.internal_completion_count, + required_attn_kv=self.attention_kv_required( + prompt_kv, + reused_kv=0, + generation_kv=generation_kv, + ), + ): + return True + required_kv = self.attention_kv_required( + prompt_kv, + reused_kv=prefix_credit_kv, + generation_kv=generation_kv, + ) + if ( + len(self.scheduler.unused_sequences) >= required_sequence_ids + and self.scheduler.sequence_history.size + required_kv <= self.scheduler.model.n_ctx + ): + return True + best_free = match_seq_id if claimable else None + for seq_id in self.reclaim_order(best_free): + if len(self.scheduler.unused_sequences) < required_sequence_ids: + self.scheduler.delete_free_sequence(seq_id) + elif self.scheduler.sequence_history.size + required_kv > self.scheduler.model.n_ctx: + if seq_id == best_free and request.match_length > 0: + self.scheduler.truncate_free_sequence(seq_id, request.match_length) + else: + self.scheduler.delete_free_sequence(seq_id) + if ( + len(self.scheduler.unused_sequences) >= required_sequence_ids + and self.scheduler.sequence_history.size + required_kv <= self.scheduler.model.n_ctx + ): + request.match_sequence_id, request.match_length = self.match_prefix( + request.prompt_tokens, + ) + request.match_length = request.prompt_plan.clamp_to_reusable_boundary( + request.match_length + ) + return True + return False + + def copy_prompt_state( + self, + source_sequence_id: int, + dest_sequence_id: int, + keep_len: int, + ) -> None: + self.scheduler.copy_sequence_state( + source_sequence_id, + dest_sequence_id, + keep_len, + ) + + +class PartitionedAttentionMemoryPolicy(AttentionMemoryPolicy): + def can_admit(self, request: CompletionRequest) -> bool: + match_seq_id, match_length = self.match_prefix(request.prompt_tokens) + match_length = request.prompt_plan.clamp_to_reusable_boundary(match_length) + request.match_sequence_id = match_seq_id + request.match_length = match_length + claimable = match_seq_id in self.scheduler.free_sequences + required_sequence_ids = request.internal_completion_count - int(claimable) + reuse_len = self.reuse_len_for_request(request, match_length) + if self.try_set_sequence_cache_match( + request, + resident_reuse_len=reuse_len, + required_sequence_ids=request.internal_completion_count, + ): + return True + if len(self.scheduler.unused_sequences) >= required_sequence_ids: + return True + best_free = match_seq_id if claimable else None + for seq_id in self.reclaim_order(best_free): + self.scheduler.delete_free_sequence(seq_id) + if len(self.scheduler.unused_sequences) >= required_sequence_ids: + request.match_sequence_id, request.match_length = self.match_prefix( + request.prompt_tokens, + ) + request.match_length = request.prompt_plan.clamp_to_reusable_boundary( + request.match_length + ) + return True + return False + + def copy_prompt_state( + self, + source_sequence_id: int, + dest_sequence_id: int, + keep_len: int, + ) -> None: + self.scheduler.copy_sequence_state( + source_sequence_id, + dest_sequence_id, + keep_len, + copy_all_state=True, + ) + + +class CheckpointMemoryPolicy(MemoryPolicy): + def exact_checkpoint_match(self, tokens: Sequence[int]) -> Tuple[int, int]: + match_seq_id, match_length = self.scheduler.radix_trie.longest_prefix( + tokens, + preferred_sequences=self.scheduler.free_sequences, + exact_only=True, + ) + if match_seq_id not in self.scheduler.free_sequences: + return -1, 0 + return match_seq_id, match_length + + def can_admit(self, request: CompletionRequest) -> bool: + match_seq_id, match_length = self.exact_checkpoint_match(request.prompt_tokens) + request.match_sequence_id = match_seq_id + request.match_length = match_length + claimable = match_seq_id in self.scheduler.free_sequences + required_sequence_ids = request.internal_completion_count - int(claimable) + prompt_length = len(request.prompt_tokens) + prompt_kv = request.prompt_plan.eval_token_count + generation_kv = self.generation_kv_for_request(request, prompt_length) + if self.try_set_sequence_cache_match( + request, + resident_reuse_len=match_length, + required_sequence_ids=request.internal_completion_count, + required_attn_kv=self.attention_kv_required( + prompt_kv, + reused_kv=0, + generation_kv=generation_kv, + ), + skip_attention_budget_when_unbounded=True, + ): + return True + required_attn_kv = self.attention_kv_required( + prompt_kv, + reused_kv=max(0, min(match_length, request.prompt_plan.length)), + generation_kv=generation_kv, + ) + if len(self.scheduler.unused_sequences) >= required_sequence_ids and ( + not self.scheduler.model.has_attention_budget + or self.scheduler.sequence_history.size + required_attn_kv <= self.scheduler.model.n_ctx + ): + return True + best_free = match_seq_id if claimable else None + for seq_id in self.reclaim_order(best_free): + self.scheduler.delete_free_sequence(seq_id) + if len(self.scheduler.unused_sequences) >= required_sequence_ids and ( + not self.scheduler.model.has_attention_budget + or self.scheduler.sequence_history.size + required_attn_kv <= self.scheduler.model.n_ctx + ): + request.match_sequence_id, request.match_length = self.exact_checkpoint_match( + request.prompt_tokens, + ) + return True + return False + + def admit_request(self, request: CompletionRequest) -> None: + match_seq_id = request.match_sequence_id + match_length = request.match_length + claimable = match_seq_id in self.scheduler.free_sequences + if request.sequence_cache_match is not None: + base_seq_id = self.scheduler.claim_unused_sequence() + match_length, request.prompt_logits = self.scheduler.hydrate_sequence_cache_match( + request, + base_seq_id, + ) + elif claimable: + base_seq_id = self.scheduler.claim_free_sequence(match_seq_id) + request.prompt_logits = self.scheduler.checkpoint_logits.get(base_seq_id) + self.scheduler.metrics.checkpoint_hits_total += 1 + else: + base_seq_id = self.scheduler.claim_unused_sequence() + request.prompt_logits = None + sibling_count = request.internal_completion_count - 1 + sibling_seq_ids: List[int] = [] + for _ in range(sibling_count): + seq_id = self.scheduler.claim_unused_sequence() + sibling_seq_ids.append(seq_id) + self.scheduler.activate_request( + request, + base_seq_id=base_seq_id, + sibling_seq_ids=sibling_seq_ids, + ) + request.prompt_cursor = match_length + if request.prompt_cursor == len(request.prompt_tokens): + request.prompt_done = True + self.scheduler.maybe_save_prompt_checkpoint(request) + self.scheduler.maybe_save_sequence_cache(request) + self.scheduler.start_completions( + request, + prompt_output_index=None, + prompt_logits=request.prompt_logits, + ) + + def copy_prompt_state( + self, + source_sequence_id: int, + dest_sequence_id: int, + keep_len: int, + ) -> None: + self.scheduler.copy_sequence_state( + source_sequence_id, + dest_sequence_id, + keep_len, + ) + + +class CompletionScheduler: + @dataclass + class BatchItem: + @dataclass + class Prefill: + embeddings: Optional[np.ndarray] = None + positions: Optional[np.ndarray] = None + non_causal: bool = False + prompt_advance_to: Optional[int] = None + + @dataclass + class Decode: + completion_index: int + pending_count: int + accepted_draft_count: int = 0 + sampled_pending_token: Optional[int] = None + rollback_keep_len: Optional[int] = None + rollback_accepted_draft_count: int = 0 + rollback_draft_processed: bool = False + deferred_accept_draft_count: Optional[int] = None + deferred_truncate_draft_len: Optional[int] = None + + kind: Literal["prefill", "decode"] + request_id: str + seq_id: int + start_pos: int + llama_start_pos: int + tokens: List[int] + identity_tokens: List[int] + output_indices: List[Optional[int]] + output_positions: List[int] + position_increments: List[int] + prefill_state: Optional["CompletionScheduler.BatchItem.Prefill"] = None + decode_state: Optional["CompletionScheduler.BatchItem.Decode"] = None + + def require_prefill(self) -> "CompletionScheduler.BatchItem.Prefill": + if self.prefill_state is None: + raise RuntimeError("batch item is not a prefill item") + return self.prefill_state + + def require_decode(self) -> "CompletionScheduler.BatchItem.Decode": + if self.decode_state is None: + raise RuntimeError("batch item is not a decode item") + return self.decode_state + + @property + def batch_token_count(self) -> int: + if ( + self.prefill_state is not None + and self.prefill_state.embeddings is not None + ): + return int(self.prefill_state.embeddings.shape[0]) + return len(self.tokens) + + @classmethod + def prefill( + cls, + *, + request_id: str, + seq_id: int, + start_pos: int, + llama_start_pos: int, + tokens: List[int], + identity_tokens: List[int], + output_indices: List[Optional[int]], + output_positions: List[int], + position_increments: List[int], + embeddings: Optional[np.ndarray] = None, + positions: Optional[np.ndarray] = None, + non_causal: bool = False, + prompt_advance_to: Optional[int] = None, + ) -> "CompletionScheduler.BatchItem": + return cls( + kind="prefill", + request_id=request_id, + seq_id=seq_id, + start_pos=start_pos, + llama_start_pos=llama_start_pos, + tokens=tokens, + identity_tokens=identity_tokens, + output_indices=output_indices, + output_positions=output_positions, + position_increments=position_increments, + prefill_state=cls.Prefill( + embeddings=embeddings, + positions=positions, + non_causal=non_causal, + prompt_advance_to=prompt_advance_to, + ), + ) + + @classmethod + def decode( + cls, + *, + request_id: str, + seq_id: int, + start_pos: int, + llama_start_pos: int, + tokens: List[int], + identity_tokens: List[int], + output_indices: List[Optional[int]], + output_positions: List[int], + position_increments: List[int], + completion_index: int, + pending_count: int, + ) -> "CompletionScheduler.BatchItem": + return cls( + kind="decode", + request_id=request_id, + seq_id=seq_id, + start_pos=start_pos, + llama_start_pos=llama_start_pos, + tokens=tokens, + identity_tokens=identity_tokens, + output_indices=output_indices, + output_positions=output_positions, + position_increments=position_increments, + decode_state=cls.Decode( + completion_index=completion_index, + pending_count=pending_count, + ), + ) + + def __init__( + self, + model: Model, + sequence_cache: Optional[SequenceCache] = None, + ) -> None: + self.model = model + self.sequence_cache = sequence_cache + self.formatter = OpenAIFormatter(model) + self.radix_trie = RadixTrie() + self.sequence_history = SequenceHistory() + self.metrics = SchedulerMetrics() + self.checkpoint_logits: Dict[int, np.ndarray] = {} + self.claimed_sequences: set[int] = set() + self.free_sequences: "OrderedDict[int, None]" = OrderedDict() + self.unused_sequences: List[int] = list(range(self.model.n_seq_max - 1, -1, -1)) + self.requests: Dict[str, CompletionRequest] = {} + self.pending_requests: Deque[CompletionRequest] = deque() + self.active_request_ids: set[str] = set() + self.closed = False + self.sequence_round_robin = 0 + self.speculative_stats: Dict[str, int] = { + "draft_proposals": 0, + "draft_tokens_proposed": 0, + "draft_tokens_accepted": 0, + "draft_tokens_rejected": 0, + } + self.draft_acceptance_length_counts: Dict[int, int] = {} + self.defer_sampled_draft_processing = False + self.memory_policy = self.build_memory_policy() + + def build_memory_policy(self) -> MemoryPolicy: + if self.model.exact_checkpoints_only: + return CheckpointMemoryPolicy(self) + if self.model.attention_partitioned: + return PartitionedAttentionMemoryPolicy(self) + return UnifiedAttentionMemoryPolicy(self) + + @staticmethod + def request_needs_prompt_logits(request: CompletionRequest) -> bool: + return request.payload.max_tokens != 0 and request.effective_max_len > len( + request.prompt_tokens + ) + + @staticmethod + def request_needs_uncached_prompt_logprobs(request: CompletionRequest) -> bool: + return request.payload.echo and request.payload.logprobs is not None + + @staticmethod + def clear_sequence_cache_match(request: CompletionRequest) -> None: + request.sequence_cache_match = None + request.sequence_cache_match_length = 0 + + def can_lookup_sequence_cache(self, request: CompletionRequest) -> bool: + return ( + self.sequence_cache is not None + and bool(request.prompt_tokens) + and not self.request_needs_uncached_prompt_logprobs(request) + ) + + def is_sequence_cache_match_usable( + self, + request: CompletionRequest, + match: SequenceCache.Match, + resident_reuse_len: int, + ) -> bool: + match_length = match.length + if ( + match_length <= resident_reuse_len + or match_length > len(request.prompt_tokens) + or tuple(request.prompt_tokens[:match_length]) != match.tokens + or not request.prompt_plan.is_reusable_boundary(match_length) + ): + return False + return not ( + self.request_needs_prompt_logits(request) + and match_length == len(request.prompt_tokens) + and not match.has_prompt_logits + ) + + def find_sequence_cache_match( + self, + request: CompletionRequest, + resident_reuse_len: int, + ) -> int: + self.clear_sequence_cache_match(request) + if not self.can_lookup_sequence_cache(request): + return 0 + assert self.sequence_cache is not None + try: + match = self.sequence_cache.lookup(request.prompt_tokens) + except Exception: # noqa: BLE001 + self.metrics.sequence_cache_lookup_failures_total += 1 + return 0 + if match is None: + return 0 + if not self.is_sequence_cache_match_usable( + request, + match, + resident_reuse_len, + ): + return 0 + request.sequence_cache_match = match + request.sequence_cache_match_length = match.length + return match.length + + def fail_sequence_cache_load( + self, + request: CompletionRequest, + seq_id: int, + ) -> Tuple[int, Optional[np.ndarray]]: + llama_cpp.llama_memory_seq_rm(self.model.mem, seq_id, 0, -1) + self.model.truncate_draft_sequence(seq_id, 0) + self.clear_sequence_cache_match(request) + self.metrics.sequence_cache_load_failures_total += 1 + return 0, None + + def load_sequence_state_bytes( + self, + seq_id: int, + state_bytes: np.ndarray, + ) -> bool: + state = np.ascontiguousarray(state_bytes, dtype=np.uint8) + state_size = int(state.size) + if state_size <= 0: + return False + state_buffer = (ctypes.c_uint8 * state_size).from_buffer(cast(Any, state)) + loaded_bytes = int( + llama_cpp.llama_state_seq_set_data( + self.model.ctx, + state_buffer, + state_size, + cast(llama_cpp.llama_seq_id, seq_id), + ) + ) + return loaded_bytes == state_size + + def save_sequence_state_bytes(self, seq_id: int) -> Optional[np.ndarray]: + c_seq_id = cast(llama_cpp.llama_seq_id, seq_id) + state_size = int(llama_cpp.llama_state_seq_get_size(self.model.ctx, c_seq_id)) + if state_size <= 0: + return None + state_buffer = (ctypes.c_uint8 * state_size)() + state_bytes = int( + llama_cpp.llama_state_seq_get_data( + self.model.ctx, + state_buffer, + state_size, + c_seq_id, + ) + ) + if state_bytes <= 0: + return None + return np.ctypeslib.as_array(state_buffer, shape=(state_bytes,)).copy() + + def hydrate_sequence_cache_match( + self, + request: CompletionRequest, + seq_id: int, + ) -> Tuple[int, Optional[np.ndarray]]: + match = request.sequence_cache_match + if self.sequence_cache is None or match is None: + return 0, None + try: + loaded = self.sequence_cache.load(match) + except Exception: # noqa: BLE001 + return self.fail_sequence_cache_load(request, seq_id) + if loaded is None: + return self.fail_sequence_cache_load(request, seq_id) + tokens = list(loaded.tokens) + expected_length = request.sequence_cache_match_length + if ( + len(tokens) != expected_length + or tokens != request.prompt_tokens[:expected_length] + ): + return self.fail_sequence_cache_load(request, seq_id) + if ( + len(tokens) == len(request.prompt_tokens) + and self.request_needs_prompt_logits(request) + and loaded.prompt_logits is None + ): + return self.fail_sequence_cache_load(request, seq_id) + if not self.load_sequence_state_bytes(seq_id, loaded.state_bytes): + return self.fail_sequence_cache_load(request, seq_id) + self.radix_trie.extend(seq_id, tokens) + self.sequence_history.extend( + seq_id, + tokens, + request.prompt_plan.position_increments_up_to(len(tokens)), + ) + self.model.truncate_draft_sequence(seq_id, 0) + self.metrics.sequence_cache_hits_total += 1 + self.metrics.sequence_cache_tokens_loaded_total += len(tokens) + if len(tokens) == len(request.prompt_tokens): + return len(tokens), loaded.prompt_logits + return len(tokens), None + + def maybe_save_sequence_cache(self, request: CompletionRequest) -> None: + if ( + self.sequence_cache is None + or request.base_seq_id is None + or not request.prompt_tokens + or request.sequence_cache_match_length == len(request.prompt_tokens) + ): + return + state_bytes = self.save_sequence_state_bytes(request.base_seq_id) + if state_bytes is None: + return + try: + self.sequence_cache.save( + request.prompt_tokens, + state_bytes, + request.prompt_logits, + ) + except Exception: # noqa: BLE001 + self.metrics.sequence_cache_save_failures_total += 1 + return + self.metrics.sequence_cache_save_requests_total += 1 + + def close(self) -> None: + self.closed = True + self.model.close() + + def submit_request(self, request: CompletionRequest) -> str: + if self.closed: + raise RuntimeError("scheduler closed") + self.requests[request.id] = request + self.pending_requests.append(request) + self.metrics.requests_submitted_total += 1 + return request.id + + def cancel(self, request_id: str) -> None: + request = self.requests.get(request_id) + if request is not None: + request.cancelled = True + + def is_idle(self) -> bool: + return not self.pending_requests and not self.active_request_ids + + @staticmethod + def logits_to_logprobs(logits: np.ndarray) -> np.ndarray: + logits = logits.astype(np.float32, copy=False) + max_logit = float(np.max(logits)) + shifted = logits - max_logit + return shifted - math.log(float(np.sum(np.exp(shifted, dtype=np.float64)))) + + def step(self) -> bool: + step_started_at = time.perf_counter() + if self.closed: + return False + try: + self.admit_waiting() + batch_items = self.build_batch() + if not batch_items: + if self.maybe_fill_batched_draft_tokens(): + self.finalize_cancelled() + return True + return self.finalize_cancelled() + draft_processing_enabled = self.batch_needs_draft_processing(batch_items) + self.defer_sampled_draft_processing = ( + draft_processing_enabled + and self.supports_sampled_draft_processing + and all(item.kind == "decode" for item in batch_items) + ) + self.model.set_draft_processing_enabled(draft_processing_enabled) + self.model.clear_batch() + for item in batch_items: + prefill = item.prefill_state + if prefill is not None and prefill.embeddings is not None: + if prefill.positions is None: + raise RuntimeError("embedding batch item is missing positions") + self.model.add_batch_embeddings( + seq_id=item.seq_id, + embeddings=prefill.embeddings, + positions=prefill.positions, + output_indices=item.output_indices, + ) + else: + self.model.add_batch_tokens( + seq_id=item.seq_id, + start_pos=item.llama_start_pos, + tokens=item.tokens, + output_indices=item.output_indices, + ) + decode_started_at = time.perf_counter() + try: + non_causal_decode = any( + item.prefill_state is not None and item.prefill_state.non_causal + for item in batch_items + ) + try: + if non_causal_decode: + llama_cpp.llama_set_causal_attn(self.model.ctx, False) + self.model.decode() + finally: + if non_causal_decode: + llama_cpp.llama_set_causal_attn(self.model.ctx, True) + decode_elapsed = time.perf_counter() - decode_started_at + if draft_processing_enabled and not self.defer_sampled_draft_processing: + draft_process_started_at = time.perf_counter() + self.model.process_draft_batch() + draft_process_elapsed = time.perf_counter() - draft_process_started_at + self.metrics.draft_process_seconds_total += draft_process_elapsed + self.metrics.draft_process_calls_total += 1 + self.observe_draft_process(batch_items, draft_process_elapsed) + except BaseException as exc: # noqa: BLE001 + for request_id in list(self.active_request_ids): + self.fail_request(self.requests[request_id], exc) + for request in list(self.pending_requests): + self.fail_request(request, exc) + return True + self.metrics.observe_decode( + batch_items, + decode_elapsed, + ) + process_batch_started_at = time.perf_counter() + self.process_batch(batch_items) + self.metrics.process_batch_seconds_total += ( + time.perf_counter() - process_batch_started_at + ) + self.metrics.process_batch_calls_total += 1 + if self.defer_sampled_draft_processing: + try: + self.process_sampled_draft_batch(batch_items) + finally: + self.apply_deferred_draft_rollbacks(batch_items) + self.defer_sampled_draft_processing = False + self.finalize_cancelled() + return True + finally: + self.metrics.scheduler_step_seconds_total += ( + time.perf_counter() - step_started_at + ) + self.metrics.scheduler_step_calls_total += 1 + + def observe_draft_process( + self, + items: Sequence["CompletionScheduler.BatchItem"], + elapsed_seconds: float, + ) -> None: + if self.model.draft_provider is None or elapsed_seconds <= 0.0: + return + self.metrics.draft_seconds_total += elapsed_seconds + + def item_allows_mtp_processing(self, item: "CompletionScheduler.BatchItem") -> bool: + request = self.requests.get(item.request_id) + if request is None: + return False + if item.kind == "prefill": + return not any(segment.kind != "text" for segment in request.prompt_plan.segments) + decode = item.decode_state + if decode is None: + return False + return not request.completions[decode.completion_index].multimodal_prompt + + def batch_needs_draft_processing( + self, + batch_items: Sequence["CompletionScheduler.BatchItem"], + ) -> bool: + if self.model.draft_provider is None: + return False + if not self.draft_batch_size_allowed(batch_items): + return False + if isinstance(self.model.draft_provider, MTPDraftProvider) and not all( + self.item_allows_mtp_processing(item) for item in batch_items + ): + return False + for item in batch_items: + request = self.requests.get(item.request_id) + if request is None: + continue + if item.kind == "prefill": + if self.request_needs_prompt_logits(request): + return True + continue + decode = item.decode_state + if decode is None: + continue + completion = request.completions[decode.completion_index] + if not completion.finished: + return True + return False + + def draft_batch_size_allowed( + self, + batch_items: Sequence["CompletionScheduler.BatchItem"], + ) -> bool: + max_batch_size = getattr(self.model, "draft_model_max_batch_size", None) + if max_batch_size is None: + return True + draft_batch_size = sum( + 1 + for item in batch_items + if item.kind == "decode" and item.decode_state is not None + ) + return draft_batch_size <= max_batch_size + + def active_draft_batch_size_allowed(self, draft_batch_size: int) -> bool: + max_batch_size = getattr(self.model, "draft_model_max_batch_size", None) + return max_batch_size is None or draft_batch_size <= max_batch_size + + def admit_waiting(self) -> None: + while self.pending_requests: + request = self.pending_requests[0] + if request.cancelled: + self.fail_request(request, CompletionRequestCancelledError("request cancelled")) + continue + if not self.can_admit(request): + break + self.pending_requests.popleft() + self.admit_request(request) + + def can_admit(self, request: CompletionRequest) -> bool: + return self.memory_policy.can_admit(request) + + def admit_request(self, request: CompletionRequest) -> None: + self.memory_policy.admit_request(request) + if request.admitted: + self.metrics.requests_admitted_total += 1 + + def build_batch(self) -> List[CompletionScheduler.BatchItem]: + prompt_requests = [ + self.requests[request_id] + for request_id in self.active_request_ids + if self.requests[request_id].admitted + and not self.requests[request_id].prompt_done + and not self.requests[request_id].cancelled + ] + completions = [ + completion + for request_id in self.active_request_ids + for completion in self.requests[request_id].completions + if (completion.pending_input_tokens or completion.draft_tokens) + and not completion.finished + ] + if not prompt_requests and not completions: + return [] + + for request in prompt_requests: + if request.prompt_cursor < len(request.prompt_tokens): + segment = self._current_prompt_segment(request) + if segment.kind != "text": + token_count = min( + segment.batch_rows + if segment.media is not None and segment.media.non_causal + else self._pending_tokens_length(request), + self.model.n_batch, + ) + if segment.batch_rows > self.model.n_batch: + if segment.media is not None and segment.media.non_causal: + raise RuntimeError( + "non-causal media prompt segment exceeds model.n_batch; " + "increase n_batch" + ) + item, _ = self._build_pending_batch_item( + request, + token_count, + 0, + ) + return [item] + + ordered_sequences = self._ordered_pending_sequences(prompt_requests, completions) + allocations = self._allocate_pending_tokens_for_model(ordered_sequences) + if ordered_sequences: + self.sequence_round_robin += 1 + + items: List[CompletionScheduler.BatchItem] = [] + output_index = 0 + for source in ordered_sequences: + token_count = allocations.get(self._pending_sequence_id(source), 0) + if token_count <= 0: + continue + item, output_index = self._build_pending_batch_item( + source, + token_count, + output_index, + ) + items.append(item) + return self._split_mixed_mtp_batch(items) + + def _split_mixed_mtp_batch( + self, + items: List["CompletionScheduler.BatchItem"], + ) -> List["CompletionScheduler.BatchItem"]: + if not isinstance(self.model.draft_provider, MTPDraftProvider) or len(items) <= 1: + return items + eligibility = [self.item_allows_mtp_processing(item) for item in items] + if all(eligibility) or not any(eligibility): + return items + keep_eligible = eligibility[0] + split_items = [ + item + for item, item_eligible in zip(items, eligibility) + if item_eligible == keep_eligible + ] + self._renumber_output_indices(split_items) + return split_items + + @staticmethod + def _renumber_output_indices( + items: Sequence["CompletionScheduler.BatchItem"], + ) -> None: + next_output_index = 0 + for item in items: + for index, output_index in enumerate(item.output_indices): + if output_index is None: + continue + item.output_indices[index] = next_output_index + next_output_index += 1 + + def _allocate_pending_tokens_for_model( + self, + ordered_sequences: Sequence[Union[CompletionRequest, Completion]], + ) -> Dict[int, int]: + if not self._needs_homogeneous_recurrent_draft_batching(): + return self._allocate_pending_tokens(ordered_sequences, self.model.n_batch) + + active = [ + source + for source in ordered_sequences + if self._pending_tokens_length(source) > 0 + ] + if not active: + return {} + + first_draft_count = self._recurrent_draft_batch_token_count(active[0]) + if first_draft_count > 0: + return self._allocate_homogeneous_recurrent_draft_tokens( + active, + first_draft_count, + ) + + non_draft_sources = [ + source + for source in active + if self._recurrent_draft_batch_token_count(source) == 0 + ] + return self._allocate_pending_tokens(non_draft_sources, self.model.n_batch) + + def _needs_homogeneous_recurrent_draft_batching(self) -> bool: + return bool( + self.model.draft_provider is not None + and getattr(self.model, "draft_target_batching", False) + and self.model.exact_checkpoints_only + and self.model.n_rs_seq > 0 + ) + + def _recurrent_draft_batch_token_count( + self, + source: Union[CompletionRequest, Completion], + ) -> int: + if isinstance(source, CompletionRequest): + return 0 + if ( + source.finished + or source.pending_finish_reason is not None + or not source.pending_input_tokens + or not source.draft_tokens + ): + return 0 + token_count = min( + len(source.pending_input_tokens) + len(source.draft_tokens), + len(source.pending_input_tokens) + self.model.n_rs_seq, + ) + return token_count if token_count > len(source.pending_input_tokens) else 0 + + def _allocate_homogeneous_recurrent_draft_tokens( + self, + sources: Sequence[Union[CompletionRequest, Completion]], + token_count: int, + ) -> Dict[int, int]: + # Recurrent rollback snapshots are only valid if the whole per-sequence + # pending+draft group fits in a single llama.cpp ubatch. + safe_capacity = min(self.model.n_batch, self.model.n_ubatch) + if token_count > safe_capacity: + return self._allocate_recurrent_pending_only_tokens(sources, safe_capacity) + + allocations: Dict[int, int] = {} + remaining_capacity = safe_capacity + for source in sources: + if remaining_capacity < token_count: + break + if self._recurrent_draft_batch_token_count(source) != token_count: + continue + seq_id = self._pending_sequence_id(source) + allocations[seq_id] = token_count + remaining_capacity -= token_count + return allocations + + def _allocate_recurrent_pending_only_tokens( + self, + sources: Sequence[Union[CompletionRequest, Completion]], + capacity: int, + ) -> Dict[int, int]: + allocations: Dict[int, int] = {} + remaining_capacity = capacity + for source in sources: + if remaining_capacity <= 0: + break + if self._recurrent_draft_batch_token_count(source) == 0: + continue + if isinstance(source, CompletionRequest): + continue + pending_count = min(len(source.pending_input_tokens), remaining_capacity) + if pending_count <= 0: + continue + allocations[source.seq_id] = pending_count + remaining_capacity -= pending_count + return allocations + + def _ordered_pending_sequences( + self, + prompt_requests: Sequence[CompletionRequest], + completions: Sequence[Completion], + ) -> List[Union[CompletionRequest, Completion]]: + if not prompt_requests and not completions: + return [] + prompt_by_request_id = {request.id: request for request in prompt_requests} + completions_by_request_id: Dict[str, List[Completion]] = {} + for completion in completions: + completions_by_request_id.setdefault(completion.request_id, []).append(completion) + sequence_list: List[Union[CompletionRequest, Completion]] = [] + for request_id, request in self.requests.items(): + if request_id not in self.active_request_ids or request.cancelled: + continue + prompt_request = prompt_by_request_id.get(request_id) + if prompt_request is not None: + sequence_list.append(prompt_request) + sequence_list.extend(completions_by_request_id.get(request_id, [])) + if not sequence_list: + return [] + start = self.sequence_round_robin % len(sequence_list) + return sequence_list[start:] + sequence_list[:start] + + def _pending_sequence_id( + self, + source: Union[CompletionRequest, Completion], + ) -> int: + if isinstance(source, CompletionRequest): + if source.base_seq_id is None: + raise RuntimeError("prompt sequence is missing base seq id") + return source.base_seq_id + return source.seq_id + + def _current_prompt_segment(self, request: CompletionRequest) -> PromptSegment: + return request.prompt_plan.segment_at(request.prompt_cursor) + + def _pending_tokens_length( + self, + source: Union[CompletionRequest, Completion], + ) -> int: + if isinstance(source, CompletionRequest): + if source.prompt_cursor >= len(source.prompt_tokens): + return 0 + segment = self._current_prompt_segment(source) + if segment.kind != "text": + segment_offset = source.prompt_cursor - segment.start_pos + if segment.media is not None and segment.media.non_causal: + if segment_offset != 0: + raise RuntimeError("non-causal media segment was partially scheduled") + return segment.batch_rows + return segment.end_pos - source.prompt_cursor + return segment.end_pos - source.prompt_cursor + draft_count = ( + len(source.draft_tokens) + if source.pending_finish_reason is None + and getattr(self.model, "draft_target_batching", True) + else 0 + ) + return len(source.pending_input_tokens) + draft_count + + def _pending_allocation_for_capacity( + self, + source: Union[CompletionRequest, Completion], + already_allocated: int, + capacity: int, + ) -> int: + if capacity <= 0: + return 0 + if not isinstance(source, CompletionRequest): + remaining = self._pending_tokens_length(source) - already_allocated + return min(remaining, capacity) + segment = self._current_prompt_segment(source) + if segment.kind == "text": + remaining = self._pending_tokens_length(source) - already_allocated + return min(remaining, capacity) + if segment.media is not None and segment.media.non_causal: + if already_allocated == 0 and segment.batch_rows <= capacity: + return segment.batch_rows + return 0 + segment_offset = source.prompt_cursor - segment.start_pos + already_allocated + return segment.rows_for_capacity(segment_offset, capacity) + + def _allocate_pending_tokens( + self, + sources: Sequence[Union[CompletionRequest, Completion]], + capacity: int, + ) -> Dict[int, int]: + allocations: Dict[int, int] = {} + active = [source for source in sources if self._pending_tokens_length(source) > 0] + remaining_capacity = capacity + + for source in active: + if remaining_capacity <= 0: + break + seq_id = self._pending_sequence_id(source) + allocation = self._pending_allocation_for_capacity( + source, + allocations.get(seq_id, 0), + remaining_capacity, + ) + if allocation <= 0: + continue + allocations[seq_id] = allocations.get(seq_id, 0) + allocation + remaining_capacity -= allocation + + while remaining_capacity > 0 and active: + share = max(1, remaining_capacity // len(active)) + progress = False + next_active: List[Union[CompletionRequest, Completion]] = [] + for source in active: + seq_id = self._pending_sequence_id(source) + remaining_tokens = self._pending_tokens_length(source) - allocations.get(seq_id, 0) + if remaining_tokens <= 0: + continue + allocation = self._pending_allocation_for_capacity( + source, + allocations.get(seq_id, 0), + min(share, remaining_capacity), + ) + if allocation <= 0: + next_active.append(source) + continue + allocations[seq_id] = allocations.get(seq_id, 0) + allocation + remaining_capacity -= allocation + progress = True + if remaining_tokens > allocation and remaining_capacity > 0: + next_active.append(source) + if not progress: + break + active = next_active + return allocations + + def _build_pending_batch_item( + self, + source: Union[CompletionRequest, Completion], + token_count: int, + output_index: int, + ) -> Tuple["CompletionScheduler.BatchItem", int]: + if isinstance(source, CompletionRequest): + if source.base_seq_id is None: + raise RuntimeError("prompt sequence is missing base seq id") + segment = self._current_prompt_segment(source) + if segment.kind != "text": + segment_offset = source.prompt_cursor - segment.start_pos + non_causal = segment.media is not None and segment.media.non_causal + if non_causal: + if segment_offset != 0 or token_count < segment.batch_rows: + raise RuntimeError("non-causal media segment must be scheduled atomically") + row_count = segment.batch_rows + embeddings, positions, position_increments = segment.media_slice( + 0, + row_count, + ) + else: + row_count = segment.rows_for_capacity(segment_offset, token_count) + if row_count <= 0: + raise RuntimeError("media prompt allocation is too small") + embeddings, positions, position_increments = segment.media_slice( + segment_offset, + row_count, + ) + logical_start = source.prompt_cursor + logical_end = logical_start + row_count + output_indices = [None] * row_count + output_positions = [logical_start] * row_count + if output_positions: + output_positions[-1] = logical_end - 1 + needs_last_output = ( + (source.payload.echo and source.payload.logprobs is not None + and source.prompt_plan.has_text_token_at(logical_end)) + or self.model.exact_checkpoints_only + or logical_end == len(source.prompt_tokens) + ) + if needs_last_output and output_indices: + output_indices[-1] = output_index + output_index += 1 + return ( + CompletionScheduler.BatchItem.prefill( + request_id=source.id, + seq_id=source.base_seq_id, + start_pos=logical_start, + llama_start_pos=segment.decode_start_pos + sum( + segment.decoder_position_increments[:segment_offset] + ), + tokens=[], + identity_tokens=list( + segment.identity_tokens[ + segment_offset : segment_offset + row_count + ] + ), + output_indices=output_indices, + output_positions=output_positions, + position_increments=position_increments, + embeddings=embeddings, + positions=positions, + non_causal=non_causal, + prompt_advance_to=logical_end, + ), + output_index, + ) + segment_offset = source.prompt_cursor - segment.start_pos + max_count = min(token_count, segment.end_pos - source.prompt_cursor) + chunk = list(segment.text_tokens[segment_offset : segment_offset + max_count]) + identity_chunk = list( + segment.identity_tokens[segment_offset : segment_offset + max_count] + ) + ends_prompt = source.prompt_cursor + len(identity_chunk) == len(source.prompt_tokens) + output_indices: List[Optional[int]] = [None] * len(chunk) + output_positions = [source.prompt_cursor + index for index in range(len(chunk))] + for index, output_position in enumerate(output_positions): + if ( + source.payload.echo + and source.payload.logprobs is not None + and source.prompt_plan.has_text_token_at(output_position + 1) + ): + output_indices[index] = output_index + output_index += 1 + if self.model.exact_checkpoints_only and chunk and output_indices[-1] is None: + output_indices[-1] = output_index + output_index += 1 + elif ends_prompt and chunk and output_indices[-1] is None: + output_indices[-1] = output_index + output_index += 1 + return ( + CompletionScheduler.BatchItem.prefill( + request_id=source.id, + seq_id=source.base_seq_id, + start_pos=source.prompt_cursor, + llama_start_pos=segment.decode_start_pos + segment_offset, + tokens=chunk, + identity_tokens=identity_chunk, + output_indices=output_indices, + output_positions=output_positions, + position_increments=segment.decoder_position_increments[ + segment_offset : segment_offset + len(identity_chunk) + ], + ), + output_index, + ) + + request = self.requests[source.request_id] + pending_count = min(token_count, len(source.pending_input_tokens)) + draft_count = ( + max(0, token_count - pending_count) + if getattr(self.model, "draft_target_batching", True) + else 0 + ) + scheduled_tokens = [ + *source.pending_input_tokens[:pending_count], + *( + source.draft_tokens[:draft_count] + if source.pending_finish_reason is None + else [] + ), + ] + output_indices = list(range(output_index, output_index + len(scheduled_tokens))) + start_pos = self.radix_trie.length(source.seq_id) + return ( + CompletionScheduler.BatchItem.decode( + request_id=request.id, + seq_id=source.seq_id, + start_pos=start_pos, + llama_start_pos=self.sequence_history.position_length(source.seq_id), + tokens=list(scheduled_tokens), + identity_tokens=list(scheduled_tokens), + output_indices=output_indices, + output_positions=[ + start_pos + index + for index in range(len(scheduled_tokens)) + ], + position_increments=[1] * len(scheduled_tokens), + completion_index=source.index, + pending_count=pending_count, + ), + output_index + len(scheduled_tokens), + ) + + def process_batch(self, items: List[CompletionScheduler.BatchItem]) -> None: + output_count = sum( + output_index is not None + for item in items + for output_index in item.output_indices + ) + for item in items: + request = self.requests[item.request_id] + if request.cancelled: + continue + self.radix_trie.extend(item.seq_id, item.identity_tokens) + self.sequence_history.extend( + item.seq_id, + item.identity_tokens, + item.position_increments, + ) + if item.kind == "prefill": + request.capture_prompt_logprobs( + model=self.model, + formatter=self.formatter, + output_indices=item.output_indices, + output_positions=item.output_positions, + output_count=output_count, + output_index_to_logits_index=self.output_index_to_logits_index, + ) + prompt_output_index = self.output_index_to_logits_index( + self.last_output_index(item.output_indices), + output_count, + ) + prompt_logits = ( + self.model.logits(prompt_output_index) + if prompt_output_index is not None + else None + ) + prefill = item.require_prefill() + if prefill.prompt_advance_to is not None: + request.prompt_cursor = prefill.prompt_advance_to + else: + request.prompt_cursor += len(item.identity_tokens) + if request.prompt_cursor == len(request.prompt_tokens): + request.prompt_done = True + request.prompt_logits = prompt_logits + self.maybe_save_prompt_checkpoint(request) + self.maybe_save_sequence_cache(request) + self.start_completions( + request, + prompt_output_index=prompt_output_index, + prompt_logits=request.prompt_logits, + ) + else: + decode = item.require_decode() + completion = request.completions[decode.completion_index] + self.process_generation_item( + completion, + item, + output_count, + ) + self.finalize_request_if_ready(request) + if not self.defer_sampled_draft_processing: + self.maybe_fill_batched_draft_tokens() + + @property + def should_defer_draft_fill(self) -> bool: + return bool( + self.model.draft_provider is not None + and getattr(self.model.draft_provider, "batched_draft", False) + ) + + @property + def supports_sampled_draft_processing(self) -> bool: + return bool( + self.model.draft_provider is not None + and getattr(self.model.draft_provider, "sampled_batch_draft", False) + ) + + def maybe_fill_draft_tokens(self, completion: Completion) -> None: + if ( + self.model.draft_provider is None + or completion.finished + or completion.pending_finish_reason is not None + or completion.draft_tokens + ): + return + remaining_tokens = completion.max_total_tokens - completion.total_tokens + if not getattr(self.model, "draft_target_batching", True): + remaining_tokens = min(remaining_tokens, 1) + if remaining_tokens <= 0: + return + input_ids = self.draft_input_ids(completion) + if isinstance(self.model.draft_provider, MTPDraftProvider) and completion.multimodal_prompt: + return + can_draft = getattr(self.model.draft_provider, "can_draft", None) + if can_draft is not None and not can_draft( + int(input_ids.shape[0]), + seq_id=completion.seq_id, + ): + return + draft_started_at = time.perf_counter() + try: + proposed = self.model.draft_provider.draft( + input_ids, + seq_id=completion.seq_id, + max_tokens=remaining_tokens, + ) + finally: + draft_elapsed = time.perf_counter() - draft_started_at + self.metrics.draft_seconds_total += draft_elapsed + self.metrics.draft_generate_seconds_total += draft_elapsed + self.metrics.draft_generate_calls_total += 1 + if proposed.size == 0: + return + limited = [int(token) for token in proposed[:remaining_tokens]] + if not limited: + return + completion.draft_tokens = limited + self.speculative_stats["draft_proposals"] += 1 + self.speculative_stats["draft_tokens_proposed"] += len(limited) + + def maybe_fill_batched_draft_tokens(self) -> bool: + if not self.should_defer_draft_fill or self.model.draft_provider is None: + return False + draft_many = getattr(self.model.draft_provider, "draft_many", None) + if draft_many is None: + return False + + completions: List[Completion] = [] + draft_requests: List[Tuple[np.ndarray, int, Optional[int]]] = [] + remaining_by_completion: List[int] = [] + for request_id in self.active_request_ids: + request = self.requests.get(request_id) + if request is None or not request.admitted: + continue + for completion in request.completions: + if ( + completion.finished + or completion.pending_finish_reason is not None + or completion.draft_tokens + ): + continue + remaining_tokens = completion.max_total_tokens - completion.total_tokens + if not getattr(self.model, "draft_target_batching", True): + remaining_tokens = min(remaining_tokens, 1) + if remaining_tokens <= 0: + continue + input_ids = self.draft_input_ids(completion) + if ( + isinstance(self.model.draft_provider, MTPDraftProvider) + and completion.multimodal_prompt + ): + continue + can_draft = getattr(self.model.draft_provider, "can_draft", None) + if can_draft is not None and not can_draft( + int(input_ids.shape[0]), + seq_id=completion.seq_id, + ): + continue + completions.append(completion) + draft_requests.append((input_ids, completion.seq_id, remaining_tokens)) + remaining_by_completion.append(remaining_tokens) + + if not draft_requests: + return False + if not self.active_draft_batch_size_allowed(len(draft_requests)): + return False + + draft_started_at = time.perf_counter() + proposed_many = draft_many(draft_requests) + draft_elapsed = time.perf_counter() - draft_started_at + self.metrics.draft_seconds_total += draft_elapsed + self.metrics.draft_generate_seconds_total += draft_elapsed + self.metrics.draft_generate_calls_total += 1 + made_progress = False + + for completion, proposed, remaining_tokens in zip( + completions, + proposed_many, + remaining_by_completion, + ): + if proposed.size == 0: + continue + limited = [int(token) for token in proposed[:remaining_tokens]] + if not limited: + continue + self.speculative_stats["draft_proposals"] += 1 + self.speculative_stats["draft_tokens_proposed"] += len(limited) + completion.draft_tokens = limited + made_progress = True + return made_progress + + def draft_input_ids(self, completion: Completion) -> np.ndarray: + return np.array( + [*completion.prompt_tokens, *completion.completion_tokens], + dtype=np.intc, + ) + + def process_sampled_draft_batch( + self, + items: Sequence["CompletionScheduler.BatchItem"], + ) -> None: + if self.model.draft_provider is None: + return + process_sampled_batch = getattr( + self.model.draft_provider, + "process_sampled_batch", + None, + ) + if process_sampled_batch is None: + return + if not self.draft_batch_size_allowed(items): + return + + output_count = sum( + output_index is not None + for item in items + for output_index in item.output_indices + ) + completions: List[Completion] = [] + update_items: List["CompletionScheduler.BatchItem"] = [] + updates: List["MTPDraftProvider.SampledBatchUpdate"] = [] + remaining_by_completion: List[int] = [] + batch_row_offset = 0 + for item in items: + item_batch_rows = list( + range(batch_row_offset, batch_row_offset + len(item.tokens)) + ) + batch_row_offset += len(item.tokens) + decode = item.decode_state + if item.kind != "decode" or decode is None: + continue + request = self.requests.get(item.request_id) + if request is None: + continue + completion = request.completions[decode.completion_index] + if completion.finished or completion.pending_finish_reason is not None: + continue + remaining_tokens = completion.max_total_tokens - completion.total_tokens + if remaining_tokens <= 0: + continue + resolved_output_indices = [ + self.output_index_to_logits_index(output_index, output_count) + for output_index in item.output_indices + ] + if any(output_index is None for output_index in resolved_output_indices): + continue + sample_index = decode.pending_count + decode.accepted_draft_count - 1 + target_count = len(item.tokens) + if sample_index < 0 or sample_index >= target_count: + continue + if decode.sampled_pending_token is None: + continue + completions.append(completion) + update_items.append(item) + updates.append( + MTPDraftProvider.SampledBatchUpdate( + seq_id=item.seq_id, + start_pos=item.llama_start_pos, + tokens=item.tokens, + row_indices=item_batch_rows, + target_count=target_count, + sample_index=sample_index, + pending_token=decode.sampled_pending_token, + max_tokens=remaining_tokens, + ) + ) + remaining_by_completion.append(remaining_tokens) + + if not updates: + return + + draft_started_at = time.perf_counter() + proposed_many = process_sampled_batch(updates) + for item in update_items: + item.require_decode().rollback_draft_processed = True + draft_elapsed = time.perf_counter() - draft_started_at + self.metrics.draft_seconds_total += draft_elapsed + self.metrics.draft_sampled_batch_seconds_total += draft_elapsed + self.metrics.draft_sampled_batch_calls_total += 1 + for completion, proposed, remaining_tokens in zip( + completions, + proposed_many, + remaining_by_completion, + ): + if proposed.size == 0: + continue + limited = [int(token) for token in proposed[:remaining_tokens]] + if not limited: + continue + completion.draft_tokens = limited + self.speculative_stats["draft_proposals"] += 1 + self.speculative_stats["draft_tokens_proposed"] += len(limited) + + def rollback_draft_verification( + self, + item: "CompletionScheduler.BatchItem", + keep_len: int, + accepted_draft_count: int, + *, + defer_draft_state: bool, + ) -> None: + decode = item.require_decode() + if defer_draft_state: + decode.rollback_keep_len = keep_len + decode.rollback_accepted_draft_count = accepted_draft_count + decode.rollback_draft_processed = False + return + self.model.accept_draft_tokens(item.seq_id, accepted_draft_count) + self.truncate_sequence(item.seq_id, keep_len) + + def apply_deferred_draft_rollbacks( + self, + items: Sequence["CompletionScheduler.BatchItem"], + ) -> None: + for item in items: + decode = item.require_decode() + rollback_keep_len = decode.rollback_keep_len + if rollback_keep_len is None: + deferred_accept_draft_count = decode.deferred_accept_draft_count + if ( + not decode.rollback_draft_processed + and deferred_accept_draft_count is not None + ): + self.model.accept_draft_tokens( + item.seq_id, + deferred_accept_draft_count, + ) + deferred_truncate_draft_len = decode.deferred_truncate_draft_len + if deferred_truncate_draft_len is not None: + self.model.truncate_draft_sequence( + item.seq_id, + deferred_truncate_draft_len, + ) + decode.deferred_accept_draft_count = None + decode.deferred_truncate_draft_len = None + decode.rollback_draft_processed = False + continue + + draft_state_needs_fallback = not decode.rollback_draft_processed + if draft_state_needs_fallback: + self.model.accept_draft_tokens( + item.seq_id, + decode.rollback_accepted_draft_count, + ) + self.truncate_sequence( + item.seq_id, + rollback_keep_len, + truncate_draft=draft_state_needs_fallback, + ) + decode.rollback_keep_len = None + decode.rollback_accepted_draft_count = 0 + decode.rollback_draft_processed = False + decode.deferred_accept_draft_count = None + decode.deferred_truncate_draft_len = None + + def sample_completion_token(self, completion: Completion, output_index: int) -> int: + sample_started_at = time.perf_counter() + try: + return completion.sampler.sample(self.model.ctx, output_index) + finally: + self.metrics.sample_seconds_total += time.perf_counter() - sample_started_at + self.metrics.sample_calls_total += 1 + + def process_generation_item( + self, + completion: Completion, + item: CompletionScheduler.BatchItem, + output_count: int, + ) -> None: + if completion.finished: + return + decode = item.require_decode() + pending_count = decode.pending_count + resolved_output_indices = [ + self.output_index_to_logits_index(output_index, output_count) + for output_index in item.output_indices + ] + if any(output_index is None for output_index in resolved_output_indices): + raise RuntimeError("generation outputs are required") + logits_indices: List[int] = [] + for output_index in resolved_output_indices: + assert output_index is not None + logits_indices.append(int(output_index)) + if completion.pending_finish_reason is not None: + if self.model.store_logits: + self.checkpoint_logits[completion.seq_id] = self.model.logits( + logits_indices[-1] + ) + completion.pending_input_tokens = completion.pending_input_tokens[pending_count:] + finish_reason: str = completion.pending_finish_reason + completion.pending_finish_reason = None + self.finish_completion(completion, finish_reason) + return + + if pending_count: + completion.pending_input_tokens = completion.pending_input_tokens[pending_count:] + + decoded_draft_tokens = item.tokens[pending_count:] + accepted_draft_count = 0 + defer_draft_state = self.defer_sampled_draft_processing + if decoded_draft_tokens and pending_count <= 0: + raise RuntimeError("draft verification requires at least one pending token") + if decoded_draft_tokens: + self.metrics.draft_batches_verified_total += 1 + self.metrics.draft_target_tokens_verified_total += len(decoded_draft_tokens) + + for draft_index, draft_token in enumerate(decoded_draft_tokens): + verify_output_index = pending_count + draft_index - 1 + if verify_output_index >= len(logits_indices): + raise RuntimeError("missing target output for draft verification") + logits_index = logits_indices[verify_output_index] + sampled_token = self.sample_completion_token(completion, logits_index) + need_logits = self.model.store_logits or completion.needs_token_logprob + logits = self.model.logits(logits_index) if need_logits else None + if self.model.store_logits and logits is not None: + self.checkpoint_logits[completion.seq_id] = logits + if sampled_token != draft_token: + self.record_draft_acceptance_length(accepted_draft_count) + rejected = max(0, len(completion.draft_tokens) - accepted_draft_count) + if rejected > 0: + self.speculative_stats["draft_tokens_rejected"] += rejected + self.metrics.draft_target_tokens_wasted_total += ( + max(0, len(decoded_draft_tokens) - accepted_draft_count - 1) + ) + keep_len = item.start_pos + pending_count + accepted_draft_count + self.rollback_draft_verification( + item, + keep_len, + accepted_draft_count, + defer_draft_state=defer_draft_state, + ) + completion.draft_tokens.clear() + prev_tokens = [*completion.prompt_tokens, *completion.completion_tokens] + if logits is not None: + record = Token.from_logits( + model=self.model, + formatter=self.formatter, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=sampled_token, + logits=logits, + logprobs_count=completion.logprobs, + need_token_logprob=completion.needs_token_logprob, + ) + else: + record = Token.from_token( + model=self.model, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=sampled_token, + ) + mismatch_finish_reason: Optional[str] = self.handle_completion_token( + completion, + sampled_token, + record, + decoded=False, + ) + if mismatch_finish_reason is not None: + completion.pending_finish_reason = mismatch_finish_reason + else: + decode.sampled_pending_token = sampled_token + decode.accepted_draft_count = accepted_draft_count + return + prev_tokens = [*completion.prompt_tokens, *completion.completion_tokens] + if logits is not None: + record = Token.from_logits( + model=self.model, + formatter=self.formatter, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=draft_token, + logits=logits, + logprobs_count=completion.logprobs, + need_token_logprob=completion.needs_token_logprob, + ) + else: + record = Token.from_token( + model=self.model, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=draft_token, + ) + accepted_finish_reason: Optional[str] = self.handle_completion_token( + completion, + draft_token, + record, + decoded=True, + ) + accepted_draft_count += 1 + self.speculative_stats["draft_tokens_accepted"] += 1 + if accepted_finish_reason is not None: + self.record_draft_acceptance_length(accepted_draft_count) + rejected = max(0, len(completion.draft_tokens) - accepted_draft_count) + if rejected > 0: + self.speculative_stats["draft_tokens_rejected"] += rejected + self.metrics.draft_target_tokens_wasted_total += ( + len(decoded_draft_tokens) - accepted_draft_count + ) + keep_len = item.start_pos + pending_count + accepted_draft_count + self.rollback_draft_verification( + item, + keep_len, + accepted_draft_count, + defer_draft_state=defer_draft_state, + ) + completion.draft_tokens.clear() + decode.accepted_draft_count = accepted_draft_count + self.finish_completion(completion, accepted_finish_reason) + return + + decode.accepted_draft_count = accepted_draft_count + if decoded_draft_tokens: + self.record_draft_acceptance_length(accepted_draft_count) + if not defer_draft_state: + self.model.accept_draft_tokens(completion.seq_id, accepted_draft_count) + elif decoded_draft_tokens: + decode.deferred_accept_draft_count = accepted_draft_count + if accepted_draft_count: + completion.draft_tokens = completion.draft_tokens[accepted_draft_count:] + + final_logits_index = logits_indices[-1] + next_token = self.sample_completion_token(completion, final_logits_index) + need_final_logits = self.model.store_logits or completion.needs_token_logprob + final_logits = ( + self.model.logits(final_logits_index) if need_final_logits else None + ) + if self.model.store_logits and final_logits is not None: + self.checkpoint_logits[completion.seq_id] = final_logits + if completion.draft_tokens and next_token != completion.draft_tokens[0]: + self.speculative_stats["draft_tokens_rejected"] += len(completion.draft_tokens) + if not defer_draft_state: + self.model.truncate_draft_sequence( + completion.seq_id, + item.start_pos + len(item.tokens), + ) + else: + decode.deferred_truncate_draft_len = item.start_pos + len(item.tokens) + completion.draft_tokens.clear() + elif completion.draft_tokens and next_token == completion.draft_tokens[0]: + if getattr(self.model, "draft_target_batching", True): + completion.draft_tokens = completion.draft_tokens[1:] + else: + completion.draft_tokens.clear() + self.speculative_stats["draft_tokens_accepted"] += 1 + self.metrics.draft_tokens_reused_as_pending_total += 1 + + prev_tokens = [*completion.prompt_tokens, *completion.completion_tokens] + if final_logits is not None: + record = Token.from_logits( + model=self.model, + formatter=self.formatter, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=next_token, + logits=final_logits, + logprobs_count=completion.logprobs, + need_token_logprob=completion.needs_token_logprob, + ) + else: + record = Token.from_token( + model=self.model, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=next_token, + ) + final_finish_reason: Optional[str] = self.handle_completion_token( + completion, + next_token, + record, + decoded=False, + ) + if final_finish_reason is not None: + completion.pending_finish_reason = final_finish_reason + else: + decode.sampled_pending_token = next_token + + def maybe_save_prompt_checkpoint(self, request: CompletionRequest) -> None: + if ( + not self.model.exact_checkpoints_only + or request.prompt_checkpoint_saved + or request.base_seq_id is None + or not request.prompt_tokens + or request.prompt_logits is None + or not self.unused_sequences + ): + return + checkpoint_seq_id = self.unused_sequences.pop() + self.copy_sequence_state( + request.base_seq_id, + checkpoint_seq_id, + len(request.prompt_tokens), + ) + self.checkpoint_logits[checkpoint_seq_id] = request.prompt_logits + self.free_sequences[checkpoint_seq_id] = None + self.free_sequences.move_to_end(checkpoint_seq_id) + request.prompt_checkpoint_saved = True + self.metrics.checkpoint_saves_total += 1 + + @staticmethod + def output_index_to_logits_index( + output_index: Optional[int], + output_count: int, + ) -> Optional[int]: + if output_index is None: + return None + return output_index - output_count + + @staticmethod + def last_output_index(output_indices: Sequence[Optional[int]]) -> Optional[int]: + for output_index in reversed(output_indices): + if output_index is not None: + return output_index + return None + + def start_completions( + self, + request: CompletionRequest, + prompt_output_index: Optional[int], + prompt_logits: Optional[np.ndarray] = None, + ) -> None: + if request.completions: + return + assert request.base_seq_id is not None + prompt_tokens = list(request.prompt_plan.text_tokens) + prompt_length = len(request.prompt_tokens) + multimodal_prompt = any( + segment.kind != "text" for segment in request.prompt_plan.segments + ) + prompt_text = request.prompt_text + if request.payload.stop is None: + stop_sequences: List[bytes] = [] + elif isinstance(request.payload.stop, str): + stop_sequences = [request.payload.stop.encode("utf-8")] + else: + stop_sequences = [item.encode("utf-8") for item in request.payload.stop] + logit_bias = ( + {int(token): float(bias) for token, bias in request.payload.logit_bias.items()} + if request.payload.logit_bias + else None + ) + prompt_text_bytes = self.model.detokenize(prompt_tokens) if prompt_tokens else b"" + for offset, seq_id in enumerate(request.completion_seq_ids): + if offset > 0: + if prompt_length: + self.memory_policy.copy_prompt_state( + request.base_seq_id, + seq_id, + prompt_length, + ) + sampler = Sampler( + seed=(request.payload.seed or llama_cpp.LLAMA_DEFAULT_SEED) + offset, + vocab=self.model.vocab, + n_vocab=self.model.n_vocab, + top_p=request.payload.top_p, + temperature=request.payload.temperature, + frequency_penalty=request.payload.frequency_penalty or 0.0, + presence_penalty=request.payload.presence_penalty or 0.0, + logit_bias=logit_bias, + grammar_text=request.grammar_text, + grammar_root=request.grammar_root, + ) + request.completions.append( + Completion( + request_id=request.id, + index=offset, + seq_id=seq_id, + sampler=sampler, + prompt_tokens=prompt_tokens, + prompt_length=prompt_length, + prompt_text=prompt_text, + multimodal_prompt=multimodal_prompt, + max_total_tokens=request.effective_max_len, + stop_sequences=stop_sequences, + logprobs=request.payload.logprobs, + detokenized_prefix_bytes=bytearray(prompt_text_bytes), + rank_by_score=( + request.payload.best_of is not None + and request.payload.best_of > request.payload.n + ), + ) + ) + if request.payload.max_tokens == 0 or request.effective_max_len == prompt_length: + for completion in request.completions: + self.finish_completion(completion, "length") + self.finalize_request_if_ready(request) + return + if prompt_output_index is None: + if prompt_logits is not None: + for completion in request.completions: + self.sample_completion_from_logits(completion, prompt_logits) + return + raise RuntimeError("prompt output is required to start generation") + for completion in request.completions: + self.sample_completion(completion, prompt_output_index) + + def sample_completion( + self, + completion: Completion, + output_index: Optional[int], + ) -> None: + if completion.finished: + return + if output_index is None: + raise RuntimeError("missing logits output") + token = self.sample_completion_token(completion, output_index) + prev_tokens = [*completion.prompt_tokens, *completion.completion_tokens] + if self.model.store_logits or completion.needs_token_logprob: + logits = self.model.logits(output_index) + if self.model.store_logits: + self.checkpoint_logits[completion.seq_id] = logits + record = Token.from_logits( + model=self.model, + formatter=self.formatter, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=token, + logits=logits, + logprobs_count=completion.logprobs, + need_token_logprob=completion.needs_token_logprob, + ) + else: + record = Token.from_token( + model=self.model, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=token, + ) + finish_reason = self.handle_completion_token( + completion, + token, + record, + decoded=False, + ) + if finish_reason is not None: + completion.pending_finish_reason = finish_reason + + def sample_completion_from_logits( + self, + completion: Completion, + logits: np.ndarray, + ) -> None: + if completion.finished: + return + token = completion.sampler.sample_logits(logits) + prev_tokens = [*completion.prompt_tokens, *completion.completion_tokens] + if self.model.store_logits or completion.needs_token_logprob: + if self.model.store_logits: + self.checkpoint_logits[completion.seq_id] = logits.copy() + record = Token.from_logits( + model=self.model, + formatter=self.formatter, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=token, + logits=logits, + logprobs_count=completion.logprobs, + need_token_logprob=completion.needs_token_logprob, + ) + else: + record = Token.from_token( + model=self.model, + prev_tokens=prev_tokens, + prev_text_bytes=completion.detokenized_prefix_bytes, + token=token, + ) + finish_reason = self.handle_completion_token( + completion, + token, + record, + decoded=False, + ) + if finish_reason is not None: + completion.pending_finish_reason = finish_reason + + def handle_completion_token( + self, + completion: Completion, + token: int, + record: Token, + *, + decoded: bool, + ) -> Optional[str]: + if record.token_logprob is not None: + completion.score_sum += record.token_logprob + rendered_start = len(completion.rendered_bytes) + completion.completion_tokens.append(token) + self.metrics.observe_predicted_token() + completion.token_records.append(record) + completion.rendered_bytes.extend(record.text_bytes) + completion.detokenized_prefix_bytes.extend(record.text_bytes) + finish_reason: Optional[str] = None + if llama_cpp.llama_vocab_is_eog(self.model.vocab, token): + finish_reason = "stop" + elif completion.total_tokens >= completion.max_total_tokens: + finish_reason = "length" + else: + max_stop_length = completion.max_stop_sequence_length + search_start = max(0, rendered_start - max_stop_length + 1) + if any( + completion.rendered_bytes.find(stop, search_start) != -1 + for stop in completion.stop_sequences + ): + finish_reason = "stop" + if not decoded: + completion.pending_input_tokens.append(token) + if ( + completion.request_id in self.requests + and self.requests[completion.request_id].payload.stream + and finish_reason is None + ): + self.flush_stream_updates(completion, finish_reason=None) + if not decoded and finish_reason is None and not self.should_defer_draft_fill: + self.maybe_fill_draft_tokens(completion) + return finish_reason + + def flush_stream_updates( + self, + completion: Completion, + finish_reason: Optional[str], + ) -> None: + request = self.requests[completion.request_id] + for payload in self.formatter.stream_completion_chunks( + request, + completion, + finish_reason, + ): + if request.on_stream_chunk is not None: + request.on_stream_chunk(payload) + + def finish_completion(self, completion: Completion, finish_reason: str) -> None: + if completion.finished: + return + completion.finished = True + completion.finish_reason = finish_reason + completion.pending_input_tokens.clear() + completion.draft_tokens.clear() + request = self.requests[completion.request_id] + if request.payload.stream: + self.flush_stream_updates(completion, finish_reason=finish_reason) + if request.on_stream_chunk is not None: + request.on_stream_chunk( + self.formatter.completion_finish_chunk( + request, + completion, + finish_reason, + ) + ) + + def finalize_request_if_ready(self, request: CompletionRequest) -> None: + if not request.completions or not all(completion.finished for completion in request.completions): + return + selected = request.selected_completions() + result = self.formatter.build_completion_response(request, selected) + self.metrics.requests_completed_total += 1 + self.release_request(request) + if request.on_done is not None: + request.on_done(result) + + def truncate_sequence( + self, + seq_id: int, + keep_len: int, + *, + truncate_draft: bool = True, + ) -> None: + current_len = self.radix_trie.length(seq_id) + if current_len <= keep_len: + return + keep_pos = self.sequence_history.position_length_for_prefix(seq_id, keep_len) + if not llama_cpp.llama_memory_seq_rm(self.model.mem, seq_id, keep_pos, -1): + raise RuntimeError( + f"failed to truncate model sequence {seq_id} at position {keep_pos}" + ) + if truncate_draft: + self.model.truncate_draft_sequence(seq_id, keep_pos) + self.truncate_sequence_metadata(seq_id, current_len, keep_len) + + def copy_sequence_state( + self, + source_sequence_id: int, + dest_sequence_id: int, + keep_len: int, + *, + copy_all_state: bool = False, + ) -> None: + if keep_len <= 0: + return + source_length = self.radix_trie.length(source_sequence_id) + keep_pos = self.sequence_history.position_length_for_prefix( + source_sequence_id, + keep_len, + ) + copy_p0 = 0 + copy_p1 = keep_pos + if copy_all_state or not self.model.kv_unified: + copy_p0 = -1 + copy_p1 = -1 + llama_cpp.llama_memory_seq_cp( + self.model.mem, + source_sequence_id, + dest_sequence_id, + copy_p0, + copy_p1, + ) + self.model.copy_draft_sequence( + source_sequence_id, + dest_sequence_id, + copy_p0, + copy_p1, + ) + if copy_all_state and source_length > keep_len: + if not llama_cpp.llama_memory_seq_rm( + self.model.mem, + dest_sequence_id, + keep_pos, + -1, + ): + raise RuntimeError( + f"failed to truncate copied model sequence {dest_sequence_id} " + f"at position {keep_pos}" + ) + self.model.truncate_draft_sequence(dest_sequence_id, keep_pos) + self.radix_trie.copy(source_sequence_id, dest_sequence_id, keep_len) + self.sequence_history.copy( + source_sequence_id, + dest_sequence_id, + source_length, + keep_len, + ) + + def truncate_sequence_metadata( + self, + seq_id: int, + current_len: int, + keep_len: int, + ) -> None: + self.radix_trie.truncate(seq_id, keep_len) + self.sequence_history.truncate(seq_id, current_len, keep_len) + self.checkpoint_logits.pop(seq_id, None) + + def truncate_free_sequence(self, seq_id: int, keep_len: int) -> None: + if seq_id not in self.free_sequences: + return + self.truncate_sequence(seq_id, keep_len) + self.free_sequences.move_to_end(seq_id) + + def delete_free_sequence(self, seq_id: int) -> None: + if seq_id not in self.free_sequences: + return + self.truncate_sequence(seq_id, 0) + del self.free_sequences[seq_id] + self.checkpoint_logits.pop(seq_id, None) + self.unused_sequences.append(seq_id) + self.metrics.checkpoint_evictions_total += 1 + + def claim_unused_sequence(self) -> int: + seq_id = self.unused_sequences.pop() + self.claimed_sequences.add(seq_id) + return seq_id + + def claim_free_sequence(self, seq_id: int) -> int: + del self.free_sequences[seq_id] + self.claimed_sequences.add(seq_id) + return seq_id + + def activate_request( + self, + request: CompletionRequest, + *, + base_seq_id: int, + sibling_seq_ids: List[int], + ) -> None: + request.base_seq_id = base_seq_id + request.sibling_seq_ids = sibling_seq_ids + request.completion_seq_ids = [base_seq_id, *sibling_seq_ids] + request.admitted = True + self.active_request_ids.add(request.id) + + def release_request(self, request: CompletionRequest) -> None: + for completion in request.completions: + completion.sampler.close() + for seq_id in request.completion_seq_ids: + if seq_id in self.claimed_sequences: + self.claimed_sequences.remove(seq_id) + self.free_sequences[seq_id] = None + self.free_sequences.move_to_end(seq_id) + self.active_request_ids.discard(request.id) + self.requests.pop(request.id, None) + + def remove_pending_request(self, request: CompletionRequest) -> None: + try: + self.pending_requests.remove(request) + except ValueError: + pass + + def fail_request(self, request: CompletionRequest, exc: BaseException) -> None: + self.remove_pending_request(request) + if request.id in self.active_request_ids or request.admitted: + self.release_request(request) + else: + self.requests.pop(request.id, None) + if isinstance(exc, CompletionRequestCancelledError): + self.metrics.requests_cancelled_total += 1 + else: + self.metrics.requests_failed_total += 1 + if request.on_error is not None: + request.on_error(exc) + + @staticmethod + def _format_prometheus_value(value: Union[int, float]) -> str: + if isinstance(value, int): + return str(value) + if not math.isfinite(value): + raise ValueError(f"invalid Prometheus metric value: {value}") + return format(value, ".16g") + + def sequence_cache_metric_definitions( + self, + ) -> List[Tuple[str, str, str, Union[int, float]]]: + return [ + ( + "counter", + "batch_server:sequence_cache_hits_total", + "Number of external sequence cache entries hydrated.", + self.metrics.sequence_cache_hits_total, + ), + ( + "counter", + "batch_server:sequence_cache_save_requests_total", + "Number of external sequence cache save requests.", + self.metrics.sequence_cache_save_requests_total, + ), + ( + "counter", + "batch_server:sequence_cache_load_failures_total", + "Number of external sequence cache load failures.", + self.metrics.sequence_cache_load_failures_total, + ), + ( + "counter", + "batch_server:sequence_cache_lookup_failures_total", + "Number of external sequence cache lookup failures.", + self.metrics.sequence_cache_lookup_failures_total, + ), + ( + "counter", + "batch_server:sequence_cache_save_failures_total", + "Number of external sequence cache save failures.", + self.metrics.sequence_cache_save_failures_total, + ), + ( + "counter", + "batch_server:sequence_cache_tokens_loaded_total", + "Number of prompt tokens hydrated from external sequence cache.", + self.metrics.sequence_cache_tokens_loaded_total, + ), + ] + + def speculative_metric_definitions( + self, + ) -> List[Tuple[str, str, str, Union[int, float]]]: + proposed = self.speculative_stats["draft_tokens_proposed"] + accepted = self.speculative_stats["draft_tokens_accepted"] + proposals = self.speculative_stats["draft_proposals"] + verified_proposals = sum(self.draft_acceptance_length_counts.values()) + verified_accepted = sum( + accepted_tokens * count + for accepted_tokens, count in self.draft_acceptance_length_counts.items() + ) + acceptance_rate = accepted / proposed if proposed > 0 else 0.0 + average_proposed_tokens = proposed / proposals if proposals > 0 else 0.0 + average_verified_acceptance_length = ( + verified_accepted / verified_proposals if verified_proposals > 0 else 0.0 + ) + return [ + ( + "counter", + "batch_server:draft_proposals_total", + "Number of speculative draft proposal batches generated.", + proposals, + ), + ( + "counter", + "batch_server:draft_tokens_proposed_total", + "Number of speculative draft tokens proposed.", + proposed, + ), + ( + "counter", + "batch_server:draft_tokens_accepted_total", + "Number of speculative draft tokens accepted.", + accepted, + ), + ( + "counter", + "batch_server:draft_tokens_rejected_total", + "Number of speculative draft tokens rejected.", + self.speculative_stats["draft_tokens_rejected"], + ), + ( + "counter", + "batch_server:draft_batches_verified_total", + "Number of completion items that target-verified draft tokens.", + self.metrics.draft_batches_verified_total, + ), + ( + "counter", + "batch_server:draft_target_tokens_verified_total", + "Number of draft tokens decoded by the target for verification.", + self.metrics.draft_target_tokens_verified_total, + ), + ( + "counter", + "batch_server:draft_target_tokens_wasted_total", + "Number of target-verified draft tokens rejected before emission.", + self.metrics.draft_target_tokens_wasted_total, + ), + ( + "counter", + "batch_server:draft_tokens_reused_as_pending_total", + "Number of draft tokens accepted by the next pending-token sample.", + self.metrics.draft_tokens_reused_as_pending_total, + ), + ( + "gauge", + "batch_server:draft_acceptance_rate", + "Fraction of proposed draft tokens accepted, including reused pending matches.", + acceptance_rate, + ), + ( + "gauge", + "batch_server:draft_average_proposed_tokens", + "Average number of tokens in generated draft proposals.", + average_proposed_tokens, + ), + ( + "gauge", + "batch_server:draft_average_verified_acceptance_length", + "Average accepted draft-token prefix length for target-verified proposals only.", + average_verified_acceptance_length, + ), + ] + + def record_draft_acceptance_length(self, accepted_tokens: int) -> None: + accepted_tokens = max(0, int(accepted_tokens)) + self.draft_acceptance_length_counts[accepted_tokens] = ( + self.draft_acceptance_length_counts.get(accepted_tokens, 0) + 1 + ) + + def draft_acceptance_length_metric_lines(self) -> List[str]: + lines = [ + "# HELP batch_server:draft_acceptance_length_total Number of verified draft proposals by accepted-token prefix length.", + "# TYPE batch_server:draft_acceptance_length_total counter", + ] + for accepted_tokens, count in sorted(self.draft_acceptance_length_counts.items()): + lines.append( + f'batch_server:draft_acceptance_length_total{{accepted_tokens="{accepted_tokens}"}} {count}' + ) + return lines + + def render_prometheus_metrics(self) -> str: + active_completions = sum( + 1 + for request_id in self.active_request_ids + for completion in self.requests[request_id].completions + if not completion.finished + ) + checkpoint_entries = sum( + 1 for seq_id in self.free_sequences if seq_id in self.checkpoint_logits + ) + prompt_tokens_seconds = ( + self.metrics.prompt_tokens_total / self.metrics.prompt_seconds_total + if self.metrics.prompt_seconds_total > 0 + else 0.0 + ) + predicted_tokens_seconds = ( + self.metrics.tokens_predicted_total + / self.metrics.tokens_predicted_seconds_total + if self.metrics.tokens_predicted_seconds_total > 0 + else 0.0 + ) + n_busy_slots_per_decode = ( + self.metrics.n_busy_slots_total / self.metrics.n_decode_total + if self.metrics.n_decode_total > 0 + else 0.0 + ) + draft_provider_metrics: List[Tuple[str, str, str, Union[int, float]]] = [] + if self.model.draft_provider is not None: + draft_metric_definitions = getattr( + self.model.draft_provider, + "metric_definitions", + None, + ) + if draft_metric_definitions is not None: + draft_provider_metrics = cast( + List[Tuple[str, str, str, Union[int, float]]], + draft_metric_definitions(), + ) + metrics_def: List[Tuple[str, str, str, Union[int, float]]] = [ + ( + "counter", + "llamacpp:prompt_tokens_total", + "Number of prompt tokens processed.", + self.metrics.prompt_tokens_total, + ), + ( + "counter", + "llamacpp:prompt_seconds_total", + "Estimated prompt processing time in seconds.", + self.metrics.prompt_seconds_total, + ), + ( + "counter", + "llamacpp:tokens_predicted_total", + "Number of generated tokens processed.", + self.metrics.tokens_predicted_total, + ), + ( + "counter", + "llamacpp:tokens_predicted_seconds_total", + "Estimated generation processing time in seconds.", + self.metrics.tokens_predicted_seconds_total, + ), + ( + "counter", + "batch_server:scheduler_step_seconds_total", + "Total scheduler step wall time.", + self.metrics.scheduler_step_seconds_total, + ), + ( + "counter", + "batch_server:process_batch_seconds_total", + "Time spent processing decoded batches after llama_decode().", + self.metrics.process_batch_seconds_total, + ), + ( + "counter", + "batch_server:sample_seconds_total", + "Time spent sampling target logits.", + self.metrics.sample_seconds_total, + ), + ( + "counter", + "batch_server:draft_seconds_total", + "Speculative draft processing time in seconds.", + self.metrics.draft_seconds_total, + ), + ( + "counter", + "batch_server:draft_process_seconds_total", + "Time spent feeding target batches into the draft context.", + self.metrics.draft_process_seconds_total, + ), + ( + "counter", + "batch_server:draft_generate_seconds_total", + "Time spent generating speculative draft tokens.", + self.metrics.draft_generate_seconds_total, + ), + ( + "counter", + "batch_server:draft_sampled_batch_seconds_total", + "Time spent generating speculative tokens from sampled target batches.", + self.metrics.draft_sampled_batch_seconds_total, + ), + ( + "counter", + "batch_server:draft_process_calls_total", + "Number of target-to-draft context processing phases.", + self.metrics.draft_process_calls_total, + ), + ( + "counter", + "batch_server:draft_generate_calls_total", + "Number of speculative draft generation phases.", + self.metrics.draft_generate_calls_total, + ), + ( + "counter", + "batch_server:draft_sampled_batch_calls_total", + "Number of sampled target batch draft phases.", + self.metrics.draft_sampled_batch_calls_total, + ), + ( + "counter", + "llamacpp:n_decode_total", + "Total number of llama_decode() calls.", + self.metrics.n_decode_total, + ), + ( + "counter", + "batch_server:scheduler_step_calls_total", + "Number of scheduler steps.", + self.metrics.scheduler_step_calls_total, + ), + ( + "counter", + "batch_server:process_batch_calls_total", + "Number of decoded batch processing phases.", + self.metrics.process_batch_calls_total, + ), + ( + "counter", + "batch_server:sample_calls_total", + "Number of target logit sampling calls.", + self.metrics.sample_calls_total, + ), + ( + "counter", + "llamacpp:n_tokens_max", + "Largest observed n_tokens.", + self.metrics.n_tokens_max, + ), + ( + "gauge", + "llamacpp:n_busy_slots_per_decode", + "Average number of busy sequences per llama_decode() call.", + n_busy_slots_per_decode, + ), + ( + "gauge", + "llamacpp:prompt_tokens_seconds", + "Average prompt throughput in tokens/s.", + prompt_tokens_seconds, + ), + ( + "gauge", + "llamacpp:predicted_tokens_seconds", + "Average generation throughput in tokens/s.", + predicted_tokens_seconds, + ), + ( + "gauge", + "llamacpp:requests_processing", + "Number of requests processing.", + len(self.active_request_ids), + ), + ( + "gauge", + "llamacpp:requests_deferred", + "Number of requests deferred.", + len(self.pending_requests), + ), + ( + "counter", + "batch_server:requests_submitted_total", + "Number of requests submitted.", + self.metrics.requests_submitted_total, + ), + ( + "counter", + "batch_server:requests_admitted_total", + "Number of requests admitted.", + self.metrics.requests_admitted_total, + ), + ( + "counter", + "batch_server:requests_completed_total", + "Number of requests completed successfully.", + self.metrics.requests_completed_total, + ), + ( + "counter", + "batch_server:requests_cancelled_total", + "Number of requests cancelled.", + self.metrics.requests_cancelled_total, + ), + ( + "counter", + "batch_server:requests_failed_total", + "Number of requests failed.", + self.metrics.requests_failed_total, + ), + ( + "gauge", + "batch_server:active_completions", + "Number of active unfinished completions.", + active_completions, + ), + ( + "gauge", + "batch_server:claimed_sequences", + "Number of claimed sequence ids.", + len(self.claimed_sequences), + ), + ( + "gauge", + "batch_server:free_sequences", + "Number of reusable free sequence ids.", + len(self.free_sequences), + ), + ( + "gauge", + "batch_server:unused_sequences", + "Number of unused sequence ids.", + len(self.unused_sequences), + ), + ( + "gauge", + "batch_server:checkpoint_entries", + "Number of free sequence checkpoints with stored logits.", + checkpoint_entries, + ), + ( + "counter", + "batch_server:checkpoint_hits_total", + "Number of exact checkpoint hits.", + self.metrics.checkpoint_hits_total, + ), + ( + "counter", + "batch_server:checkpoint_saves_total", + "Number of prompt checkpoints saved.", + self.metrics.checkpoint_saves_total, + ), + ( + "counter", + "batch_server:checkpoint_evictions_total", + "Number of free checkpoints evicted.", + self.metrics.checkpoint_evictions_total, + ), + *self.sequence_cache_metric_definitions(), + *self.speculative_metric_definitions(), + *draft_provider_metrics, + ( + "gauge", + "batch_server:radix_trie_sequences", + "Number of sequences tracked in the radix trie.", + len(self.radix_trie.sequence_lengths), + ), + ( + "gauge", + "batch_server:radix_trie_tokens", + "Total tokens tracked in the radix trie.", + self.sequence_history.size, + ), + ] + lines: List[str] = [] + for metric_type, name, help_text, value in metrics_def: + lines.append(f"# HELP {name} {help_text}") + lines.append(f"# TYPE {name} {metric_type}") + lines.append(f"{name} {self._format_prometheus_value(value)}") + lines.extend(self.draft_acceptance_length_metric_lines()) + lines.append("") + return "\n".join(lines) + + def finalize_cancelled(self) -> bool: + finalized = False + for request in list(self.pending_requests): + if request.cancelled: + self.fail_request(request, CompletionRequestCancelledError("request cancelled")) + finalized = True + for request_id in list(self.active_request_ids): + request = self.requests[request_id] + if request.cancelled: + self.fail_request(request, CompletionRequestCancelledError("request cancelled")) + finalized = True + return finalized + + +class CompletionService: + def __init__(self, scheduler: CompletionScheduler) -> None: + self.scheduler = scheduler + self.formatter = scheduler.formatter + self.condition = threading.Condition() + self.commands: Deque[Callable[[], None]] = deque() + self.closed = False + self.worker = threading.Thread( + target=self.run_loop, + name="completion-service", + daemon=True, + ) + self.worker.start() + + def close(self) -> None: + with self.condition: + self.closed = True + self.condition.notify_all() + self.worker.join() + self.scheduler.close() + + def enqueue(self, command: Callable[[], None]) -> None: + with self.condition: + if self.closed: + raise RuntimeError("completion service closed") + self.commands.append(command) + self.condition.notify_all() + + def call_on_scheduler(self, callback: Callable[[], Any]) -> Any: + result_box: Dict[str, Any] = {} + error_box: Dict[str, BaseException] = {} + done = threading.Event() + + def run_callback() -> None: + try: + result_box["result"] = callback() + except BaseException as exc: # noqa: BLE001 + error_box["error"] = exc + finally: + done.set() + + self.enqueue(run_callback) + done.wait() + if "error" in error_box: + raise error_box["error"] + return result_box.get("result") + + def render_prometheus_metrics(self) -> str: + metrics = self.call_on_scheduler(self.scheduler.render_prometheus_metrics) + return cast(str, metrics) + + def run_loop(self) -> None: + while True: + with self.condition: + while not self.closed and not self.commands and self.scheduler.is_idle(): + self.condition.wait() + if self.closed and not self.commands and self.scheduler.is_idle(): + return + commands = list(self.commands) + self.commands.clear() + for command in commands: + command() + progressed = self.scheduler.step() + if progressed: + continue + with self.condition: + if self.closed and not self.commands and self.scheduler.is_idle(): + return + if not self.commands: + self.condition.wait(timeout=0.01) + + def submit_request( + self, + request: CompletionRequest, + ) -> Tuple[CompletionStream, Callable[[], None]]: + mailbox: "queue.Queue[object]" = queue.Queue() + done = threading.Event() + cancelled = threading.Event() + sentinel = object() + result_box: Dict[str, OpenAICompletion] = {} + error_box: Dict[str, BaseException] = {} + + def on_stream_chunk(chunk: CompletionChunk) -> None: + mailbox.put(chunk) + + def on_done(result: OpenAICompletion) -> None: + result_box["result"] = result + done.set() + mailbox.put(sentinel) + + def on_error(exc: BaseException) -> None: + error_box["error"] = exc + done.set() + mailbox.put(sentinel) + + request.on_stream_chunk = on_stream_chunk + request.on_done = on_done + request.on_error = on_error + + def cancel() -> None: + if cancelled.is_set(): + return + cancelled.set() + try: + def cancel_request() -> None: + self.scheduler.cancel(request.id) + + self.enqueue(cancel_request) + except RuntimeError: + pass + + def stream() -> CompletionStream: + try: + while True: + item = mailbox.get() + if item is sentinel: + break + yield cast(CompletionChunk, item) + finally: + if not done.is_set(): + cancel() + if "error" in error_box: + raise error_box["error"] + result = result_box.get("result") + if result is None: + raise RuntimeError("missing completion result") + return result + + def submit_request() -> None: + self.scheduler.submit_request(request) + + self.enqueue(submit_request) + return stream(), cancel + + def submit( + self, + payload: CreateCompletionRequest, + ) -> Tuple[CompletionStream, Callable[[], None]]: + request = self.request_from_payload(payload) + return self.submit_request(request) + + def request_from_payload( + self, + payload: CreateCompletionRequest, + ) -> CompletionRequest: + prompt_text, prompt_plan = self.prepare_completion_prompt(payload) + return self.request_from_prepared( + payload=payload, + prompt_text=prompt_text, + prompt_plan=prompt_plan, + ) + + def request_from_prepared( + self, + *, + payload: CreateCompletionRequest, + prompt_text: str, + prompt_plan: PromptPlan, + grammar_text: Optional[str] = None, + chat_tool_name: Optional[str] = None, + on_stream_chunk: Optional[Callable[[CompletionChunk], None]] = None, + on_done: Optional[Callable[[OpenAICompletion], None]] = None, + on_error: Optional[Callable[[BaseException], None]] = None, + ) -> CompletionRequest: + model = self.scheduler.model + prompt_visible_start = self.prompt_visible_start(prompt_plan) + return CompletionRequest.from_prepared( + payload=payload, + prompt_text=prompt_text, + prompt_plan=prompt_plan, + max_seq_len=model.max_seq_len, + max_output_tokens=model.max_output_tokens, + prompt_visible_start=prompt_visible_start, + prompt_records=self.initial_prompt_records( + payload, + prompt_plan, + prompt_visible_start, + ), + grammar_text=grammar_text, + chat_tool_name=chat_tool_name, + on_stream_chunk=on_stream_chunk, + on_done=on_done, + on_error=on_error, + ) + + def prepare_completion_prompt( + self, + payload: CreateCompletionRequest, + ) -> Tuple[str, PromptPlan]: + model = self.scheduler.model + prompts = payload.normalized_prompt() + if len(prompts) != 1: + raise CompletionRequestValidationError("multiple prompts are not supported") + prompt_item = prompts[0] + if isinstance(prompt_item, str): + prompt_text = prompt_item + try: + prompt_tokens = model.build_prompt_tokens(prompt_text, payload.suffix) + except ValueError as exc: + raise CompletionRequestValidationError(str(exc)) from exc + else: + if payload.suffix is not None: + raise CompletionRequestValidationError( + "suffix is not supported with token id prompts" + ) + prompt_tokens = list(prompt_item) + prompt_text = model.detokenize(prompt_tokens).decode("utf-8", errors="ignore") + return prompt_text, PromptPlan.from_tokens(prompt_text, prompt_tokens) + + def prompt_visible_start(self, prompt_plan: PromptPlan) -> int: + model = self.scheduler.model + if prompt_plan.text_tokens and prompt_plan.text_tokens[0] == model.bos_token: + return 1 + return 0 + + def initial_prompt_records( + self, + payload: CreateCompletionRequest, + prompt_plan: PromptPlan, + prompt_visible_start: int, + ) -> List[Token]: + if not (payload.echo and payload.logprobs is not None): + return [] + if prompt_visible_start >= len(prompt_plan.text_tokens): + return [] + model = self.scheduler.model + first_pos = prompt_visible_start + first_token = prompt_plan.text_tokens[first_pos] + return [ + Token( + token=first_token, + text_bytes=model.token_bytes_with_prev( + prompt_plan.text_tokens[:first_pos], + first_token, + ), + token_logprob=None, + top_logprobs=None, + ) + ] + + +def create_app() -> FastAPI: + app = FastAPI() + app.state.service = None + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + async def watch_http_disconnect( + http_request: Request, + cancel: Callable[[], None], + ) -> None: + while True: + message = await http_request.receive() + if message["type"] == "http.disconnect": + cancel() + return + + async def disconnected_cancelled_response_or_raise( + http_request: Request, + exc: BaseException, + ) -> Response: + if isinstance(exc, CompletionRequestCancelledError): + if await http_request.is_disconnected(): + return Response(status_code=204) + raise exc + + def bad_request(exc: CompletionRequestValidationError) -> HTTPException: + return HTTPException(status_code=400, detail=str(exc)) + + async def collect_completion_result( + formatter: OpenAIFormatter, + http_request: Request, + stream: CompletionStream, + cancel: Callable[[], None], + ) -> OpenAICompletion | Response: + disconnect_task = asyncio.create_task( + watch_http_disconnect(http_request, cancel) + ) + try: + return await asyncio.to_thread(formatter.collect_completion, stream) + except asyncio.CancelledError: + cancel() + raise + except BaseException as exc: + return await disconnected_cancelled_response_or_raise(http_request, exc) + finally: + disconnect_task.cancel() + stream.close() + + async def collect_completion_results( + formatter: OpenAIFormatter, + http_request: Request, + submissions: Sequence[ + Tuple[ + CompletionStream, + Callable[[], None], + ] + ], + ) -> List[OpenAICompletion] | Response: + streams = [stream for stream, _ in submissions] + cancel_all = [cancel for _, cancel in submissions] + + def cancel_all_requests() -> None: + for cancel in cancel_all: + cancel() + + disconnect_task = asyncio.create_task( + watch_http_disconnect( + http_request, + cancel_all_requests, + ) + ) + try: + return await asyncio.gather( + *( + asyncio.to_thread(formatter.collect_completion, stream) + for stream in streams + ) + ) + except asyncio.CancelledError: + cancel_all_requests() + raise + except BaseException as exc: + return await disconnected_cancelled_response_or_raise(http_request, exc) + finally: + disconnect_task.cancel() + for stream in streams: + stream.close() + + async def stream_sse_chunks( + formatter: OpenAIFormatter, + http_request: Request, + stream: CompletionStream, + cancel: Callable[[], None], + chunk_payloads: Callable[[CompletionChunk], Iterable[Any]], + ) -> AsyncIterator[bytes]: + disconnect_task = asyncio.create_task( + watch_http_disconnect(http_request, cancel) + ) + try: + while True: + done, chunk = await asyncio.to_thread( + formatter.next_stream_chunk, + stream, + ) + if done: + break + assert chunk is not None + for payload in chunk_payloads(chunk): + yield formatter.encode_sse_payload(payload) + yield b"data: [DONE]\n\n" + except asyncio.CancelledError: + cancel() + raise + except BaseException as exc: + cancel() + if ( + isinstance(exc, CompletionRequestCancelledError) + and await http_request.is_disconnected() + ): + return + raise + finally: + disconnect_task.cancel() + + async def stream_sse_outputs( + formatter: OpenAIFormatter, + http_request: Request, + stream: CompletionStream, + cancel: Callable[[], None], + *, + initial_payloads: Iterable[BaseModel | Dict[str, Any]], + chunk_payloads: Callable[[CompletionChunk], Iterable[Any]], + done_payloads: Callable[ + [Optional[OpenAICompletion]], Iterable[BaseModel | Dict[str, Any]] + ], + ) -> AsyncIterator[bytes]: + disconnect_task = asyncio.create_task( + watch_http_disconnect(http_request, cancel) + ) + try: + for payload in initial_payloads: + yield formatter.encode_sse_payload(payload) + while True: + done, chunk, result = await asyncio.to_thread( + formatter.next_stream_output, + stream, + ) + if done: + for payload in done_payloads(result): + yield formatter.encode_sse_payload(payload) + break + assert chunk is not None + for payload in chunk_payloads(chunk): + yield formatter.encode_sse_payload(payload) + yield b"data: [DONE]\n\n" + except asyncio.CancelledError: + cancel() + raise + except BaseException as exc: + cancel() + if ( + isinstance(exc, CompletionRequestCancelledError) + and await http_request.is_disconnected() + ): + return + raise + finally: + disconnect_task.cancel() + + async def stream_websocket_responses( + websocket: WebSocket, + formatter: OpenAIFormatter, + stream: CompletionStream, + _: Callable[[], None], + *, + initial_payloads: Iterable[BaseModel | Dict[str, Any]], + chunk_payloads: Callable[[CompletionChunk], Iterable[Any]], + done_payloads: Callable[ + [Optional[OpenAICompletion]], Iterable[BaseModel | Dict[str, Any]] + ], + ) -> None: + try: + for payload in initial_payloads: + await websocket.send_json(payload) + while True: + done, chunk, result = await asyncio.to_thread( + formatter.next_stream_output, + stream, + ) + if done: + for payload in done_payloads(result): + await websocket.send_json(payload) + break + assert chunk is not None + for payload in chunk_payloads(chunk): + await websocket.send_json(payload) + finally: + stream.close() + + @app.post("/v1/completions") + async def create_completion( # pyright: ignore[reportUnusedFunction] + http_request: Request, + body: CreateCompletionRequest, + ): + service: CompletionService = app.state.service + formatter = service.formatter + prompts = body.normalized_prompt() + if len(prompts) > 1: + if body.stream: + raise HTTPException( + status_code=400, + detail="streaming does not support multiple prompts", + ) + try: + submissions = [ + service.submit(body.model_copy(update={"prompt": prompt})) + for prompt in prompts + ] + except CompletionRequestValidationError as exc: + raise bad_request(exc) from exc + results = await collect_completion_results( + formatter, http_request, submissions + ) + if isinstance(results, Response): + return results + return JSONResponse( + formatter.aggregate_completion_results(results).model_dump( + mode="json", + exclude_none=True, + ) + ) + try: + stream, cancel = service.submit( + body.model_copy(update={"prompt": prompts[0]}) + ) + except CompletionRequestValidationError as exc: + raise bad_request(exc) from exc + if body.stream: + return StreamingResponse( + stream_sse_chunks( + formatter, + http_request, + stream, + cancel, + lambda chunk: [chunk], + ), + media_type="text/event-stream", + ) + result = await collect_completion_result( + formatter, http_request, stream, cancel + ) + if isinstance(result, Response): + return result + return JSONResponse(result.model_dump(mode="json", exclude_none=True)) + + @app.post("/v1/chat/completions") + async def create_chat_completion( # pyright: ignore[reportUnusedFunction] + http_request: Request, body: CreateChatCompletionRequest + ): + service: CompletionService = app.state.service + formatter = service.formatter + try: + parts = formatter.completion_request_from_chat_request( + body, + ) + except CompletionRequestValidationError as exc: + raise bad_request(exc) from exc + template_functions = ( + [function.to_template_function() for function in body.functions] + if body.functions is not None + else None + ) + template_tools = ( + [tool.to_template_tool() for tool in body.tools] + if body.tools is not None + else None + ) + try: + request = service.request_from_prepared( + payload=parts.payload, + prompt_text=parts.prompt_text, + prompt_plan=parts.prompt_plan, + grammar_text=parts.grammar_text, + chat_tool_name=parts.tool_name, + ) + stream, cancel = service.submit_request(request) + except CompletionRequestValidationError as exc: + raise bad_request(exc) from exc + if body.stream: + started_indices: set[int] = set() + + def chat_chunk_payloads( + completion_chunk: CompletionChunk, + ) -> Iterable[BaseModel | Dict[str, Any]]: + return formatter.convert_completion_chunk_to_chat_chunks( + completion_chunk, + started_indices, + parts.tool_name, + functions=template_functions, + tools=template_tools, + parsed_states=parsed_states, + generation_prompt=parts.generation_prompt, + ) + + parsed_states: Dict[int, Dict[str, Any]] = {} + return StreamingResponse( + stream_sse_chunks( + formatter, + http_request, + stream, + cancel, + chat_chunk_payloads, + ), + media_type="text/event-stream", + ) + completion = await collect_completion_result( + formatter, http_request, stream, cancel + ) + if isinstance(completion, Response): + return completion + chat_response = formatter.convert_completion_response_to_chat( + completion, + parts.tool_name, + functions=template_functions, + tools=template_tools, + generation_prompt=parts.generation_prompt, + ) + if isinstance(chat_response, BaseModel): + return JSONResponse( + chat_response.model_dump(mode="json", exclude_none=True) + ) + return JSONResponse(chat_response) + + @app.post("/v1/responses") + async def create_response( # pyright: ignore[reportUnusedFunction] + http_request: Request, + body: CreateResponseRequest, + ): + service: CompletionService = app.state.service + formatter = service.formatter + try: + chat_parts = formatter.response_request_to_chat_parts(body) + parts = formatter.completion_request_from_response_chat_parts(chat_parts) + except CompletionRequestValidationError as exc: + raise bad_request(exc) from exc + response_tools = chat_parts.tools + try: + request = service.request_from_prepared( + payload=parts.payload, + prompt_text=parts.prompt_text, + prompt_plan=parts.prompt_plan, + grammar_text=parts.grammar_text, + chat_tool_name=parts.tool_name, + ) + stream, cancel = service.submit_request(request) + except CompletionRequestValidationError as exc: + raise bad_request(exc) from exc + if body.stream: + started_indices: set[int] = set() + parsed_states: Dict[int, ResponseParser] = {} + stream_state = OpenAIFormatter.ResponsesStream( + body=body, + response_id="resp_" + request.id, + created_at=float(request.created), + model=service.scheduler.model.model_path, + ) + + def response_chunk_payloads( + completion_chunk: CompletionChunk, + ) -> Iterable[BaseModel | Dict[str, Any]]: + chat_chunks = formatter.convert_completion_chunk_to_chat_chunks( + completion_chunk, + started_indices, + parts.tool_name, + tools=response_tools, + parsed_states=parsed_states, + generation_prompt=parts.generation_prompt, + ) + payloads: List[BaseModel | Dict[str, Any]] = [] + for chat_chunk in chat_chunks: + payloads.extend( + formatter.convert_chat_chunk_to_response_events( + chat_chunk, + stream_state, + ) + ) + return payloads + + return StreamingResponse( + stream_sse_outputs( + formatter, + http_request, + stream, + cancel, + initial_payloads=formatter.start_response_stream(stream_state), + chunk_payloads=response_chunk_payloads, + done_payloads=lambda completion: ( + formatter.response_stream_terminal_events( + stream_state, + completion, + ) + ), + ), + media_type="text/event-stream", + ) + completion = await collect_completion_result( + formatter, http_request, stream, cancel + ) + if isinstance(completion, Response): + return completion + return JSONResponse( + formatter.convert_completion_response_to_response( + completion, + body, + parts.tool_name, + tools=response_tools, + generation_prompt=parts.generation_prompt, + ) + ) + + @app.websocket("/v1/responses") + async def responses_websocket( # pyright: ignore[reportUnusedFunction] + websocket: WebSocket, + ): + await websocket.accept() + service: CompletionService = app.state.service + formatter = service.formatter + # HTTP /v1/responses remains stateless. The websocket transport keeps + # per-connection response history so Codex can use previous_response_id + # within a single live session. + websocket_response_history: Dict[str, ResponsesWebSocketState] = {} + + def websocket_request_with_ephemeral_history( + ws_body: ResponseCreateWebSocketRequest, + ) -> CreateResponseRequest: + body = ws_body.to_create_response_request() + previous_response_id = body.previous_response_id + if previous_response_id is None: + return body + prior_state = websocket_response_history.get(previous_response_id) + if prior_state is None: + raise CompletionRequestValidationError( + f"unknown previous_response_id: {previous_response_id}" + ) + current_items = formatter._clone_response_input_items(body.input) + replay_items = formatter._clone_response_input_items( + prior_state.input_items + ) + for item in prior_state.output_items: + normalized = formatter._normalize_response_output_item_for_input(item) + if normalized is not None: + replay_items.append(normalized) + replay_items.extend(current_items) + return body.model_copy( + update={ + "input": replay_items, + "previous_response_id": None, + } + ) + + async def send_error(message: str) -> None: + await websocket.send_json( + { + "type": "error", + "error": { + "message": message, + }, + } + ) + + try: + while True: + try: + payload = await websocket.receive_json() + except WebSocketDisconnect: + break + except Exception: + await send_error("invalid websocket request payload") + continue + + try: + ws_body = ResponseCreateWebSocketRequest.model_validate(payload) + body = websocket_request_with_ephemeral_history(ws_body) + if ws_body.generate is False: + body = body.model_copy(update={"max_output_tokens": 0}) + chat_parts = formatter.response_request_to_chat_parts(body) + parts = formatter.completion_request_from_response_chat_parts(chat_parts) + response_tools = chat_parts.tools + request = service.request_from_prepared( + payload=parts.payload, + prompt_text=parts.prompt_text, + prompt_plan=parts.prompt_plan, + grammar_text=parts.grammar_text, + chat_tool_name=parts.tool_name, + ) + stream, cancel = service.submit_request(request) + except CompletionRequestValidationError as exc: + await send_error(str(exc)) + continue + + started_indices: set[int] = set() + parsed_states: Dict[int, ResponseParser] = {} + stream_state = OpenAIFormatter.ResponsesStream( + body=body, + response_id="resp_" + request.id, + created_at=float(request.created), + model=service.scheduler.model.model_path, + ) + + def response_chunk_payloads( + completion_chunk: CompletionChunk, + ) -> Iterable[BaseModel | Dict[str, Any]]: + chat_chunks = formatter.convert_completion_chunk_to_chat_chunks( + completion_chunk, + started_indices, + parts.tool_name, + tools=response_tools, + parsed_states=parsed_states, + generation_prompt=parts.generation_prompt, + ) + payloads: List[BaseModel | Dict[str, Any]] = [] + for chat_chunk in chat_chunks: + payloads.extend( + formatter.convert_chat_chunk_to_response_events( + chat_chunk, + stream_state, + ) + ) + return payloads + + try: + await stream_websocket_responses( + websocket, + formatter, + stream, + cancel, + initial_payloads=formatter.start_response_stream(stream_state), + chunk_payloads=response_chunk_payloads, + done_payloads=lambda completion: ( + formatter.response_stream_terminal_events( + stream_state, + completion, + ) + ), + ) + websocket_response_history[stream_state.response_id] = ( + ResponsesWebSocketState( + input_items=formatter._clone_response_input_items( + body.input + ), + output_items=copy.deepcopy(stream_state.output), + ) + ) + except WebSocketDisconnect: + cancel() + break + except BaseException as exc: + cancel() + await send_error(str(exc)) + except WebSocketDisconnect: + pass + + @app.get("/v1/models", response_model=ModelListResponse) + async def list_models() -> ModelListResponse: # pyright: ignore[reportUnusedFunction] + service = app.state.service + model = service.scheduler.model + model_id = getattr(model, "model_alias", None) or model.model_path + created = int(time.time()) + return ModelListResponse( + data=[ + ModelCardResponse( + id=model_id, + created=created, + owned_by="llama-cpp-python", + ) + ] + ) + + @app.get("/healthz", response_model=HealthzResponse) + async def healthz() -> HealthzResponse: # pyright: ignore[reportUnusedFunction] + return HealthzResponse() + + @app.get("/metrics") + async def metrics() -> Response: # pyright: ignore[reportUnusedFunction] + service = cast(Optional[CompletionService], getattr(app.state, "service", None)) + if service is None: + raise HTTPException(status_code=503, detail="completion service unavailable") + payload = await asyncio.to_thread(service.render_prometheus_metrics) + return Response( + payload, + media_type="text/plain; version=0.0.4; charset=utf-8", + ) + + return app + + +APP = create_app() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("-C", "--config-file", required=True) + args = parser.parse_args() + config = ConfigFile.load(args.config_file) + model_path = config.model.resolve_model_path() + loras = [ + Model.LoraAdapter(path=lora.resolve_path(), scale=lora.scale) + for lora in config.model.loras + ] + model = Model( + model_path=model_path, + model_alias=config.model.alias, + chat_template=config.model.chat_template, + loras=loras, + n_gpu_layers=config.model.n_gpu_layers, + split_mode=config.model.split_mode, + main_gpu=config.model.main_gpu, + tensor_split=config.model.tensor_split, + vocab_only=config.model.vocab_only, + use_mmap=config.model.use_mmap, + use_mlock=config.model.use_mlock, + kv_overrides=config.model.kv_overrides, + n_ctx=config.model.n_ctx, + n_batch=config.model.n_batch, + n_ubatch=config.model.n_ubatch, + n_seq_max=config.model.n_seq_max, + n_threads=config.model.threads, + n_threads_batch=config.model.threads_batch, + rope_scaling_type=config.model.rope_scaling_type, + pooling_type=config.model.pooling_type, + attention_type=config.model.attention_type, + rope_freq_base=config.model.rope_freq_base, + rope_freq_scale=config.model.rope_freq_scale, + yarn_ext_factor=config.model.yarn_ext_factor, + yarn_attn_factor=config.model.yarn_attn_factor, + yarn_beta_fast=config.model.yarn_beta_fast, + yarn_beta_slow=config.model.yarn_beta_slow, + yarn_orig_ctx=config.model.yarn_orig_ctx, + offload_kqv=config.model.offload_kqv, + flash_attn=config.model.flash_attn, + op_offload=config.model.op_offload, + swa_full=config.model.swa_full, + no_perf=config.model.no_perf, + type_k=config.model.type_k, + type_v=config.model.type_v, + kv_unified=config.model.kv_unified, + max_seq_len=config.model.max_seq_len, + max_output_tokens=config.model.max_output_tokens, + draft_model=config.model.draft_model, + draft_model_num_pred_tokens=config.model.draft_model_num_pred_tokens, + draft_model_max_ngram_size=config.model.draft_model_max_ngram_size, + draft_model_top_k=config.model.draft_model_top_k, + draft_model_p_min=config.model.draft_model_p_min, + draft_model_max_batch_size=config.model.draft_model_max_batch_size, + draft_model_threads=config.model.draft_model_threads, + draft_model_threads_batch=config.model.draft_model_threads_batch, + response_schema=config.model.response_schema, + store_logits=config.model.store_logits, + ) + if config.model.mtmd is not None: + if model.chat_formatter is None: + raise RuntimeError("MTMD requires a GGUF chat template") + mmproj_path = config.model.mtmd.resolve_mmproj_path() + embedding_cache: Optional[MTMDEmbeddingCache] = None + if config.model.mtmd.embedding_cache is not None: + embedding_cache = MTMDEmbeddingCache( + path=config.model.mtmd.embedding_cache.path, + max_bytes=config.model.mtmd.embedding_cache.max_bytes, + model_fingerprint=MTMDEmbeddingCache.fingerprint_file(model_path), + mmproj_fingerprint=MTMDEmbeddingCache.fingerprint_file(mmproj_path), + ) + model.mtmd_processor = MTMDProcessor( + model_path=model.model_path, + llama_model=model.llama_model, + chat_formatter=model.chat_formatter, + tokenize=model.tokenize, + n_embd_inp=model.n_embd_inp, + n_batch=model.n_batch, + n_ubatch=model.n_ubatch, + n_threads_batch=model.n_threads_batch, + mmproj_path=mmproj_path, + embedding_cache=embedding_cache, + allowed_media_domains=config.model.mtmd.allowed_media_domains, + allowed_local_media_path=config.model.mtmd.allowed_local_media_path, + image_max_bytes=config.model.mtmd.image_max_bytes, + audio_max_bytes=config.model.mtmd.audio_max_bytes, + image_timeout_seconds=config.model.mtmd.image_timeout_seconds, + ) + sequence_cache: Optional[SequenceCache] = None + if config.disk_cache is not None: + sequence_cache = SequenceDiskCache( + path=config.disk_cache.path, + max_bytes=config.disk_cache.max_bytes, + min_tokens=config.disk_cache.min_tokens, + compatibility_key=SequenceDiskCache.compatibility_key_for_model(model), + ) + scheduler = CompletionScheduler(model, sequence_cache=sequence_cache) + APP.state.service = CompletionService(scheduler) + try: + uvicorn.run( + APP, host=config.server.host, port=config.server.port, log_level="info" + ) + finally: + APP.state.service.close() + + +if __name__ == "__main__": + main() diff --git a/llama_cpp/llama_cpp_ext.py b/llama_cpp/llama_cpp_ext.py new file mode 100644 index 0000000000..f6ab9197b8 --- /dev/null +++ b/llama_cpp/llama_cpp_ext.py @@ -0,0 +1,99 @@ +"""Experimental bindings for non-public llama.cpp APIs from `llama-ext.h`. + +This module is not part of the stable llama-cpp-python public API. +Downstream code should not import or depend on it directly. +""" + +from __future__ import annotations + +import ctypes +import functools + +from typing import Any, Iterable, Union + +from . import llama_cpp + +_lib = llama_cpp._lib + + +def _ctypes_function_from_names( + names: Iterable[str], + argtypes: list[Any], + restype: Any, +): + """Decorator for extension functions whose exported symbol name can vary by ABI.""" + + def decorator(f): + missing: list[str] = [] + for name in names: + try: + func = getattr(_lib, name) + except AttributeError: + missing.append(name) + continue + func.argtypes = argtypes + func.restype = restype + functools.wraps(f)(func) + return func + raise AttributeError( + f"None of the shared library symbols were found: {', '.join(missing)}" + ) + + return decorator + + +# LLAMA_API void llama_set_embeddings_pre_norm(struct llama_context * ctx, bool value, bool masked); +@_ctypes_function_from_names( + ( + "llama_set_embeddings_pre_norm", + "_Z29llama_set_embeddings_pre_normP13llama_contextbb", + "?llama_set_embeddings_pre_norm@@YAXPEAUllama_context@@_N1@Z", + ), + [llama_cpp.llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], + None, +) +def llama_set_embeddings_pre_norm( + ctx: llama_cpp.llama_context_p, + value: bool, + masked: bool, + /, +): + """Set whether the context outputs pre-norm embeddings or not.""" + ... + + +# LLAMA_API float * llama_get_embeddings_pre_norm(struct llama_context * ctx); +@_ctypes_function_from_names( + ( + "llama_get_embeddings_pre_norm", + "_Z29llama_get_embeddings_pre_normP13llama_context", + "?llama_get_embeddings_pre_norm@@YAPEAMPEAUllama_context@@@Z", + ), + [llama_cpp.llama_context_p_ctypes], + ctypes.POINTER(ctypes.c_float), +) +def llama_get_embeddings_pre_norm( + ctx: llama_cpp.llama_context_p, + /, +): + """Get the pre-norm embeddings from the last evaluation.""" + ... + + +# LLAMA_API float * llama_get_embeddings_pre_norm_ith(struct llama_context * ctx, int32_t i); +@_ctypes_function_from_names( + ( + "llama_get_embeddings_pre_norm_ith", + "_Z33llama_get_embeddings_pre_norm_ithP13llama_contexti", + "?llama_get_embeddings_pre_norm_ith@@YAPEAMPEAUllama_context@@H@Z", + ), + [llama_cpp.llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(ctypes.c_float), +) +def llama_get_embeddings_pre_norm_ith( + ctx: llama_cpp.llama_context_p, + i: Union[ctypes.c_int32, int], + /, +): + """Get the pre-norm embeddings for the ith output row from the last evaluation.""" + ... From cf1883063bfa8b80cdd95a5e65fb923e97381105 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 03:22:56 -0700 Subject: [PATCH 546/561] feat: update llama.cpp to ggml-org/llama.cpp@465b1f0e7 (#2278) --- CHANGELOG.md | 2 +- examples/server/server.py | 1 + llama_cpp/llama_chat_format.py | 2 ++ llama_cpp/mtmd_cpp.py | 11 ++++++----- vendor/llama.cpp | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbe7b13bdc..7c6a7d969b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: update llama.cpp to ggml-org/llama.cpp@5a69c9743 +- feat: update llama.cpp to ggml-org/llama.cpp@465b1f0e7 - feat(example): Updated server example (batch processing, multi-token prediction, `/v1/responses` api, response parsing) by @abetlen in #2174 ## [0.3.26] diff --git a/examples/server/server.py b/examples/server/server.py index 6a500ceec9..590a214208 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -10102,6 +10102,7 @@ def _create_bitmap(self, media_bytes: bytes, kind: Literal["image", "audio"]) -> self.ctx, buffer, len(media_bytes), + False, ) if bitmap is None: raise CompletionRequestValidationError(f"failed to create MTMD {kind} bitmap") diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 4298642f67..0034bdae98 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2841,6 +2841,7 @@ def _create_bitmap_from_bytes(self, image_bytes: bytes): self.mtmd_ctx, (ctypes.c_uint8 * len(image_bytes)).from_buffer(bytearray(image_bytes)), len(image_bytes), + False, ) if bitmap is None: @@ -3332,6 +3333,7 @@ def _create_bitmap_from_bytes(self, image_bytes: bytes): self.mtmd_ctx, (ctypes.c_uint8 * len(image_bytes)).from_buffer(bytearray(image_bytes)), len(image_bytes), + False, ) if bitmap is None: diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 8c2ebbfef4..abe4c7a679 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -551,29 +551,30 @@ def mtmd_test_create_input_chunks() -> Optional[mtmd_input_chunks_p]: ################################################ -# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname); +# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); @ctypes_function( "mtmd_helper_bitmap_init_from_file", - [mtmd_context_p_ctypes, c_char_p], + [mtmd_context_p_ctypes, c_char_p, c_bool], mtmd_bitmap_p_ctypes, ) def mtmd_helper_bitmap_init_from_file( - ctx: mtmd_context_p, fname: bytes, / + ctx: mtmd_context_p, fname: bytes, placeholder: Union[c_bool, bool], / ) -> Optional[mtmd_bitmap_p]: """Initialize an MTMD bitmap from a file.""" ... -# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len); +# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); @ctypes_function( "mtmd_helper_bitmap_init_from_buf", - [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t], + [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t, c_bool], mtmd_bitmap_p_ctypes, ) def mtmd_helper_bitmap_init_from_buf( ctx: mtmd_context_p, buf: CtypesArray[c_uint8], length: Union[c_size_t, int], + placeholder: Union[c_bool, bool], /, ) -> Optional[mtmd_bitmap_p]: ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 5a69c97439..465b1f0e75 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 5a69c974392020e514c3b2b2910bb92f847cb4c9 +Subproject commit 465b1f0e75c590426cff3ca998bcd25297071a5b From 380177b7b9c09f018d51de7d6410ab8adf5c9754 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 03:51:23 -0700 Subject: [PATCH 547/561] chore: bump version to 0.3.27 (#2279) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c6a7d969b..229921847f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.27] + - feat: update llama.cpp to ggml-org/llama.cpp@465b1f0e7 - feat(example): Updated server example (batch processing, multi-token prediction, `/v1/responses` api, response parsing) by @abetlen in #2174 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index bbfb73de3f..369f24ca88 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.26" +__version__ = "0.3.27" From fe927bd000f045ee1b488fbe326421033b8b2716 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 15:09:29 -0700 Subject: [PATCH 548/561] feat(example): add OpenAI-compatible embeddings endpoint (#2281) * feat(example): add embeddings endpoint * docs: update changelog for embeddings endpoint * test: remove server embeddings example test * feat(example): auto-detect embedding model mode --- CHANGELOG.md | 2 + examples/server/README.md | 27 +- .../server/configs/bge-small-en-v1.5.json | 22 + examples/server/server.py | 395 +++++++++++++++++- 4 files changed, 438 insertions(+), 8 deletions(-) create mode 100644 examples/server/configs/bge-small-en-v1.5.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 229921847f..4912e00e45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat(example): add OpenAI-compatible embeddings endpoint by @abetlen in #2281 + ## [0.3.27] - feat: update llama.cpp to ggml-org/llama.cpp@465b1f0e7 diff --git a/examples/server/README.md b/examples/server/README.md index 1f6e0d3dbe..6a1cc44480 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -1,7 +1,7 @@ # Server Example This example is an updated OpenAI-compatible web server that depends only on the low-level C bindings. -It supports batched inference, prompt caching, response parsing, `/v1/responses`, disk sequence caching, MTP, LoRA, and multimodal image/audio inputs. +It supports batched inference, prompt caching, response parsing, `/v1/responses`, `/v1/embeddings`, disk sequence caching, MTP, LoRA, and multimodal image/audio inputs. ## Setup @@ -46,6 +46,7 @@ The smallest checked-in example uses Qwen3.5 0.8B so the server can be started o | Config | Model | Notes | | --- | --- | --- | +| [`configs/bge-small-en-v1.5.json`](configs/bge-small-en-v1.5.json) | [`CompendiumLabs/bge-small-en-v1.5-gguf`](https://huggingface.co/CompendiumLabs/bge-small-en-v1.5-gguf) | Small embedding model config for `/v1/embeddings`. | | [`configs/qwen3.5-0.8b.json`](configs/qwen3.5-0.8b.json) | [`lmstudio-community/Qwen3.5-0.8B-GGUF`](https://huggingface.co/lmstudio-community/Qwen3.5-0.8B-GGUF) | Default small multimodal example. | | [`configs/gemma-4-12b-it-qat.json`](configs/gemma-4-12b-it-qat.json) | [`unsloth/gemma-4-12B-it-qat-GGUF`](https://huggingface.co/unsloth/gemma-4-12B-it-qat-GGUF) | Larger Gemma 4 QAT multimodal config with projector. | | [`configs/qwen3.6-27b.json`](configs/qwen3.6-27b.json) | [`unsloth/Qwen3.6-27B-GGUF`](https://huggingface.co/unsloth/Qwen3.6-27B-GGUF) | Larger Qwen3.6 multimodal config. | @@ -86,11 +87,33 @@ response = client.responses.create( print(response.output_text) ``` +### Embeddings + +Start the server with an embedding config before calling `/v1/embeddings`. + +```bash +cd examples/server +uv run --script server.py -C configs/bge-small-en-v1.5.json +``` + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://127.0.0.1:8000/v1", api_key="not-used") + +response = client.embeddings.create( + model="bge-small-en-v1.5", + input=["The food was delicious.", "The meal was excellent."], +) +print(len(response.data[0].embedding)) +``` + ## API Surface | Endpoint | Purpose | Reference | | --- | --- | --- | | `POST /v1/completions` | Legacy text completions with streaming, stop sequences, logprobs, penalties, seeds, and grammar-backed JSON output. | [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions) | +| `POST /v1/embeddings` | OpenAI-compatible embeddings for embedding-mode GGUF models, including string inputs, token inputs, base64 output, and dimensions truncation. | [OpenAI Embeddings API](https://platform.openai.com/docs/api-reference/embeddings) | | `POST /v1/chat/completions` | Chat completions with streaming, tools, forced tool choice, reasoning parsing, multimodal content parts, and structured response parsing. | [OpenAI Chat API](https://platform.openai.com/docs/api-reference/chat) | | `POST /v1/responses` | Stateless Responses API compatibility for clients that use response items and response events. | [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) | | `WS /v1/responses` | Stateful websocket Responses transport with per-connection `previous_response_id` replay. | [OpenAI Responses API](https://platform.openai.com/docs/api-reference/responses) | @@ -190,6 +213,8 @@ Most model runtime fields map to `llama_model_params` or `llama_context_params` | `threads` | Decode thread count. | | `threads_batch` | Prefill and batch thread count. | | `kv_unified` | Selects unified or per-sequence memory layout. | +| `embedding` | Overrides embedding mode; omit to auto-detect pooled embedding GGUFs from model metadata. | +| `pooling_type` | Overrides pooled embedding behavior for embedding models, such as `1` for mean pooling. | | `store_logits` | Keeps logits after decode when needed by sampling or diagnostics. | | `use_mmap` | Memory maps model weights. | | `use_mlock` | Attempts to lock model pages into RAM. | diff --git a/examples/server/configs/bge-small-en-v1.5.json b/examples/server/configs/bge-small-en-v1.5.json new file mode 100644 index 0000000000..3fc8016df8 --- /dev/null +++ b/examples/server/configs/bge-small-en-v1.5.json @@ -0,0 +1,22 @@ +{ + "server": { + "host": "0.0.0.0", + "port": 8000 + }, + "model": { + "alias": "bge-small-en-v1.5", + "from_pretrained": { + "repo_id": "CompendiumLabs/bge-small-en-v1.5-gguf", + "filename": "bge-small-en-v1.5-q4_k_m.gguf" + }, + "n_ctx": 512, + "n_seq_max": 16, + "n_batch": 512, + "n_ubatch": 512, + "threads": 4, + "threads_batch": 8, + "kv_unified": true, + "store_logits": false, + "use_mmap": true + } +} diff --git a/examples/server/server.py b/examples/server/server.py index 590a214208..2530cf4bec 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -2457,6 +2457,7 @@ class CompletionChunk(TypedDict): CompletionStream = Generator[CompletionChunk, None, OpenAICompletion] CompletionPrompt = Union[str, List[int], List[str], List[List[int]]] +EmbeddingInput = Union[str, List[str], List[int], List[List[int]]] class CreateCompletionRequest(BaseModel): @@ -2522,6 +2523,126 @@ def normalized_prompt(self) -> List[Union[str, List[int]]]: raise ValueError("prompt must be a string, token ids, list of strings, or list of token-id lists") +class CreateEmbeddingRequest(BaseModel): + model_config = ConfigDict(extra="ignore") + + input: EmbeddingInput + model: str + encoding_format: Literal["float", "base64"] = "float" + dimensions: Optional[int] = Field(default=None, ge=1) + user: Optional[str] = None + + @staticmethod + def _validate_text_input(text: str) -> str: + if text == "": + raise ValueError("embedding input must not contain empty strings") + return text + + @staticmethod + def _validate_token_input(tokens: List[int]) -> List[int]: + if not tokens: + raise ValueError("embedding token input must not be empty") + if len(tokens) > 2048: + raise ValueError("embedding token input must not exceed 2048 tokens") + return tokens + + @model_validator(mode="after") + def validate_after(self) -> "CreateEmbeddingRequest": + self.normalized_input() + return self + + def normalized_input(self) -> List[Union[str, List[int]]]: + if isinstance(self.input, str): + return [self._validate_text_input(self.input)] + if all(isinstance(token, int) for token in self.input): + return [self._validate_token_input(cast(List[int], self.input))] + if all(isinstance(item, str) for item in self.input): + if len(self.input) > 2048: + raise ValueError("embedding input array must not exceed 2048 items") + return [ + self._validate_text_input(item) + for item in cast(List[str], self.input) + ] + if all( + isinstance(item, list) + and all(isinstance(token, int) for token in item) + for item in self.input + ): + if len(self.input) > 2048: + raise ValueError("embedding input array must not exceed 2048 items") + return [ + self._validate_token_input(item) + for item in cast(List[List[int]], self.input) + ] + raise ValueError( + "embedding input must be a string, list of strings, token ids, or list of token-id lists" + ) + + +class EmbeddingDataResponse(BaseModel): + object: Literal["embedding"] = "embedding" + embedding: Union[List[float], str] + index: int + + +class EmbeddingUsageResponse(BaseModel): + prompt_tokens: int + total_tokens: int + + +class CreateEmbeddingResponse(BaseModel): + object: Literal["list"] = "list" + data: List[EmbeddingDataResponse] + model: str + usage: EmbeddingUsageResponse + + @staticmethod + def encode_embedding( + embedding: Sequence[float], + encoding_format: Literal["float", "base64"], + dimensions: Optional[int], + ) -> Union[List[float], str]: + if dimensions is not None: + if dimensions > len(embedding): + raise CompletionRequestValidationError( + f"dimensions ({dimensions}) exceeds embedding size ({len(embedding)})" + ) + embedding = embedding[:dimensions] + if encoding_format == "float": + return [float(value) for value in embedding] + array = np.asarray(embedding, dtype=np.float32) + return base64.b64encode(array.tobytes()).decode("ascii") + + @classmethod + def from_embeddings( + cls, + *, + model: str, + embeddings: Sequence[Sequence[float]], + total_tokens: int, + encoding_format: Literal["float", "base64"], + dimensions: Optional[int], + ) -> "CreateEmbeddingResponse": + return cls( + data=[ + EmbeddingDataResponse( + embedding=cls.encode_embedding( + embedding, + encoding_format, + dimensions, + ), + index=index, + ) + for index, embedding in enumerate(embeddings) + ], + model=model, + usage=EmbeddingUsageResponse( + prompt_tokens=total_tokens, + total_tokens=total_tokens, + ), + ) + + class ChatCompletionFunctionCall(BaseModel): model_config = ConfigDict( extra="allow", @@ -3214,6 +3335,7 @@ class ModelOptions(BaseModel): rope_scaling_type: Optional[int] = None pooling_type: Optional[int] = None attention_type: Optional[int] = None + embedding: Optional[bool] = None rope_freq_base: Optional[float] = None rope_freq_scale: Optional[float] = None yarn_ext_factor: Optional[float] = None @@ -10408,6 +10530,7 @@ def __init__( rope_scaling_type: Optional[int] = None, pooling_type: Optional[int] = None, attention_type: Optional[int] = None, + embedding: Optional[bool] = None, rope_freq_base: Optional[float] = None, rope_freq_scale: Optional[float] = None, yarn_ext_factor: Optional[float] = None, @@ -10473,9 +10596,13 @@ def __init__( if vocab is None: raise RuntimeError("failed to access model vocabulary") self.vocab = vocab - if llama_cpp.llama_model_has_encoder(llama_model): + embedding = self.resolve_embedding_mode(llama_model, embedding) + self.embedding = embedding + self.has_encoder = bool(llama_cpp.llama_model_has_encoder(llama_model)) + self.has_decoder = bool(llama_cpp.llama_model_has_decoder(llama_model)) + if self.has_encoder and not embedding: raise RuntimeError("encoder models are not supported") - if not llama_cpp.llama_model_has_decoder(llama_model): + if not self.has_decoder and not (embedding and self.has_encoder): raise RuntimeError("decoder is required") if llama_cpp.llama_model_is_recurrent(llama_model): self.memory_model = "recurrent" @@ -10519,6 +10646,7 @@ def __init__( rope_scaling_type=rope_scaling_type, pooling_type=pooling_type, attention_type=attention_type, + embedding=embedding, rope_freq_base=rope_freq_base, rope_freq_scale=rope_freq_scale, yarn_ext_factor=yarn_ext_factor, @@ -10542,7 +10670,7 @@ def __init__( raise RuntimeError("failed to create context") self.ctx = ctx mem = llama_cpp.llama_get_memory(ctx) - if mem is None: + if mem is None and not embedding: raise RuntimeError("failed to access model memory") self.mem = mem self.n_ctx = int(llama_cpp.llama_n_ctx(ctx)) @@ -10571,7 +10699,11 @@ def __init__( ) self.n_ctx_train = n_ctx_train self.n_vocab = int(llama_cpp.llama_vocab_n_tokens(self.vocab)) + self.n_embd = int(llama_cpp.llama_model_n_embd(self.llama_model)) self.n_embd_inp = int(llama_cpp.llama_model_n_embd_inp(self.llama_model)) + self.n_embd_out = int(llama_cpp.llama_model_n_embd_out(self.llama_model)) + if self.n_embd_out <= 0: + self.n_embd_out = self.n_embd self.kv_unified = kv_unified self.max_seq_len_limit = min(self.request_context_limit, self.n_ctx_train) if max_seq_len is None: @@ -10644,6 +10776,7 @@ def __init__( rope_scaling_type=rope_scaling_type, pooling_type=pooling_type, attention_type=attention_type, + embedding=embedding, rope_freq_base=rope_freq_base, rope_freq_scale=rope_freq_scale, yarn_ext_factor=yarn_ext_factor, @@ -10786,6 +10919,7 @@ def build_context_params( rope_scaling_type: Optional[int], pooling_type: Optional[int], attention_type: Optional[int], + embedding: bool, rope_freq_base: Optional[float], rope_freq_scale: Optional[float], yarn_ext_factor: Optional[float], @@ -10830,6 +10964,7 @@ def build_context_params( context_params.pooling_type = pooling_type if attention_type is not None: context_params.attention_type = attention_type + context_params.embeddings = embedding if rope_freq_base is not None: context_params.rope_freq_base = rope_freq_base if rope_freq_scale is not None: @@ -10940,14 +11075,34 @@ def close(self) -> None: llama_cpp.llama_backend_free() self.backend_initialized = False - def _meta_value(self, key: str) -> Optional[str]: + @staticmethod + def _model_meta_key_by_index(llama_model: Any, index: int) -> Optional[str]: + capacity = 256 + while True: + buffer = ctypes.create_string_buffer(capacity) + count = int( + llama_cpp.llama_model_meta_key_by_index( + llama_model, + index, + cast(Any, buffer), + capacity, + ) + ) + if count < 0: + return None + if count < capacity: + return buffer.value.decode("utf-8", errors="ignore") + capacity = count + 1 + + @staticmethod + def _model_meta_value(llama_model: Any, key: str) -> Optional[str]: encoded = key.encode("utf-8") capacity = 256 while True: buffer = ctypes.create_string_buffer(capacity) count = int( llama_cpp.llama_model_meta_val_str( - self.llama_model, + llama_model, encoded, cast(Any, buffer), capacity, @@ -10959,6 +11114,50 @@ def _meta_value(self, key: str) -> Optional[str]: return buffer.value.decode("utf-8", errors="ignore") capacity = count + 1 + @staticmethod + def _parse_pooling_type(value: str) -> Optional[int]: + normalized = value.strip().lower() + try: + return int(normalized) + except ValueError: + return { + "none": llama_cpp.LLAMA_POOLING_TYPE_NONE, + "mean": llama_cpp.LLAMA_POOLING_TYPE_MEAN, + "cls": llama_cpp.LLAMA_POOLING_TYPE_CLS, + "last": llama_cpp.LLAMA_POOLING_TYPE_LAST, + "rank": llama_cpp.LLAMA_POOLING_TYPE_RANK, + }.get(normalized) + + @classmethod + def detect_embedding_model(cls, llama_model: Any) -> bool: + for index in range(int(llama_cpp.llama_model_meta_count(llama_model))): + key = cls._model_meta_key_by_index(llama_model, index) + if key is None or not key.endswith(".pooling_type"): + continue + value = cls._model_meta_value(llama_model, key) + if value is None: + continue + pooling_type = cls._parse_pooling_type(value) + return pooling_type in { + llama_cpp.LLAMA_POOLING_TYPE_MEAN, + llama_cpp.LLAMA_POOLING_TYPE_CLS, + llama_cpp.LLAMA_POOLING_TYPE_LAST, + } + return False + + @classmethod + def resolve_embedding_mode( + cls, + llama_model: Any, + embedding: Optional[bool], + ) -> bool: + if embedding is not None: + return embedding + return cls.detect_embedding_model(llama_model) + + def _meta_value(self, key: str) -> Optional[str]: + return self._model_meta_value(self.llama_model, key) + def _build_chat_formatter(self) -> Optional[Jinja2ChatFormatter]: template_text = self.chat_template_override if template_text is None: @@ -11131,6 +11330,10 @@ def clear_batch(self) -> None: self._embedding_batch = None self._embedding_batch_refs = [] + def clear_memory(self) -> None: + if self.mem is not None: + llama_cpp.llama_memory_clear(self.mem, True) + def add_batch_tokens( self, *, @@ -11206,9 +11409,14 @@ def add_batch_embeddings( def decode(self) -> None: batch = self._embedding_batch if self._embedding_batch is not None else self.batch - result = int(llama_cpp.llama_decode(self.ctx, batch)) + if self.embedding and self.has_encoder: + operation = "llama_encode" + result = int(llama_cpp.llama_encode(self.ctx, batch)) + else: + operation = "llama_decode" + result = int(llama_cpp.llama_decode(self.ctx, batch)) if result != 0: - raise RuntimeError(f"llama_decode failed with code {result}") + raise RuntimeError(f"{operation} failed with code {result}") def process_draft_batch(self) -> None: if self.draft_provider is not None: @@ -11242,6 +11450,103 @@ def logits(self, output_index: int) -> np.ndarray: raise RuntimeError(f"missing logits output {output_index}") return np.ctypeslib.as_array(ptr, shape=(self.n_vocab,)).copy() + def embed( + self, + inputs: Sequence[Union[str, List[int]]], + ) -> Tuple[List[List[float]], int]: + if not self.embedding: + raise CompletionRequestValidationError( + "model.embedding must be true to use /v1/embeddings" + ) + pooling_type = int(llama_cpp.llama_pooling_type(self.ctx)) + if pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE: + raise CompletionRequestValidationError( + "/v1/embeddings requires a pooled embedding model; " + "set model.pooling_type to MEAN, CLS, or LAST" + ) + if pooling_type == llama_cpp.LLAMA_POOLING_TYPE_RANK: + raise CompletionRequestValidationError( + "/v1/embeddings does not support reranking pooling" + ) + if len(inputs) > 2048: + raise CompletionRequestValidationError( + "embedding input batch size exceeds 2048" + ) + + embeddings: List[List[float]] = [] + total_tokens = 0 + batch_sizes: List[int] = [] + batch_token_count = 0 + + def decode_embedding_batch() -> None: + nonlocal batch_token_count + if not batch_sizes: + return + self.clear_memory() + self.decode() + self.clear_batch() + for seq_id in range(len(batch_sizes)): + ptr = llama_cpp.llama_get_embeddings_seq( + self.ctx, + llama_cpp.llama_seq_id(seq_id), + ) + if not ptr: + raise RuntimeError(f"missing embedding output for input {seq_id}") + embeddings.append( + np.ctypeslib.as_array(ptr, shape=(self.n_embd_out,)).astype( + float + ).tolist() + ) + batch_sizes.clear() + batch_token_count = 0 + + try: + self.clear_batch() + self.clear_memory() + for input_item in inputs: + tokens = ( + self.tokenize(input_item) + if isinstance(input_item, str) + else list(input_item) + ) + n_tokens = len(tokens) + if n_tokens == 0: + raise CompletionRequestValidationError( + "embedding input must not be empty" + ) + if n_tokens > self.n_ctx_seq: + raise CompletionRequestValidationError( + f"embedding input has {n_tokens} tokens, exceeding n_ctx_seq ({self.n_ctx_seq})" + ) + if n_tokens > self.n_batch: + raise CompletionRequestValidationError( + f"embedding input has {n_tokens} tokens, exceeding n_batch ({self.n_batch})" + ) + if total_tokens + n_tokens > 300_000: + raise CompletionRequestValidationError( + "embedding request exceeds 300000 total tokens" + ) + if ( + batch_token_count + n_tokens > self.n_batch + or len(batch_sizes) >= self.n_seq_max + ): + decode_embedding_batch() + seq_id = len(batch_sizes) + self.add_batch_tokens( + seq_id=seq_id, + start_pos=0, + tokens=tokens, + output_indices=[0] * n_tokens, + ) + batch_sizes.append(n_tokens) + batch_token_count += n_tokens + total_tokens += n_tokens + decode_embedding_batch() + finally: + self.clear_batch() + self.clear_memory() + return embeddings, total_tokens + class SequenceDiskCache(SequenceCache): """Directory-backed cache for serialized llama.cpp sequence state.""" @@ -12065,6 +12370,37 @@ def build_memory_policy(self) -> MemoryPolicy: return PartitionedAttentionMemoryPolicy(self) return UnifiedAttentionMemoryPolicy(self) + def clear_resident_state(self) -> None: + self.model.clear_memory() + self.model.clear_batch() + self.radix_trie = RadixTrie() + self.sequence_history = SequenceHistory() + self.checkpoint_logits.clear() + self.claimed_sequences.clear() + self.free_sequences.clear() + self.unused_sequences = list(range(self.model.n_seq_max - 1, -1, -1)) + for seq_id in range(self.model.n_seq_max): + self.model.truncate_draft_sequence(seq_id, 0) + + def create_embedding( + self, + payload: CreateEmbeddingRequest, + ) -> CreateEmbeddingResponse: + if not self.is_idle(): + raise RuntimeError("embedding requests require an idle scheduler") + self.clear_resident_state() + try: + embeddings, total_tokens = self.model.embed(payload.normalized_input()) + return CreateEmbeddingResponse.from_embeddings( + model=payload.model, + embeddings=embeddings, + total_tokens=total_tokens, + encoding_format=payload.encoding_format, + dimensions=payload.dimensions, + ) + finally: + self.clear_resident_state() + @staticmethod def request_needs_prompt_logits(request: CompletionRequest) -> bool: return request.payload.max_tokens != 0 and request.effective_max_len > len( @@ -14420,6 +14756,39 @@ def run_callback() -> None: raise error_box["error"] return result_box.get("result") + def call_on_idle_scheduler(self, callback: Callable[[], Any]) -> Any: + result_box: Dict[str, Any] = {} + error_box: Dict[str, BaseException] = {} + done = threading.Event() + + def run_callback() -> None: + if not self.scheduler.is_idle(): + with self.condition: + self.commands.appendleft(run_callback) + self.condition.notify_all() + return + try: + result_box["result"] = callback() + except BaseException as exc: # noqa: BLE001 + error_box["error"] = exc + finally: + done.set() + + self.enqueue(run_callback) + done.wait() + if "error" in error_box: + raise error_box["error"] + return result_box.get("result") + + def create_embedding( + self, + payload: CreateEmbeddingRequest, + ) -> CreateEmbeddingResponse: + embedding = self.call_on_idle_scheduler( + lambda: self.scheduler.create_embedding(payload) + ) + return cast(CreateEmbeddingResponse, embedding) + def render_prometheus_metrics(self) -> str: metrics = self.call_on_scheduler(self.scheduler.render_prometheus_metrics) return cast(str, metrics) @@ -14874,6 +15243,17 @@ async def create_completion( # pyright: ignore[reportUnusedFunction] return result return JSONResponse(result.model_dump(mode="json", exclude_none=True)) + @app.post("/v1/embeddings", response_model=CreateEmbeddingResponse) + async def create_embedding( # pyright: ignore[reportUnusedFunction] + body: CreateEmbeddingRequest, + ) -> JSONResponse: + service: CompletionService = app.state.service + try: + embedding = await asyncio.to_thread(service.create_embedding, body) + except CompletionRequestValidationError as exc: + raise bad_request(exc) from exc + return JSONResponse(embedding.model_dump(mode="json", exclude_none=True)) + @app.post("/v1/chat/completions") async def create_chat_completion( # pyright: ignore[reportUnusedFunction] http_request: Request, body: CreateChatCompletionRequest @@ -15250,6 +15630,7 @@ def main() -> None: rope_scaling_type=config.model.rope_scaling_type, pooling_type=config.model.pooling_type, attention_type=config.model.attention_type, + embedding=config.model.embedding, rope_freq_base=config.model.rope_freq_base, rope_freq_scale=config.model.rope_freq_scale, yarn_ext_factor=config.model.yarn_ext_factor, From db66da32d12408cc5bc62c27d09e09ba570dc131 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 15:20:36 -0700 Subject: [PATCH 549/561] feat: update llama.cpp to ggml-org/llama.cpp@9e3b928fd (#2282) --- CHANGELOG.md | 1 + llama_cpp/llama_cpp.py | 7 +++++++ vendor/llama.cpp | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4912e00e45..e20ed73c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@9e3b928fd - feat(example): add OpenAI-compatible embeddings endpoint by @abetlen in #2281 ## [0.3.27] diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 44c25a519f..21f85c81c3 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -949,6 +949,10 @@ class llama_sampler_seq_config(ctypes.Structure): # // ref: https://github.com/ggml-org/llama.cpp/pull/14363 # struct llama_sampler_seq_config * samplers; # size_t n_samplers; +# +# // a source/target/parent context +# // can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts +# struct llama_context * ctx_other; # }; class llama_context_params(ctypes.Structure): """Parameters for llama_context @@ -989,6 +993,7 @@ class llama_context_params(ctypes.Structure): kv_unified (bool): use a unified buffer across the input sequences when computing the attention samplers (ctypes.POINTER(llama_sampler_seq_config)): backend sampler chain configuration n_samplers (int): number of backend sampler chain configurations + ctx_other (llama_context_p): source, target, or parent context """ if TYPE_CHECKING: @@ -1027,6 +1032,7 @@ class llama_context_params(ctypes.Structure): kv_unified: bool samplers: ctypes.POINTER(llama_sampler_seq_config) n_samplers: int + ctx_other: llama_context_p _fields_ = [ ("n_ctx", ctypes.c_uint32), @@ -1064,6 +1070,7 @@ class llama_context_params(ctypes.Structure): ("kv_unified", ctypes.c_bool), ("samplers", ctypes.POINTER(llama_sampler_seq_config)), ("n_samplers", ctypes.c_size_t), + ("ctx_other", llama_context_p_ctypes), ] diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 465b1f0e75..9e3b928fd8 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 465b1f0e75c590426cff3ca998bcd25297071a5b +Subproject commit 9e3b928fd8c9d14dbf15a8768b9fdd7e5c721d66 From fddee272fc78e0e6b74cad8da10eb19347cad909 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 16:19:29 -0700 Subject: [PATCH 550/561] feat(example): align server MTP support with llama.cpp (#2283) * feat(example): align server MTP support with llama.cpp * docs: update changelog for server MTP alignment * fix: remove stale pre-norm extension bindings * fix(example): disable sampled MTP for shared memory contexts --- CHANGELOG.md | 1 + examples/server/README.md | 16 ++ examples/server/server.py | 408 +++++++++++++++++-------------------- llama_cpp/llama_cpp_ext.py | 54 +++-- 4 files changed, 238 insertions(+), 241 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e20ed73c29..b07244ed7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat(example): align server MTP support with llama.cpp by @abetlen in #2283 - feat: update llama.cpp to ggml-org/llama.cpp@9e3b928fd - feat(example): add OpenAI-compatible embeddings endpoint by @abetlen in #2281 diff --git a/examples/server/README.md b/examples/server/README.md index 6a1cc44480..ff04374fc5 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -434,6 +434,22 @@ Use MTP when the loaded model and llama.cpp build expose the required draft stat } ``` +By default `draft-mtp` creates the MTP context from the target model. +Set `draft_model_path` or `draft_model_from_pretrained` when the model uses a separate assistant GGUF. + +```json +{ + "model": { + "draft_model": "draft-mtp", + "draft_model_num_pred_tokens": 2, + "draft_model_from_pretrained": { + "repo_id": "example/gemma-assistant-GGUF", + "filename": "assistant.gguf" + } + } +} +``` + MTP currently applies to text-only requests. ## Disk Sequence Cache diff --git a/examples/server/server.py b/examples/server/server.py index 2530cf4bec..64a16f0bd1 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -36,7 +36,6 @@ import copy import shutil import inspect -import importlib.util import sys import urllib.error import urllib.parse @@ -1262,27 +1261,33 @@ def __init__( self, *, model: "Model", + draft_model: Any, context_params: Any, num_pred_tokens: int, top_k: int, p_min: float, ) -> None: self.target_ctx = model.ctx - self.model = model.llama_model + self.model = draft_model self.n_seq_max = model.n_seq_max self.n_vocab = model.n_vocab - self.n_embd = int(llama_cpp.llama_model_n_embd(self.model)) + self.n_embd = int(llama_cpp.llama_model_n_embd_out(self.model)) + if self.n_embd <= 0: + self.n_embd = int(llama_cpp.llama_model_n_embd(self.model)) + if self.n_embd != model.n_embd: + raise RuntimeError( + "MTP draft model output embedding size must match target model " + f"embedding size ({self.n_embd} != {model.n_embd})" + ) self.num_pred_tokens = max(0, int(num_pred_tokens)) self.top_k = max(1, int(top_k)) self.p_min = max(0.0, min(1.0, float(p_min))) - ( - self.target_hidden_norm_weight, - self.draft_hidden_norm_weight, - self.hidden_norm_epsilon, - ) = self._load_hidden_norm_weights(model.model_path) self.ctx = llama_cpp.llama_init_from_model(self.model, context_params) if self.ctx is None: raise RuntimeError("failed to create MTP draft context") + ctx_other = llama_cpp_ext.llama_get_ctx_other(self.ctx) + self.is_mem_shared = bool(ctx_other and ctx_other == self.target_ctx) + self.sampled_batch_draft = not self.is_mem_shared self.n_batch = int(llama_cpp.llama_n_batch(self.ctx)) mem = llama_cpp.llama_get_memory(self.ctx) if mem is None: @@ -1313,135 +1318,13 @@ def __init__( self.decode_failures_total = 0 self.target_processing_enabled = False self.set_target_processing_enabled(True) - llama_cpp_ext.llama_set_embeddings_pre_norm( + llama_cpp_ext.llama_set_embeddings_nextn( self.ctx, True, True, ) self._init_samplers() - @staticmethod - def _load_gguf_reader() -> Any: - try: - from gguf import GGUFReader # type: ignore[import-not-found] - return GGUFReader - except ImportError: - pass - - gguf_init = ( - Path(__file__).resolve().parents[2] - / "vendor/llama.cpp/gguf-py/gguf/__init__.py" - ) - spec = importlib.util.spec_from_file_location( - "gguf", - gguf_init, - submodule_search_locations=[str(gguf_init.parent)], - ) - if spec is None or spec.loader is None: - raise RuntimeError("MTP requires the gguf Python reader from llama.cpp") - module = importlib.util.module_from_spec(spec) - previous = sys.modules.get("gguf") - # Package-relative imports in gguf-py expect the package to be registered. - sys.modules["gguf"] = module - try: - spec.loader.exec_module(module) - except Exception: - if previous is None: - sys.modules.pop("gguf", None) - else: - sys.modules["gguf"] = previous - raise - return module.GGUFReader - - @staticmethod - def _gguf_field_contents(reader: Any, name: str) -> Any: - field = reader.fields.get(name) - if field is None: - return None - return field.contents() - - def _load_hidden_norm_weights( - self, - model_path: str, - ) -> Tuple[np.ndarray, np.ndarray, float]: - GGUFReader = self._load_gguf_reader() - - target_weight: Optional[np.ndarray] = None - draft_weights: List[np.ndarray] = [] - reader = GGUFReader(model_path) - arch = self._gguf_field_contents(reader, "general.architecture") - if arch not in {"qwen35", "qwen35moe"}: - raise RuntimeError( - "draft-mtp currently supports qwen35/qwen35moe GGUF models, " - f"got {arch!r}" - ) - nextn_layers = self._gguf_field_contents( - reader, - f"{arch}.nextn_predict_layers", - ) - # The current MTP path follows llama.cpp's Qwen3.5 one-nextn-layer graph. - if int(nextn_layers or 0) != 1: - raise RuntimeError( - "draft-mtp currently supports exactly one Qwen3.5 nextn prediction layer" - ) - epsilon = self._gguf_field_contents( - reader, - f"{arch}.attention.layer_norm_rms_epsilon", - ) - if epsilon is None: - raise RuntimeError( - f"MTP requires {arch}.attention.layer_norm_rms_epsilon" - ) - for tensor in reader.tensors: - if tensor.name == "output_norm.weight": - target_weight = np.asarray(tensor.data, dtype=np.float32).copy() - elif tensor.name.endswith(".nextn.shared_head_norm.weight"): - draft_weights.append(np.asarray(tensor.data, dtype=np.float32).copy()) - if target_weight is None: - raise RuntimeError("MTP requires output_norm.weight in GGUF model") - if len(draft_weights) > 1: - raise RuntimeError( - "MTP requires at most one blk.*.nextn.shared_head_norm.weight in GGUF model" - ) - draft_weight = draft_weights[0] if draft_weights else target_weight - if target_weight.shape != (self.n_embd,): - raise RuntimeError( - "MTP target norm weight shape does not match model embedding size " - f"({target_weight.shape} != ({self.n_embd},))" - ) - if draft_weight.shape != (self.n_embd,): - raise RuntimeError( - "MTP draft norm weight shape does not match model embedding size " - f"({draft_weight.shape} != ({self.n_embd},))" - ) - return target_weight, draft_weight, float(epsilon) - - def _normalize_hidden_rows(self, rows: np.ndarray, weight: np.ndarray) -> np.ndarray: - rows = np.asarray(rows, dtype=np.float32) - scale = np.reciprocal( - np.sqrt( - np.mean(np.square(rows), axis=-1, keepdims=True) - + self.hidden_norm_epsilon - ) - ) - return rows * scale * weight.reshape(1, -1) - - def _normalize_target_hidden_rows(self, rows: np.ndarray) -> np.ndarray: - return self._normalize_hidden_rows(rows, self.target_hidden_norm_weight) - - def _normalize_draft_hidden_row( - self, - row: Union[np.ndarray, ctypes.POINTER(ctypes.c_float)], - ) -> np.ndarray: - if isinstance(row, np.ndarray): - row_array = row.reshape(1, -1) - else: - row_array = np.ctypeslib.as_array(row, shape=(self.n_embd,)).reshape(1, -1) - return self._normalize_hidden_rows( - row_array, - self.draft_hidden_norm_weight, - )[0] - def _init_samplers(self) -> None: for seq_id in range(self.n_seq_max): params = llama_cpp.llama_sampler_chain_default_params() @@ -1507,7 +1390,7 @@ def close(self) -> None: def set_target_processing_enabled(self, enabled: bool) -> None: if self.target_processing_enabled == enabled: return - llama_cpp_ext.llama_set_embeddings_pre_norm( + llama_cpp_ext.llama_set_embeddings_nextn( self.target_ctx, enabled, False, @@ -1641,14 +1524,13 @@ def process(self, batch: Any, /) -> None: ): return - h_tgt = llama_cpp_ext.llama_get_embeddings_pre_norm(self.target_ctx) + h_tgt = llama_cpp_ext.llama_get_embeddings_nextn(self.target_ctx) if not h_tgt: - raise RuntimeError("missing target pre-norm embeddings for MTP") + raise RuntimeError("missing target nextn embeddings for MTP") h_tgt_rows = np.ctypeslib.as_array( h_tgt, shape=(n_tokens, self.n_embd), ) - h_tgt_rows = self._normalize_target_hidden_rows(h_tgt_rows) previous_row_by_seq: Dict[int, int] = {} first_pos_by_seq: Dict[int, int] = {} @@ -1708,17 +1590,17 @@ def _process_rows( self.ready[seq_id] and self.ready_pos[seq_id] == first_pos ) aligned_by_seq.setdefault(seq_id, aligned) - mtp_pos = ( - pos - 1 - if previous_row_by_seq.get(seq_id) is None - else int(batch.pos[previous_row_by_seq[seq_id]]) - ) - if aligned and mtp_pos >= 0 and mtp_pos >= self.context_pos[seq_id]: - previous_row = previous_row_by_seq.get(seq_id) + previous_row = previous_row_by_seq.get(seq_id) + if ( + aligned + and not self.is_mem_shared + and pos >= 0 + and pos >= self.context_pos[seq_id] + ): slot = int(self.batch.n_tokens) self._add_batch_token( token=int(batch.token[index]), - pos=mtp_pos, + pos=pos, seq_id=seq_id, logits=False, ) @@ -1726,7 +1608,7 @@ def _process_rows( self._set_batch_embedding_row(slot, self.pending_h[seq_id]) else: self._set_batch_embedding_row(slot, h_tgt_rows[previous_row]) - added_pos_by_seq[seq_id] = mtp_pos + added_pos_by_seq[seq_id] = pos previous_row_by_seq[seq_id] = index target_rows_by_seq.setdefault(seq_id, []).append(index) @@ -1760,15 +1642,15 @@ def draft( n_past = int(input_ids.size) - 1 if self.ready_pos[seq_id] != n_past: return np.array([], dtype=np.intc) - first_pos = n_past - 1 + first_pos = n_past if first_pos < 0: return np.array([], dtype=np.intc) token = int(input_ids[-1]) drafted: List[int] = [] - if self.context_pos[seq_id] > first_pos: + if not self.is_mem_shared and self.context_pos[seq_id] > first_pos: self.truncate(seq_id, first_pos) - if self.context_pos[seq_id] < first_pos: + if not self.is_mem_shared and self.context_pos[seq_id] < first_pos: self.ready[seq_id] = False return np.array([], dtype=np.intc) @@ -1782,9 +1664,11 @@ def draft( ) self._set_batch_embedding_row(0, self.pending_h[seq_id]) if not self._try_decode_batch(): - self.truncate(seq_id, first_pos) + if not self.is_mem_shared: + self.truncate(seq_id, first_pos) return np.array([], dtype=np.intc) - self.context_pos[seq_id] = n_past + if not self.is_mem_shared: + self.context_pos[seq_id] = first_pos + 1 while len(drafted) < n_predict: sampled_token = self._sample_token(seq_id=seq_id) @@ -1794,26 +1678,28 @@ def draft( drafted.append(token) if len(drafted) >= n_predict: break - h_row = llama_cpp_ext.llama_get_embeddings_pre_norm_ith(self.ctx, 0) + h_row = llama_cpp_ext.llama_get_embeddings_nextn_ith(self.ctx, 0) if not h_row: break - h_row = self._normalize_draft_hidden_row(h_row) self._clear_batch() self._add_batch_token( token=token, - pos=first_pos + len(drafted), + pos=first_pos if self.is_mem_shared else first_pos + len(drafted), seq_id=seq_id, logits=True, ) self._set_batch_embedding_row(0, h_row) if not self._try_decode_batch(): break - self.context_pos[seq_id] = first_pos + len(drafted) + 1 + if not self.is_mem_shared: + self.context_pos[seq_id] = first_pos + len(drafted) + 1 if not drafted: - self.truncate(seq_id, n_past) + if not self.is_mem_shared: + self.truncate(seq_id, n_past) return np.array([], dtype=np.intc) - self.truncate(seq_id, n_past) + if not self.is_mem_shared: + self.truncate(seq_id, n_past) return np.asarray(drafted, dtype=np.intc) def draft_many( @@ -1843,12 +1729,12 @@ def draft_many( n_past = int(input_ids.size) - 1 if self.ready_pos[seq_id] != n_past: continue - first_pos = n_past - 1 + first_pos = n_past if first_pos < 0: continue - if self.context_pos[seq_id] > first_pos: + if not self.is_mem_shared and self.context_pos[seq_id] > first_pos: self.truncate(seq_id, first_pos) - if self.context_pos[seq_id] < first_pos: + if not self.is_mem_shared and self.context_pos[seq_id] < first_pos: self.ready[seq_id] = False continue self._reset_sampler(seq_id) @@ -1904,10 +1790,11 @@ def draft_many( ): if sampled_token is None: continue - self.context_pos[representative.seq_id] = max( - self.context_pos[representative.seq_id], - representative.keep_len, - ) + if not self.is_mem_shared: + self.context_pos[representative.seq_id] = max( + self.context_pos[representative.seq_id], + representative.first_pos + 1, + ) for state in grouped[representative.cache_key]: state.drafted.append(sampled_token) active = [] @@ -1917,7 +1804,11 @@ def draft_many( for row, state in enumerate(active): self._add_batch_token( token=state.token, - pos=state.first_pos + len(state.drafted), + pos=( + state.first_pos + if self.is_mem_shared + else state.first_pos + len(state.drafted) + ), seq_id=state.seq_id, logits=True, ) @@ -1932,14 +1823,19 @@ def draft_many( for row, state in enumerate(active) ] for row, (state, sampled_token) in enumerate(zip(active, sampled_tokens)): - decoded_pos = state.first_pos + len(state.drafted) - self.context_pos[state.seq_id] = max( - self.context_pos[state.seq_id], - decoded_pos + 1, + decoded_pos = ( + state.first_pos + if self.is_mem_shared + else state.first_pos + len(state.drafted) ) + if not self.is_mem_shared: + self.context_pos[state.seq_id] = max( + self.context_pos[state.seq_id], + decoded_pos + 1, + ) if sampled_token is None: continue - h_row_ptr = llama_cpp_ext.llama_get_embeddings_pre_norm_ith( + h_row_ptr = llama_cpp_ext.llama_get_embeddings_nextn_ith( self.ctx, row ) state.drafted.append(sampled_token) @@ -1947,14 +1843,17 @@ def draft_many( continue if not h_row_ptr: continue - h_row = self._normalize_draft_hidden_row(h_row_ptr) state.token = sampled_token - state.embedding = h_row + state.embedding = np.ctypeslib.as_array( + h_row_ptr, + shape=(self.n_embd,), + ).copy() next_active.append(state) active = next_active finally: - for state in touched: - self.truncate(state.seq_id, state.keep_len) + if not self.is_mem_shared: + for state in touched: + self.truncate(state.seq_id, state.keep_len) for state in touched: if state.drafted: @@ -2036,7 +1935,7 @@ def _build_sampled_batch_plan( pending_token = update.pending_token if ( pending_token is None - or start_pos <= 0 + or start_pos < 0 or target_count <= 0 or target_count > len(tokens) or target_count > len(row_indices) @@ -2046,9 +1945,7 @@ def _build_sampled_batch_plan( continue for target_index in range(sample_index + 1): - mtp_pos = start_pos + target_index - 1 - if mtp_pos < 0: - continue + mtp_pos = start_pos + target_index source_row = ( None if target_index == 0 @@ -2063,7 +1960,7 @@ def _build_sampled_batch_plan( ) ) - actual_pos = start_pos + sample_index + actual_pos = start_pos + sample_index + 1 pending_rows.append( self.SampledPendingRow( update_index=update_index, @@ -2090,6 +1987,8 @@ def _decode_sampled_context_rows( self._clear_batch() decoded_context_rows: List[Tuple[int, int]] = [] for row in context_rows: + if self.is_mem_shared: + continue if row.draft_pos < self.context_pos[row.seq_id]: continue if row.source_row is None: @@ -2124,7 +2023,10 @@ def _decode_sampled_pending_rows( self._clear_batch() for pending_index, row in enumerate(pending_rows): - if row.draft_pos < self.context_pos[row.seq_id]: + if ( + not self.is_mem_shared + and row.draft_pos < self.context_pos[row.seq_id] + ): continue is_sample_pending = ( pending_index @@ -2144,17 +2046,18 @@ def _decode_sampled_pending_rows( update_index=row.update_index, seq_id=row.seq_id, output_index=slot, - keep_len=row.draft_pos + 1, - ready_pos=row.draft_pos + 1, + keep_len=row.draft_pos, + ready_pos=row.draft_pos, ) ) self._decode_batch() - for row in pending_rows: - self.context_pos[row.seq_id] = max( - self.context_pos[row.seq_id], - row.draft_pos + 1, - ) + if not self.is_mem_shared: + for row in pending_rows: + self.context_pos[row.seq_id] = max( + self.context_pos[row.seq_id], + row.draft_pos + 1, + ) return sampled_outputs @@ -2184,7 +2087,7 @@ def _start_sampled_draft_states( if n_predict <= 0: continue if n_predict > 1: - h_row = llama_cpp_ext.llama_get_embeddings_pre_norm_ith( + h_row = llama_cpp_ext.llama_get_embeddings_nextn_ith( self.ctx, output.output_index ) if h_row: @@ -2193,11 +2096,18 @@ def _start_sampled_draft_states( update_index=output.update_index, seq_id=seq_id, keep_len=output.keep_len, - pos=output.keep_len, + pos=( + output.keep_len + if self.is_mem_shared + else output.keep_len + 1 + ), token=sampled_token, drafted=[sampled_token], n_predict=n_predict, - embedding=self._normalize_draft_hidden_row(h_row), + embedding=np.ctypeslib.as_array( + h_row, + shape=(self.n_embd,), + ).copy(), ) ) results[output.update_index] = np.asarray([sampled_token], dtype=np.intc) @@ -2235,31 +2145,36 @@ def _extend_sampled_draft_states( for batch_row, (state, sampled_token) in enumerate( zip(active, sampled_tokens) ): - self.context_pos[state.seq_id] = max( - self.context_pos[state.seq_id], - state.pos + 1, - ) + if not self.is_mem_shared: + self.context_pos[state.seq_id] = max( + self.context_pos[state.seq_id], + state.pos + 1, + ) if sampled_token is None: continue state.drafted.append(sampled_token) if len(state.drafted) >= state.n_predict: continue - h_row = llama_cpp_ext.llama_get_embeddings_pre_norm_ith( + h_row = llama_cpp_ext.llama_get_embeddings_nextn_ith( self.ctx, batch_row ) if not h_row: continue - h_row = self._normalize_draft_hidden_row(h_row) state.token = sampled_token - state.embedding = h_row - state.pos += 1 + state.embedding = np.ctypeslib.as_array( + h_row, + shape=(self.n_embd,), + ).copy() + if not self.is_mem_shared: + state.pos += 1 next_active.append(state) active = next_active finally: for state in touched: cleanup_keep_len_by_seq[state.seq_id] = state.keep_len - for seq_id, keep_len in cleanup_keep_len_by_seq.items(): - self._truncate_memory(seq_id, keep_len) + if not self.is_mem_shared: + for seq_id, keep_len in cleanup_keep_len_by_seq.items(): + self._truncate_memory(seq_id, keep_len) for state in touched: if state.drafted: @@ -2276,9 +2191,9 @@ def process_sampled_batch( results = [np.array([], dtype=np.intc) for _ in updates] if self.num_pred_tokens <= 0 or not updates: return results - h_tgt = llama_cpp_ext.llama_get_embeddings_pre_norm(self.target_ctx) + h_tgt = llama_cpp_ext.llama_get_embeddings_nextn(self.target_ctx) if not h_tgt: - raise RuntimeError("missing target pre-norm embeddings for MTP") + raise RuntimeError("missing target nextn embeddings for MTP") n_target_rows = max( ( max(update.row_indices) + 1 @@ -2290,7 +2205,6 @@ def process_sampled_batch( if n_target_rows <= 0: return results h_tgt_rows = np.ctypeslib.as_array(h_tgt, shape=(n_target_rows, self.n_embd)) - h_tgt_rows = self._normalize_target_hidden_rows(h_tgt_rows) plan = self._build_sampled_batch_plan(updates) if not plan.context_rows and not plan.pending_rows: @@ -2313,8 +2227,9 @@ def process_sampled_batch( self.ready_pos[output.seq_id] = output.ready_pos cleanup_keep_len_by_seq[output.seq_id] = output.keep_len - for seq_id, keep_len in cleanup_keep_len_by_seq.items(): - self._truncate_memory(seq_id, keep_len) + if not self.is_mem_shared: + for seq_id, keep_len in cleanup_keep_len_by_seq.items(): + self._truncate_memory(seq_id, keep_len) if sampled_outputs: active = self._start_sampled_draft_states( @@ -2344,6 +2259,9 @@ def accept(self, seq_id: int, accepted_draft_tokens: int) -> None: def _truncate_memory(self, seq_id: int, keep_len: int) -> None: if seq_id < 0 or seq_id >= self.n_seq_max: return + if self.is_mem_shared: + self.context_pos[seq_id] = min(self.context_pos[seq_id], keep_len) + return if not llama_cpp.llama_memory_seq_rm( self.mem, seq_id, @@ -2386,13 +2304,14 @@ def copy_sequence( or dest_seq_id >= self.n_seq_max ): return - llama_cpp.llama_memory_seq_cp( - self.mem, - source_seq_id, - dest_seq_id, - p0, - p1, - ) + if not self.is_mem_shared: + llama_cpp.llama_memory_seq_cp( + self.mem, + source_seq_id, + dest_seq_id, + p0, + p1, + ) source_ready_pos = self.ready_pos[source_seq_id] copied_full_ready_state = p1 < 0 or p1 == source_ready_pos if self.ready[source_seq_id] and copied_full_ready_state: @@ -3354,6 +3273,10 @@ class ModelOptions(BaseModel): max_output_tokens: Optional[int] = Field(default=None, ge=0) kv_unified: bool = True draft_model: Optional[Literal["prompt-lookup-decoding", "draft-mtp"]] = None + draft_model_path: Optional[str] = None + draft_model_from_pretrained: Optional[ + "ConfigFile.FromPretrainedOptions" + ] = None draft_model_num_pred_tokens: int = 16 draft_model_max_ngram_size: int = 2 draft_model_top_k: int = Field(default=1, ge=1) @@ -3368,8 +3291,21 @@ class ModelOptions(BaseModel): def validate_source(self) -> "ConfigFile.ModelOptions": if (self.path is None) == (self.from_pretrained is None): raise ValueError("exactly one of model.path or model.from_pretrained is required") + if ( + self.draft_model_path is not None + and self.draft_model_from_pretrained is not None + ): + raise ValueError( + "model.draft_model_path and model.draft_model_from_pretrained " + "are mutually exclusive" + ) return self + def resolve_draft_model_path(self) -> Optional[str]: + if self.draft_model_from_pretrained is not None: + return self.draft_model_from_pretrained.resolve_model_path() + return self.draft_model_path + @field_validator("chat_template", mode="before") @classmethod def normalize_chat_template(cls, value: Any) -> Any: @@ -10549,6 +10485,7 @@ def __init__( max_seq_len: Optional[int] = None, max_output_tokens: Optional[int] = None, draft_model: Optional[str] = None, + draft_model_path: Optional[str] = None, draft_model_num_pred_tokens: int = 16, draft_model_max_ngram_size: int = 2, draft_model_top_k: int = 1, @@ -10573,6 +10510,7 @@ def __init__( self._lora_adapters: List[Any] = [] self._lora_adapter_array: Optional[Any] = None self._lora_scales_array: Optional[Any] = None + self.draft_llama_model: Optional[Any] = None model_params, self._c_tensor_split, self._kv_overrides_array = ( self.build_model_params( n_gpu_layers=n_gpu_layers, @@ -10630,11 +10568,6 @@ def __init__( "speculative decoding is only supported for attention models" ) n_ctx_train = int(llama_cpp.llama_model_n_ctx_train(llama_model)) - target_n_rs_seq = ( - max(1, draft_model_num_pred_tokens) - if normalized_draft_model == "draft-mtp" - else None - ) context_params = self.build_context_params( n_ctx=n_ctx if n_ctx is not None else n_ctx_train, @@ -10662,7 +10595,7 @@ def __init__( type_k=type_k, type_v=type_v, kv_unified=kv_unified, - n_rs_seq=target_n_rs_seq, + n_rs_seq=None, ctx_type=None, ) ctx = llama_cpp.llama_init_from_model(llama_model, context_params) @@ -10692,11 +10625,6 @@ def __init__( "MTP requires runtime n_batch to fit the pending token plus draft tokens " f"(required {required_mtp_batch}, got {self.n_batch})" ) - if target_n_rs_seq is not None and self.n_rs_seq < target_n_rs_seq: - raise RuntimeError( - "MTP requires retained recurrent-state slots for rollback " - f"(required {target_n_rs_seq}, got {self.n_rs_seq})" - ) self.n_ctx_train = n_ctx_train self.n_vocab = int(llama_cpp.llama_vocab_n_tokens(self.vocab)) self.n_embd = int(llama_cpp.llama_model_n_embd(self.llama_model)) @@ -10747,6 +10675,22 @@ def __init__( num_pred_tokens=draft_model_num_pred_tokens, ) elif normalized_draft_model == "draft-mtp": + draft_llama_model = self.llama_model + if draft_model_path is not None: + draft_llama_model = llama_cpp.llama_model_load_from_file( + draft_model_path.encode("utf-8"), + model_params, + ) + if draft_llama_model is None: + llama_cpp.llama_batch_free(self.batch) + llama_cpp.llama_free(self.ctx) + self._free_lora_adapters() + llama_cpp.llama_model_free(self.llama_model) + if self.backend_initialized: + llama_cpp.llama_backend_free() + self.backend_initialized = False + raise RuntimeError(f"failed to load MTP draft model: {draft_model_path}") + self.draft_llama_model = draft_llama_model if self.n_ubatch < self.n_seq_max: mtp_n_batch = self.n_batch else: @@ -10792,13 +10736,15 @@ def __init__( type_k=type_k, type_v=type_v, kv_unified=kv_unified, - n_rs_seq=target_n_rs_seq, + n_rs_seq=0, ctx_type=llama_cpp.LLAMA_CONTEXT_TYPE_MTP, n_outputs_max=min(mtp_n_batch, self.n_seq_max), + ctx_other=self.ctx, ) try: self.draft_provider = MTPDraftProvider( model=self, + draft_model=draft_llama_model, context_params=mtp_context_params, num_pred_tokens=draft_model_num_pred_tokens, top_k=draft_model_top_k, @@ -10808,6 +10754,9 @@ def __init__( llama_cpp.llama_batch_free(self.batch) llama_cpp.llama_free(self.ctx) self._free_lora_adapters() + if self.draft_llama_model is not None: + llama_cpp.llama_model_free(self.draft_llama_model) + self.draft_llama_model = None llama_cpp.llama_model_free(self.llama_model) if self.backend_initialized: llama_cpp.llama_backend_free() @@ -10818,7 +10767,10 @@ def __init__( try: self._load_lora_adapters(self.loras) self._apply_lora_adapters(self.ctx, "target") - if isinstance(self.draft_provider, MTPDraftProvider): + if ( + isinstance(self.draft_provider, MTPDraftProvider) + and self.draft_llama_model is None + ): self._apply_lora_adapters(self.draft_provider.ctx, "MTP draft") except BaseException: if self.draft_provider is not None: @@ -10826,6 +10778,9 @@ def __init__( llama_cpp.llama_batch_free(self.batch) llama_cpp.llama_free(self.ctx) self._free_lora_adapters() + if self.draft_llama_model is not None: + llama_cpp.llama_model_free(self.draft_llama_model) + self.draft_llama_model = None llama_cpp.llama_model_free(self.llama_model) if self.backend_initialized: llama_cpp.llama_backend_free() @@ -10938,6 +10893,7 @@ def build_context_params( n_rs_seq: Optional[int] = None, ctx_type: Optional[int] = None, n_outputs_max: Optional[int] = None, + ctx_other: Optional[Any] = None, ) -> Any: context_params = llama_cpp.llama_context_default_params() if n_ctx is not None: @@ -10958,6 +10914,8 @@ def build_context_params( context_params.ctx_type = ctx_type if n_outputs_max is not None: context_params.n_outputs_max = n_outputs_max + if ctx_other is not None: + context_params.ctx_other = ctx_other if rope_scaling_type is not None: context_params.rope_scaling_type = rope_scaling_type if pooling_type is not None: @@ -11070,6 +11028,9 @@ def close(self) -> None: llama_cpp.llama_batch_free(self.batch) llama_cpp.llama_free(self.ctx) self._free_lora_adapters() + if self.draft_llama_model is not None: + llama_cpp.llama_model_free(self.draft_llama_model) + self.draft_llama_model = None llama_cpp.llama_model_free(self.llama_model) if self.backend_initialized: llama_cpp.llama_backend_free() @@ -15649,6 +15610,7 @@ def main() -> None: max_seq_len=config.model.max_seq_len, max_output_tokens=config.model.max_output_tokens, draft_model=config.model.draft_model, + draft_model_path=config.model.resolve_draft_model_path(), draft_model_num_pred_tokens=config.model.draft_model_num_pred_tokens, draft_model_max_ngram_size=config.model.draft_model_max_ngram_size, draft_model_top_k=config.model.draft_model_top_k, diff --git a/llama_cpp/llama_cpp_ext.py b/llama_cpp/llama_cpp_ext.py index f6ab9197b8..284811086a 100644 --- a/llama_cpp/llama_cpp_ext.py +++ b/llama_cpp/llama_cpp_ext.py @@ -42,58 +42,76 @@ def decorator(f): return decorator -# LLAMA_API void llama_set_embeddings_pre_norm(struct llama_context * ctx, bool value, bool masked); +# LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); @_ctypes_function_from_names( ( - "llama_set_embeddings_pre_norm", - "_Z29llama_set_embeddings_pre_normP13llama_contextbb", - "?llama_set_embeddings_pre_norm@@YAXPEAUllama_context@@_N1@Z", + "llama_set_embeddings_nextn", + "_Z26llama_set_embeddings_nextnP13llama_contextbb", + "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z", ), [llama_cpp.llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], None, ) -def llama_set_embeddings_pre_norm( +def llama_set_embeddings_nextn( ctx: llama_cpp.llama_context_p, value: bool, masked: bool, /, ): - """Set whether the context outputs pre-norm embeddings or not.""" + """Set whether the context outputs nextn embeddings or not.""" ... -# LLAMA_API float * llama_get_embeddings_pre_norm(struct llama_context * ctx); +# LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); @_ctypes_function_from_names( ( - "llama_get_embeddings_pre_norm", - "_Z29llama_get_embeddings_pre_normP13llama_context", - "?llama_get_embeddings_pre_norm@@YAPEAMPEAUllama_context@@@Z", + "llama_get_embeddings_nextn", + "_Z26llama_get_embeddings_nextnP13llama_context", + "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z", ), [llama_cpp.llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float), ) -def llama_get_embeddings_pre_norm( +def llama_get_embeddings_nextn( ctx: llama_cpp.llama_context_p, /, ): - """Get the pre-norm embeddings from the last evaluation.""" + """Get the nextn embeddings from the last evaluation.""" ... -# LLAMA_API float * llama_get_embeddings_pre_norm_ith(struct llama_context * ctx, int32_t i); +# LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); @_ctypes_function_from_names( ( - "llama_get_embeddings_pre_norm_ith", - "_Z33llama_get_embeddings_pre_norm_ithP13llama_contexti", - "?llama_get_embeddings_pre_norm_ith@@YAPEAMPEAUllama_context@@H@Z", + "llama_get_embeddings_nextn_ith", + "_Z30llama_get_embeddings_nextn_ithP13llama_contexti", + "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z", ), [llama_cpp.llama_context_p_ctypes, ctypes.c_int32], ctypes.POINTER(ctypes.c_float), ) -def llama_get_embeddings_pre_norm_ith( +def llama_get_embeddings_nextn_ith( ctx: llama_cpp.llama_context_p, i: Union[ctypes.c_int32, int], /, ): - """Get the pre-norm embeddings for the ith output row from the last evaluation.""" + """Get the nextn embeddings for the ith output row from the last evaluation.""" + ... + + +# LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); +@_ctypes_function_from_names( + ( + "llama_get_ctx_other", + "_Z19llama_get_ctx_otherP13llama_context", + "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z", + ), + [llama_cpp.llama_context_p_ctypes], + llama_cpp.llama_context_p_ctypes, +) +def llama_get_ctx_other( + ctx: llama_cpp.llama_context_p, + /, +): + """Get the context linked through llama_context_params.ctx_other.""" ... From 8e470ac585a2e4363e0c089c479015ed8470ecb1 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 16:49:11 -0700 Subject: [PATCH 551/561] chore: bump version to 0.3.28 (#2284) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b07244ed7b..b83ccc83bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.28] + - feat(example): align server MTP support with llama.cpp by @abetlen in #2283 - feat: update llama.cpp to ggml-org/llama.cpp@9e3b928fd - feat(example): add OpenAI-compatible embeddings endpoint by @abetlen in #2281 diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 369f24ca88..13668893fe 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.27" +__version__ = "0.3.28" From a72325b0d5be7592afe87f01571e913e04a8e394 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 18:48:40 -0700 Subject: [PATCH 552/561] fix(example): avoid duplicate streamed response deltas (#2285) --- examples/server/server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/server/server.py b/examples/server/server.py index 64a16f0bd1..ec83fe6e3c 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -8695,7 +8695,7 @@ def _ensure_reasoning_stream_item( state, "response.output_item.added", output_index=item_state.output_index, - item=item, + item=copy.deepcopy(item), ), self._response_event( state, @@ -8703,7 +8703,7 @@ def _ensure_reasoning_stream_item( item_id=cast(str, item["id"]), output_index=item_state.output_index, content_index=0, - part=part, + part=copy.deepcopy(part), ), ], item_state @@ -8727,7 +8727,7 @@ def _ensure_message_stream_item( state, "response.output_item.added", output_index=item_state.output_index, - item=item, + item=copy.deepcopy(item), ), self._response_event( state, @@ -8735,7 +8735,7 @@ def _ensure_message_stream_item( item_id=cast(str, item["id"]), output_index=item_state.output_index, content_index=0, - part=part, + part=copy.deepcopy(part), ), ], item_state @@ -8777,7 +8777,7 @@ def _ensure_tool_stream_item( state, "response.output_item.added", output_index=item_state.output_index, - item=item, + item=copy.deepcopy(item), ) ], item_state From 411e0f40e767b2ac3b1155be73916ea94c2ddfc6 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 20:29:46 -0700 Subject: [PATCH 553/561] fix(example): derive streaming response parser boundaries from schema (#2287) --- examples/server/server.py | 329 +++++++++++++++++++++++++++++--------- 1 file changed, 249 insertions(+), 80 deletions(-) diff --git a/examples/server/server.py b/examples/server/server.py index ec83fe6e3c..28fc8f4eb3 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -4344,6 +4344,11 @@ def capture(match: re.Match[str]) -> str: @staticmethod def _regex_literal_prefix(pattern: str) -> str: + literal, _ = ResponseParser._regex_literal_prefix_and_remainder(pattern) + return literal + + @staticmethod + def _regex_literal_prefix_and_remainder(pattern: str) -> Tuple[str, str]: literal: List[str] = [] index = 0 while index < len(pattern): @@ -4365,7 +4370,181 @@ def _regex_literal_prefix(pattern: str) -> str: break literal.append(char) index += 1 - return "".join(literal) + return "".join(literal), pattern[index:] + + @staticmethod + def _find_regex_group_end(pattern: str, start: int) -> int: + depth = 0 + escaped = False + in_character_class = False + for index in range(start, len(pattern)): + char = pattern[index] + if escaped: + escaped = False + continue + if char == "\\": + escaped = True + continue + if char == "[": + in_character_class = True + continue + if char == "]" and in_character_class: + in_character_class = False + continue + if in_character_class: + continue + if char == "(": + depth += 1 + continue + if char == ")": + depth -= 1 + if depth == 0: + return index + return -1 + + @classmethod + def _consume_optional_literal_prefix( + cls, + pattern: str, + ) -> Optional[Tuple[str, str]]: + if not pattern.startswith("(?:"): + return None + group_end = cls._find_regex_group_end(pattern, 0) + if group_end < 0 or group_end + 1 >= len(pattern) or pattern[group_end + 1] != "?": + return None + literal, remainder = cls._regex_literal_prefix_and_remainder(pattern[3:group_end]) + if not literal or remainder: + return None + return literal, pattern[group_end + 2 :] + + @staticmethod + def _split_regex_alternatives(pattern: str) -> List[str]: + alternatives: List[str] = [] + start = 0 + depth = 0 + escaped = False + in_character_class = False + for index, char in enumerate(pattern): + if escaped: + escaped = False + continue + if char == "\\": + escaped = True + continue + if char == "[": + in_character_class = True + continue + if char == "]" and in_character_class: + in_character_class = False + continue + if in_character_class: + continue + if char == "(": + depth += 1 + continue + if char == ")": + depth -= 1 + continue + if char == "|" and depth == 0: + alternatives.append(pattern[start:index]) + start = index + 1 + alternatives.append(pattern[start:]) + return alternatives + + @classmethod + def _regex_lookahead_literal_specs(cls, pattern: str) -> List[Tuple[str, bool]]: + if not pattern.startswith("(?="): + return [] + group_end = cls._find_regex_group_end(pattern, 0) + if group_end < 0: + return [] + literals: List[Tuple[str, bool]] = [] + for alternative in cls._split_regex_alternatives(pattern[3:group_end]): + strip_leading_whitespace = False + while alternative.startswith(r"\s*"): + strip_leading_whitespace = True + alternative = alternative[3:] + if alternative == "$": + continue + if alternative.endswith("$"): + alternative = alternative[:-1] + literal, _ = cls._regex_literal_prefix_and_remainder(alternative) + if literal: + literals.append((literal, strip_leading_whitespace)) + return literals + + @classmethod + def _regex_capture_parts( + cls, + pattern: str, + ) -> Optional[Tuple[str, str]]: + normalized = pattern.lstrip("^") + captures = [ + (index, token) + for token in ("(.*?)", "(.*)") + if (index := normalized.find(token)) >= 0 + ] + if not captures: + return None + capture_index, capture_token = min(captures, key=lambda item: item[0]) + return normalized[:capture_index], normalized[capture_index + len(capture_token) :] + + @classmethod + def _regex_capture_end_literal_specs(cls, pattern: str) -> List[Tuple[str, bool]]: + capture_parts = cls._regex_capture_parts(pattern) + if capture_parts is None: + return [] + _, suffix_pattern = capture_parts + literal_specs = cls._regex_lookahead_literal_specs(suffix_pattern) + if literal_specs: + return literal_specs + literal, _ = cls._regex_literal_prefix_and_remainder(suffix_pattern) + return [(literal, False)] if literal else [] + + @classmethod + def _regex_capture_end_literals(cls, pattern: str) -> List[str]: + return [literal for literal, _ in cls._regex_capture_end_literal_specs(pattern)] + + @classmethod + def _regex_leading_capture( + cls, + *, + field_name: str, + field_regex: str, + content_regex: Optional[str], + ) -> Optional[Dict[str, Any]]: + capture_parts = cls._regex_capture_parts(field_regex) + if capture_parts is None: + return None + prefix_pattern, _ = capture_parts + prefix_pattern = prefix_pattern.lstrip("^") + optional_prefix = cls._consume_optional_literal_prefix(prefix_pattern) + if optional_prefix is not None: + prefix_pattern = optional_prefix[1] + implicit_at_start = False + optional_capture_start = cls._consume_optional_literal_prefix(prefix_pattern) + if optional_capture_start is not None: + capture_start, prefix_pattern = optional_capture_start + implicit_at_start = True + else: + capture_start, prefix_pattern = cls._regex_literal_prefix_and_remainder(prefix_pattern) + if not capture_start or prefix_pattern: + return None + end_literals = cls._regex_capture_end_literals(field_regex) + if not end_literals: + return None + capture_end = end_literals[0] + strip_after = False + if isinstance(content_regex, str): + escaped_end = re.escape(capture_end) + strip_after = bool(re.search(escaped_end + r"\\s\*", content_regex)) + return { + "field": field_name, + "start": capture_start, + "end": capture_end, + "strip_after": strip_after, + "implicit_at_start": implicit_at_start, + } @staticmethod def _literal_suffix_prefix_length(text: str, literal: str) -> int: @@ -4702,7 +4881,14 @@ def _compile_tagged_message_plan( return None iterator = cls._compile_iterator_pattern(iterator_pattern) if iterator is None: - return None + iterator_capture = cls._compile_iterator_block_pattern(iterator_pattern) + if ( + not isinstance(iterator_capture, dict) + or not iterator_capture["start"] + or iterator_capture["allow_eof"] + ): + return None + iterator = (iterator_capture["start"], iterator_capture["end"]) items_schema = tool_calls_schema.get("items") if not isinstance(items_schema, dict): return None @@ -4715,10 +4901,11 @@ def _compile_tagged_message_plan( if isinstance(content_schema, dict) else None ) - object_regex = schema.get("x-regex") if isinstance(schema.get("x-regex"), str) else None assistant_prefix: Optional[str] = None - if isinstance(content_regex, str) and r"<\|im_start\|>assistant\n" in content_regex: - assistant_prefix = "<|im_start|>assistant\n" + if isinstance(content_regex, str): + optional_prefix = cls._consume_optional_literal_prefix(content_regex.lstrip("^")) + if optional_prefix is not None: + assistant_prefix = optional_prefix[0] leading_capture: Optional[Dict[str, Any]] = None for field_name, value_schema in properties.items(): if not isinstance(value_schema, dict): @@ -4726,43 +4913,31 @@ def _compile_tagged_message_plan( field_regex = value_schema.get("x-regex") if not isinstance(field_regex, str): continue - if "<think>\\n" in field_regex and "</think>" in field_regex: - leading_capture = { - "field": field_name, - "start": "<think>\n", - "end": "</think>", - "strip_after": True, - "implicit_at_start": "(?:<think>\\n)?" in field_regex, - } + if field_name == "content": + continue + capture = cls._regex_leading_capture( + field_name=field_name, + field_regex=field_regex, + content_regex=content_regex, + ) + if capture is not None: + leading_capture = capture break - if leading_capture is None and isinstance(object_regex, str): - if ( - "(?P<thinking>" in object_regex - and r"<\|channel\>thought\n" in object_regex - and r"\<channel\|\>" in object_regex - ): - leading_capture = { - "field": "thinking", - "start": "<|channel>thought\n", - "end": "<channel|>", - "strip_after": False, - "implicit_at_start": False, - } end_markers: List[str] = [] + content_end_marker_specs: List[Tuple[str, bool]] = [] iterator_start, iterator_end = iterator if "content" in properties: end_markers.append(iterator_start) - if isinstance(content_regex, str) and r"<\|im_end\|>" in content_regex: - end_markers.append("<|im_end|>") - if isinstance(object_regex, str) and r"<turn\|>" in object_regex: - end_markers.append("<turn|>") + if isinstance(content_regex, str): + content_end_marker_specs = cls._regex_capture_end_literal_specs(content_regex) + end_markers.extend(literal for literal, _ in content_end_marker_specs) if not end_markers and iterator_start: end_markers.append(iterator_start) - trim_before_iterator = ( - isinstance(content_regex, str) - and r"\s*<tool_call>\n" in content_regex + deduped_end_markers = tuple(dict.fromkeys(end_markers)) + trim_before_iterator = any( + literal == iterator_start and strip_leading_whitespace + for literal, strip_leading_whitespace in content_end_marker_specs ) - end_marker_tuple = tuple(end_markers) direct_deltas = item_plan["kind"] == "tagged-parameters" direct_init = ( ( @@ -4773,8 +4948,8 @@ def _compile_tagged_message_plan( bool(leading_capture.get("strip_after")) if leading_capture is not None else False, bool(leading_capture.get("implicit_at_start")) if leading_capture is not None else False, trim_before_iterator, - end_marker_tuple, - tuple(marker for marker in end_marker_tuple if marker != iterator_start), + deduped_end_markers, + tuple(marker for marker in deduped_end_markers if marker != iterator_start), iterator_start, iterator_end, item_plan["function_start"], @@ -4793,9 +4968,9 @@ def _compile_tagged_message_plan( "assistant_prefix": assistant_prefix, "leading_capture": leading_capture, "content_field": "content" if "content" in properties else None, - "content_end_markers": end_marker_tuple, + "content_end_markers": deduped_end_markers, "trim_before_iterator": trim_before_iterator, - "stop_markers": tuple(marker for marker in end_marker_tuple if marker != iterator_start), + "stop_markers": tuple(marker for marker in deduped_end_markers if marker != iterator_start), "direct_init": direct_init, "iterator": { "start": iterator_start, @@ -7192,19 +7367,17 @@ def consume_completion_chunk( logprobs=logprobs, leading_delta=role_delta, ) - if self._stream_state_complete(): - return self._chunk_payloads( - chunk_id=chunk_id, - created=created, - model=model, - deltas=stream_deltas, - finish_reason=( - "tool_calls" if self._direct.saw_tool_calls else finish_reason - ), - logprobs=logprobs, - leading_delta=role_delta, - ) - self._stream_failed = True + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=stream_deltas, + finish_reason=( + "tool_calls" if self._direct.saw_tool_calls else finish_reason + ), + logprobs=logprobs, + leading_delta=role_delta, + ) elif self._stream_plan["kind"] == "segment-message": if role_delta is not None: stream_deltas = [role_delta, *stream_deltas] @@ -7219,18 +7392,16 @@ def consume_completion_chunk( finish_reason=None, logprobs=logprobs, ) - if self._stream_state_complete(): - return self._chunk_payloads( - chunk_id=chunk_id, - created=created, - model=model, - deltas=stream_deltas, - finish_reason=( - "tool_calls" if self._message.get("tool_calls") else finish_reason - ), - logprobs=logprobs, - ) - self._stream_failed = True + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=stream_deltas, + finish_reason=( + "tool_calls" if self._message.get("tool_calls") else finish_reason + ), + logprobs=logprobs, + ) else: previous_message = self._message partial_deltas: List[Dict[str, Any]] = [] @@ -7238,7 +7409,7 @@ def consume_completion_chunk( parsed = cast(Dict[str, Any], self._stream_state.parsed) message = self._parsed_chat_message( parsed=parsed, - partial=finish_reason is None, + partial=finish_reason is None or not self._stream_state_complete(), ) if finish_reason is None: if role_delta is not None: @@ -7253,22 +7424,20 @@ def consume_completion_chunk( finish_reason=None, logprobs=logprobs, ) - if self._stream_state_complete(): - if role_delta is not None: - partial_deltas.append(role_delta) - partial_deltas.extend(self._message_deltas(previous_message, message)) - self._message = message - return self._chunk_payloads( - chunk_id=chunk_id, - created=created, - model=model, - deltas=partial_deltas, - finish_reason=( - "tool_calls" if message.get("tool_calls") else finish_reason - ), - logprobs=logprobs, - ) - self._stream_failed = True + if role_delta is not None: + partial_deltas.append(role_delta) + partial_deltas.extend(self._message_deltas(previous_message, message)) + self._message = message + return self._chunk_payloads( + chunk_id=chunk_id, + created=created, + model=model, + deltas=partial_deltas, + finish_reason=( + "tool_calls" if message.get("tool_calls") else finish_reason + ), + logprobs=logprobs, + ) else: self._stream_failed = True From 7eb494d83ef5fe67f348a0b1a14627fa8da50e36 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 22:35:22 -0700 Subject: [PATCH 554/561] fix(ci): repair Linux accelerator wheels (#2286) --- .github/workflows/build-wheels-cuda.yaml | 53 +++++++++++++++-- .github/workflows/build-wheels-rocm.yaml | 69 ++++++++++++++++++---- .github/workflows/build-wheels-vulkan.yaml | 57 ++++++++++-------- CHANGELOG.md | 2 + 4 files changed, 140 insertions(+), 41 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 905b9bee9b..59dc3558bf 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -1,6 +1,12 @@ name: Build Wheels (CUDA) -on: workflow_dispatch +on: + workflow_dispatch: + inputs: + release_tag: + description: Release tag to upload wheel assets to + required: false + type: string permissions: contents: write @@ -131,7 +137,9 @@ jobs: if ($IsWindows) { python -m pip install build wheel ninja } else { - python -m pip install build wheel + sudo apt-get update + sudo apt-get install -y patchelf + python -m pip install auditwheel build wheel } - name: Build Wheel @@ -169,6 +177,12 @@ jobs: $env:CPLUS_INCLUDE_PATH = "$cudaRoot/include$pathSeparator$env:CPLUS_INCLUDE_PATH" $env:LIBRARY_PATH = "$cudaRoot/lib$pathSeparator$env:CONDA_PREFIX/lib$pathSeparator$env:LIBRARY_PATH" $env:LD_LIBRARY_PATH = "$cudaRoot/lib$pathSeparator$env:CONDA_PREFIX/lib$pathSeparator$env:LD_LIBRARY_PATH" + $cudaLibraryPaths = @( + (Join-Path $cudaRoot 'lib'), + (Join-Path $cudaRoot 'lib64'), + (Join-Path $env:CONDA_PREFIX 'lib') + ) | Where-Object { Test-Path $_ } + Write-Output "CUDA_LIBRARY_PATHS=$($cudaLibraryPaths -join ':')" >> $env:GITHUB_ENV } elseif ($IsWindows) { $ninjaPath = ((Get-Command ninja -ErrorAction Stop).Source).Replace('\', '/') $env:CMAKE_GENERATOR = 'Ninja' @@ -218,15 +232,44 @@ jobs: # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit. $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=$cudaArchs -DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler $env:CMAKE_ARGS" $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' + if ($IsLinux) { + $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_OPENMP=OFF' + } python -m build --wheel # Publish tags that reflect the actual installed toolkit version. Write-Output "CUDA_VERSION=$cudaTagVersion" >> $env:GITHUB_ENV + - name: Repair Linux wheel + if: runner.os == 'Linux' + shell: bash + run: | + set -euxo pipefail + mkdir -p wheelhouse + export LD_LIBRARY_PATH="$PWD/llama_cpp/lib:${CUDA_LIBRARY_PATHS}:${LD_LIBRARY_PATH:-}" + auditwheel_bin="${CONDA}/envs/llamacpp/bin/auditwheel" + "${auditwheel_bin}" repair \ + --exclude libcuda.so \ + --exclude libcuda.so.1 \ + --exclude libcudart.so.11.0 \ + --exclude libcudart.so.12 \ + --exclude libcudart.so.13 \ + --exclude libcublas.so.11 \ + --exclude libcublas.so.12 \ + --exclude libcublas.so.13 \ + --exclude libcublasLt.so.11 \ + --exclude libcublasLt.so.12 \ + --exclude libcublasLt.so.13 \ + -w wheelhouse \ + dist/*.whl + rm dist/*.whl + cp wheelhouse/*.whl dist/ + "${auditwheel_bin}" show dist/*.whl + - uses: softprops/action-gh-release@v3 - if: startsWith(github.ref, 'refs/tags/') + if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.release_tag != '') with: files: dist/* - # Set tag_name to <tag>-cu<cuda_version> - tag_name: ${{ github.ref_name }}-cu${{ env.CUDA_VERSION }} + # Set tag_name to <tag>-cu<cuda_version>. + tag_name: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }}-cu${{ env.CUDA_VERSION }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-wheels-rocm.yaml b/.github/workflows/build-wheels-rocm.yaml index 6ad0b49541..70fca5edb7 100644 --- a/.github/workflows/build-wheels-rocm.yaml +++ b/.github/workflows/build-wheels-rocm.yaml @@ -1,6 +1,20 @@ name: Build Wheels (ROCm) -on: workflow_dispatch +on: + workflow_dispatch: + inputs: + release_tag: + description: Release tag to upload wheel assets to + required: false + type: string + amdgpu_targets: + description: AMDGPU targets to compile into the Linux ROCm wheel + required: false + default: gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1200;gfx1201 + windows_amdgpu_targets: + description: AMDGPU targets to compile into the Windows HIP Radeon wheel + required: false + default: gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032 permissions: contents: write @@ -24,7 +38,7 @@ jobs: - name: Install system dependencies run: | apt-get update - apt-get install -y --no-install-recommends git cmake lsb-release ninja-build + apt-get install -y --no-install-recommends git cmake lsb-release ninja-build patchelf - uses: actions/checkout@v6 with: @@ -37,9 +51,12 @@ jobs: - name: Install build dependencies run: | python -m pip install --upgrade pip - python -m pip install build wheel + python -m pip install auditwheel build wheel - name: Build ROCm wheel + env: + MATRIX_AMDGPU_TARGETS: ${{ matrix.amdgpu_targets }} + INPUT_AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }} run: | export ROCM_PATH="${ROCM_PATH:-/opt/rocm}" export HIP_PATH="${HIP_PATH:-$ROCM_PATH}" @@ -56,11 +73,38 @@ jobs: rocm_tag="$(hipconfig --version | sed -E 's/^([0-9]+)\.([0-9]+).*/\1\2/')" echo "ROCM_VERSION=$rocm_tag" >> "$GITHUB_ENV" - amdgpu_targets="${{ matrix.amdgpu_targets }}" - export CMAKE_ARGS="-DGGML_HIP=on -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off -DAMDGPU_TARGETS=$amdgpu_targets -DCMAKE_HIP_ARCHITECTURES=$amdgpu_targets" + amdgpu_targets="${INPUT_AMDGPU_TARGETS:-$MATRIX_AMDGPU_TARGETS}" + export CMAKE_ARGS="-DGGML_HIP=on -DGGML_NATIVE=off -DGGML_OPENMP=OFF -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off -DAMDGPU_TARGETS=$amdgpu_targets -DCMAKE_HIP_ARCHITECTURES=$amdgpu_targets" python -m build --wheel + + - name: Repair Linux wheel + run: | + export ROCM_PATH="${ROCM_PATH:-/opt/rocm}" + export LD_LIBRARY_PATH="$PWD/llama_cpp/lib:$ROCM_PATH/lib:$ROCM_PATH/lib64:${LD_LIBRARY_PATH:-}" mkdir -p wheelhouse - cp dist/*.whl wheelhouse/ + python -m auditwheel repair \ + --exclude libamdhip64.so \ + --exclude libamdhip64.so.6 \ + --exclude libamdhip64.so.7 \ + --exclude libhiprtc.so \ + --exclude libhiprtc.so.6 \ + --exclude libhiprtc.so.7 \ + --exclude libhipblas.so \ + --exclude libhipblas.so.2 \ + --exclude libhipblas.so.3 \ + --exclude libhipblaslt.so \ + --exclude libhipblaslt.so.0 \ + --exclude libhipblaslt.so.1 \ + --exclude librocblas.so \ + --exclude librocblas.so.4 \ + --exclude librocblas.so.5 \ + --exclude libhsa-runtime64.so.1 \ + --exclude libhsakmt.so.1 \ + -w wheelhouse \ + dist/*.whl + rm dist/*.whl + cp wheelhouse/*.whl dist/ + python -m auditwheel show dist/*.whl - uses: actions/upload-artifact@v7 with: @@ -139,11 +183,14 @@ jobs: & $clangPath.FullName --version - name: Build HIP wheel + env: + MATRIX_AMDGPU_TARGETS: ${{ matrix.amdgpu_targets }} + INPUT_AMDGPU_TARGETS: ${{ inputs.windows_amdgpu_targets }} run: | $ErrorActionPreference = "Stop" $hipPath = Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Split-Path | Split-Path $rocwmmaInclude = (Join-Path $PWD 'opt\rocm-7.2.1\include').Replace('\', '/') - $amdgpuTargets = "${{ matrix.amdgpu_targets }}" + $amdgpuTargets = if ($env:INPUT_AMDGPU_TARGETS) { $env:INPUT_AMDGPU_TARGETS } else { $env:MATRIX_AMDGPU_TARGETS } $env:HIP_PATH = $hipPath $env:ROCM_PATH = $hipPath @@ -198,7 +245,7 @@ jobs: release_rocm: name: Release ROCm needs: [build_wheels] - if: startsWith(github.ref, 'refs/tags/') + if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.release_tag != '') runs-on: ubuntu-latest steps: @@ -211,14 +258,14 @@ jobs: with: files: dist/* # Set release name to <tag>-rocm<rocm_version>. - tag_name: ${{ github.ref_name }}-rocm72 + tag_name: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }}-rocm72 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} release_hip: name: Release HIP needs: [build_wheels_windows_hip] - if: startsWith(github.ref, 'refs/tags/') + if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.release_tag != '') runs-on: ubuntu-latest steps: @@ -231,6 +278,6 @@ jobs: with: files: dist/* # Set release name to <tag>-hip-radeon. - tag_name: ${{ github.ref_name }}-hip-radeon + tag_name: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }}-hip-radeon env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-wheels-vulkan.yaml b/.github/workflows/build-wheels-vulkan.yaml index 8790cc20e9..822662c3fb 100644 --- a/.github/workflows/build-wheels-vulkan.yaml +++ b/.github/workflows/build-wheels-vulkan.yaml @@ -1,6 +1,12 @@ name: Build Wheels (Vulkan) -on: workflow_dispatch +on: + workflow_dispatch: + inputs: + release_tag: + description: Release tag to upload wheel assets to + required: false + type: string permissions: contents: write @@ -40,23 +46,6 @@ jobs: python-version: ${{ matrix.pyver }} cache: "pip" - - name: Install Vulkan SDK - if: runner.os == 'Linux' - run: | - curl -fL \ - "https://sdk.lunarg.com/sdk/download/${VULKAN_SDK_VERSION}/linux/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.xz" \ - -o vulkan-sdk.tar.xz - echo "${VULKAN_SDK_LINUX_SHA256} vulkan-sdk.tar.xz" | sha256sum -c - - mkdir -p "$RUNNER_TEMP/vulkan-sdk" - tar -xf vulkan-sdk.tar.xz -C "$RUNNER_TEMP/vulkan-sdk" - source "$RUNNER_TEMP/vulkan-sdk/${VULKAN_SDK_VERSION}/setup-env.sh" - { - echo "VULKAN_SDK=$VULKAN_SDK" - echo "LD_LIBRARY_PATH=$VULKAN_SDK/lib:${LD_LIBRARY_PATH:-}" - } >> "$GITHUB_ENV" - echo "$VULKAN_SDK/bin" >> "$GITHUB_PATH" - "$VULKAN_SDK/bin/glslc" --version - - name: Install Vulkan SDK if: runner.os == 'Windows' shell: pwsh @@ -71,6 +60,7 @@ jobs: & "$vulkanSdk\Bin\glslc.exe" --version - name: Install build dependencies + if: runner.os == 'Windows' run: | python -m pip install --upgrade pip python -m pip install build wheel @@ -81,11 +71,28 @@ jobs: - name: Build Vulkan wheel if: runner.os == 'Linux' - run: | - export CMAKE_ARGS="-DGGML_NATIVE=off -DGGML_METAL=OFF -DGGML_VULKAN=on" - python -m build --wheel - mkdir -p wheelhouse - cp dist/*.whl wheelhouse/ + uses: pypa/cibuildwheel@v3.4.1 + env: + CIBW_BUILD: "cp38-manylinux_*" + CIBW_ARCHS: "auto64" + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014" + CIBW_BEFORE_ALL_LINUX: > + yum install -y xz && + curl -L https://micro.mamba.pm/api/micromamba/linux-64/latest -o /tmp/micromamba.tar.bz2 && + mkdir -p /tmp/micromamba && + tar -xjf /tmp/micromamba.tar.bz2 -C /tmp/micromamba bin/micromamba && + /tmp/micromamba/bin/micromamba create -y -p /opt/vulkan -c conda-forge shaderc libvulkan-loader spirv-headers && + /opt/vulkan/bin/glslc --version && + curl -fL "https://sdk.lunarg.com/sdk/download/${{ env.VULKAN_SDK_VERSION }}/linux/vulkansdk-linux-x86_64-${{ env.VULKAN_SDK_VERSION }}.tar.xz" -o /tmp/vulkan-sdk.tar.xz && + echo "${{ env.VULKAN_SDK_LINUX_SHA256 }} /tmp/vulkan-sdk.tar.xz" | sha256sum -c - && + mkdir -p /opt/vulkan-sdk && + tar -xf /tmp/vulkan-sdk.tar.xz -C /opt/vulkan-sdk + CIBW_ENVIRONMENT_LINUX: > + CMAKE_ARGS="-DGGML_NATIVE=off -DGGML_METAL=OFF -DGGML_OPENMP=OFF -DGGML_VULKAN=on -DCMAKE_PREFIX_PATH=/opt/vulkan -DVulkan_INCLUDE_DIR=/opt/vulkan-sdk/${{ env.VULKAN_SDK_VERSION }}/x86_64/include -DVulkan_LIBRARY=/opt/vulkan/lib/libvulkan.so -DVulkan_GLSLC_EXECUTABLE=/opt/vulkan/bin/glslc" + CIBW_REPAIR_WHEEL_COMMAND_LINUX: "LD_LIBRARY_PATH=/project/llama_cpp/lib:/opt/vulkan/lib auditwheel repair --exclude libvulkan.so.1 -w {dest_dir} {wheel}" + with: + package-dir: . + output-dir: wheelhouse - name: Build Vulkan wheel if: runner.os == 'Windows' @@ -105,7 +112,7 @@ jobs: release: name: Release needs: [build_wheels] - if: startsWith(github.ref, 'refs/tags/') + if: startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && inputs.release_tag != '') runs-on: ubuntu-latest steps: @@ -118,6 +125,6 @@ jobs: with: files: dist/* # Set release name to <tag>-vulkan. - tag_name: ${{ github.ref_name }}-vulkan + tag_name: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || github.ref_name }}-vulkan env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index b83ccc83bf..e358b871fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(ci): Repair Linux accelerator wheels for manylinux publishing + ## [0.3.28] - feat(example): align server MTP support with llama.cpp by @abetlen in #2283 From d4ac2c2cc0dfab3e078c72101961b93a1261d791 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Sun, 7 Jun 2026 22:56:19 -0700 Subject: [PATCH 555/561] fix(example): support multi-step Responses tool streaming (#2288) * fix(example): support multi-step Responses tool streaming * docs: add Responses tool streaming changelog --- CHANGELOG.md | 1 + examples/server/server.py | 112 +++++++++++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e358b871fe..ac792bc2a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(example): support multi-step Responses tool streaming by @abetlen in #2288 - fix(ci): Repair Linux accelerator wheels for manylinux publishing ## [0.3.28] diff --git a/examples/server/server.py b/examples/server/server.py index 28fc8f4eb3..fb00501cfe 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -2812,6 +2812,7 @@ def to_chat_template_tool(self) -> ChatTemplateTool: class ResponsesCustomToolFormat(BaseModel): model_config = ConfigDict(extra="ignore") + type: Optional[str] = None syntax: Optional[str] = None definition: Optional[str] = None @@ -2880,10 +2881,24 @@ class ResponsesWebSearchTool(BaseModel): type: Literal["web_search"] +class ResponsesNamespaceTool(BaseModel): + model_config = ConfigDict(extra="ignore") + + type: Literal["namespace"] + + +class ResponsesImageGenerationTool(BaseModel): + model_config = ConfigDict(extra="ignore") + + type: Literal["image_generation"] + + ResponsesToolDefinition = Union[ ResponsesFunctionTool, ResponsesCustomTool, ResponsesWebSearchTool, + ResponsesNamespaceTool, + ResponsesImageGenerationTool, ] @@ -5069,6 +5084,68 @@ def _tool_content_type(self, tool_name: str) -> Optional[str]: return content_type return None + def _raw_string_tool_arguments(self, tool_name: str, value: str) -> Optional[Dict[str, str]]: + if self._tools is None: + return None + for tool in self._tools: + if tool.get("type") != "function": + continue + function = tool.get("function", {}) + if function.get("name") != tool_name: + continue + parameters = function.get("parameters") + if not isinstance(parameters, dict): + return None + required = parameters.get("required") + if not isinstance(required, list) or len(required) != 1: + return None + argument_name = required[0] + if not isinstance(argument_name, str): + return None + properties = parameters.get("properties") + if not isinstance(properties, dict): + return None + argument_schema = properties.get(argument_name) + if not isinstance(argument_schema, dict): + return None + argument_type = argument_schema.get("type") + if argument_type == "string" or ( + isinstance(argument_type, list) and "string" in argument_type + ): + return {argument_name: value} + return None + return None + + @classmethod + def _raw_object_tool_arguments(cls, value: str) -> Optional[Dict[str, Any]]: + candidates = [value] + stripped = value.strip() + if stripped.startswith("{{") and stripped.endswith("}}"): + candidates.append(stripped[1:-1]) + for candidate in candidates: + normalized = cls._gemma4_tool_call_to_json(candidate) + for allow_partial in (False, True): + try: + parsed = from_json(normalized, allow_partial=allow_partial) + except ValueError: + continue + if isinstance(parsed, dict): + return { + key: cls._trim_partial_gemma_quote_marker(value) + if isinstance(value, str) + else value + for key, value in parsed.items() + } + return None + + @staticmethod + def _trim_partial_gemma_quote_marker(value: str) -> str: + quote_marker = '<|"|>' + for prefix_length in range(len(quote_marker) - 1, 0, -1): + if value.endswith(quote_marker[:prefix_length]): + return value[:-prefix_length] + return value + def _has_text_tools(self) -> bool: return any( isinstance(tool_schema, dict) and tool_schema.get("content_type") == "text" @@ -5637,6 +5714,18 @@ def _advance_direct_stream_state(self, text: str) -> Tuple[bool, List[Dict[str, self._direct.saw_tool_calls = saw_tool_calls self._direct.done = done return True, deltas + if leading_capture_field is not None: + if buffer.startswith(leading_capture_start): + buffer = buffer[len(leading_capture_start) :] + mode = self.DIRECT_MODE_LEADING_CAPTURE + continue + if leading_capture_start.startswith(buffer): + self._direct.pending = buffer + self._direct.mode = mode + self._direct.tool_call_count = tool_call_count + self._direct.saw_tool_calls = saw_tool_calls + self._direct.done = done + return True, deltas if buffer.startswith(iterator_start): saw_tool_calls = True self._start_direct_tool_call(tool_call_count) @@ -6302,6 +6391,16 @@ def _advance_stream_state(self, text: str) -> Tuple[bool, List[Dict[str, Any]]]: if not buffer: state.pending = "" return True, deltas + leading_capture = plan.get("leading_capture") + if leading_capture is not None: + capture_start = leading_capture["start"] + if buffer.startswith(capture_start): + buffer = buffer[len(capture_start) :] + state.mode = "leading-capture" + continue + if capture_start.startswith(buffer): + state.pending = buffer + return True, deltas if buffer.startswith(iterator_start): item_state = self._new_tool_call_state(plan["iterator"]["item"]) state.saw_tool_calls = True @@ -6866,6 +6965,10 @@ def _normalize_tool_call_item( }, } arguments = function.get("arguments", {}) + if isinstance(arguments, str): + arguments = self._raw_object_tool_arguments(arguments) or self._raw_string_tool_arguments( + tool_name, arguments + ) if not isinstance(arguments, (dict, ResponseParser.PartialJsonObject)): if partial: return None @@ -8009,7 +8112,14 @@ def _responses_tools_to_chat_tools( return None chat_tools: List[ChatTemplateTool] = [] for tool in tools: - if isinstance(tool, ResponsesWebSearchTool): + if isinstance( + tool, + ( + ResponsesWebSearchTool, + ResponsesNamespaceTool, + ResponsesImageGenerationTool, + ), + ): continue if isinstance(tool, ResponsesFunctionTool): chat_tools.append(tool.to_chat_template_tool()) From e8191f002770aa4631d960dae165dcb84df39531 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 8 Jun 2026 02:25:24 -0700 Subject: [PATCH 556/561] fix(example): correct GPT-OSS tool calling config for server example (#2289) --- examples/server/configs/gpt-oss-120b.json | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/examples/server/configs/gpt-oss-120b.json b/examples/server/configs/gpt-oss-120b.json index 2fa7a76f1e..64aaf040d7 100644 --- a/examples/server/configs/gpt-oss-120b.json +++ b/examples/server/configs/gpt-oss-120b.json @@ -42,7 +42,7 @@ }, "tool_calls": { "type": "array", - "x-regex-iterator": "(<\\|start\\|>assistant to=functions\\.\\w+<\\|channel\\|>commentary json<\\|message\\|>.*?<\\|call\\|>)", + "x-regex-iterator": "(<\\|start\\|>assistant<\\|channel\\|>commentary to=functions\\.\\w+ <\\|constrain\\|>json<\\|message\\|>.*?<\\|call\\|>)", "items": { "type": "object", "properties": { @@ -54,11 +54,11 @@ "properties": { "name": { "type": "string", - "x-regex": "^<\\|start\\|>assistant to=functions\\.(\\w+)<\\|channel\\|>commentary json<\\|message\\|>" + "x-regex": "^<\\|start\\|>assistant<\\|channel\\|>commentary to=functions\\.(\\w+) <\\|constrain\\|>json<\\|message\\|>" }, "arguments": { "type": "object", - "x-regex": "^<\\|start\\|>assistant to=functions\\.\\w+<\\|channel\\|>commentary json<\\|message\\|>(.*?)<\\|call\\|>$", + "x-regex": "^<\\|start\\|>assistant<\\|channel\\|>commentary to=functions\\.\\w+ <\\|constrain\\|>json<\\|message\\|>(.*?)<\\|call\\|>$", "x-parser": "json", "additionalProperties": true } @@ -410,7 +410,21 @@ "\n", "{#- Generation prompt #}\n", "{%- if add_generation_prompt -%}\n", + "{%- set forced_tool_name = none %}\n", + "{%- if tool_choice is mapping %}\n", + " {%- if tool_choice.function is defined and tool_choice.function.name is defined %}\n", + " {%- set forced_tool_name = tool_choice.function.name %}\n", + " {%- elif tool_choice.name is defined %}\n", + " {%- set forced_tool_name = tool_choice.name %}\n", + " {%- endif %}\n", + "{%- elif function_call is mapping and function_call.name is defined %}\n", + " {%- set forced_tool_name = function_call.name %}\n", + "{%- endif %}\n", + "{%- if forced_tool_name %}\n", + "{{- \"<|start|>assistant<|channel|>commentary to=functions.\" + forced_tool_name + \" <|constrain|>json<|message|>\" }}\n", + "{%- else %}\n", "<|start|>assistant\n", + "{%- endif %}\n", "{%- endif -%}" ] } From e1079999f8a339d39c60dee4d751a3c311b2d72d Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 8 Jun 2026 08:14:59 -0700 Subject: [PATCH 557/561] feat: update llama.cpp to 8f83d6c27 (#2290) --- CHANGELOG.md | 1 + llama_cpp/mtmd_cpp.py | 217 ++++++++++++++++++++++++++++++++++++++++-- vendor/llama.cpp | 2 +- 3 files changed, 213 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac792bc2a9..10209aacfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@8f83d6c27 - fix(example): support multi-step Responses tool streaming by @abetlen in #2288 - fix(ci): Repair Linux accelerator wheels for manylinux publishing diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index abe4c7a679..919cefb357 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -3,9 +3,11 @@ import os import warnings from ctypes import ( + CFUNCTYPE, c_bool, c_char_p, c_int, + c_int64, c_uint8, c_uint32, c_size_t, @@ -62,6 +64,9 @@ mtmd_bitmap_p = NewType("mtmd_bitmap_p", int) mtmd_bitmap_p_ctypes = c_void_p +mtmd_helper_video_p = NewType("mtmd_helper_video_p", int) +mtmd_helper_video_p_ctypes = c_void_p + mtmd_image_tokens_p = NewType("mtmd_image_tokens_p", int) mtmd_image_tokens_p_ctypes = c_void_p @@ -151,6 +156,60 @@ class mtmd_caps(Structure): ] +mtmd_bitmap_lazy_callback = CFUNCTYPE( + c_int, + c_size_t, + c_void_p, + POINTER(mtmd_bitmap_p_ctypes), + POINTER(c_char_p), +) + + +class mtmd_helper_bitmap_wrapper(Structure): + """Bitmap wrapper returned by MTMD helper media loaders.""" + + if TYPE_CHECKING: + bitmap: Optional[mtmd_bitmap_p] + video_ctx: Optional[mtmd_helper_video_p] + + _fields_ = [ + ("bitmap", mtmd_bitmap_p_ctypes), + ("video_ctx", mtmd_helper_video_p_ctypes), + ] + + +class mtmd_helper_video_info(Structure): + """Metadata for a decoded video stream.""" + + if TYPE_CHECKING: + width: int + height: int + fps: float + n_frames: int + + _fields_ = [ + ("width", c_uint32), + ("height", c_uint32), + ("fps", c_float), + ("n_frames", c_int), + ] + + +class mtmd_helper_video_init_params(Structure): + """Parameters for initializing an MTMD helper video stream.""" + + if TYPE_CHECKING: + fps_target: float + ffmpeg_bin_dir: Optional[bytes] + timestamp_interval_ms: int + + _fields_ = [ + ("fps_target", c_float), + ("ffmpeg_bin_dir", c_char_p), + ("timestamp_interval_ms", c_int64), + ] + + ################################################ # mtmd.h functions ################################################ @@ -234,6 +293,13 @@ def mtmd_get_audio_sample_rate(ctx: mtmd_context_p, /) -> int: ... +# MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx); +@ctypes_function("mtmd_get_marker", [mtmd_context_p_ctypes], c_char_p) +def mtmd_get_marker(ctx: mtmd_context_p, /) -> Optional[bytes]: + """Get the current media marker string.""" + ... + + # Deprecated compatibility wrapper for the renamed mtmd_get_audio_sample_rate(). def mtmd_get_audio_bitrate(ctx: mtmd_context_p, /) -> int: warnings.warn( @@ -325,6 +391,26 @@ def mtmd_bitmap_set_id(bitmap: mtmd_bitmap_p, id: Optional[bytes], /): ... +# MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx, +# const char * id, +# void * user_data, +# mtmd_bitmap_lazy_callback callback); +@ctypes_function( + "mtmd_bitmap_init_lazy", + [mtmd_context_p_ctypes, c_char_p, c_void_p, mtmd_bitmap_lazy_callback], + mtmd_bitmap_p_ctypes, +) +def mtmd_bitmap_init_lazy( + ctx: mtmd_context_p, + id: Optional[bytes], + user_data: c_void_p, + callback: mtmd_bitmap_lazy_callback, + /, +) -> Optional[mtmd_bitmap_p]: + """Initialize a lazy MTMD bitmap.""" + ... + + # MTMD_API mtmd_input_chunks * mtmd_input_chunks_init(void); @ctypes_function("mtmd_input_chunks_init", [], mtmd_input_chunks_p_ctypes) def mtmd_input_chunks_init() -> Optional[mtmd_input_chunks_p]: ... @@ -551,32 +637,63 @@ def mtmd_test_create_input_chunks() -> Optional[mtmd_input_chunks_p]: ################################################ -# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); +# MTMD_API bool mtmd_helper_support_video(mtmd_context * ctx); +@ctypes_function( + "mtmd_helper_support_video", + [mtmd_context_p_ctypes], + c_bool, +) +def mtmd_helper_support_video(ctx: mtmd_context_p, /) -> bool: + """Check whether MTMD helper video support is available.""" + ... + + +# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); @ctypes_function( "mtmd_helper_bitmap_init_from_file", [mtmd_context_p_ctypes, c_char_p, c_bool], - mtmd_bitmap_p_ctypes, + mtmd_helper_bitmap_wrapper, ) +def mtmd_helper_bitmap_init_from_file_wrapper( + ctx: mtmd_context_p, fname: bytes, placeholder: Union[c_bool, bool], / +) -> mtmd_helper_bitmap_wrapper: + """Initialize an MTMD bitmap wrapper from a file.""" + ... + + def mtmd_helper_bitmap_init_from_file( ctx: mtmd_context_p, fname: bytes, placeholder: Union[c_bool, bool], / ) -> Optional[mtmd_bitmap_p]: """Initialize an MTMD bitmap from a file.""" - ... + return mtmd_helper_bitmap_init_from_file_wrapper(ctx, fname, placeholder).bitmap -# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); +# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); @ctypes_function( "mtmd_helper_bitmap_init_from_buf", [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t, c_bool], - mtmd_bitmap_p_ctypes, + mtmd_helper_bitmap_wrapper, ) +def mtmd_helper_bitmap_init_from_buf_wrapper( + ctx: mtmd_context_p, + buf: CtypesArray[c_uint8], + length: Union[c_size_t, int], + placeholder: Union[c_bool, bool], + /, +) -> mtmd_helper_bitmap_wrapper: ... + + def mtmd_helper_bitmap_init_from_buf( ctx: mtmd_context_p, buf: CtypesArray[c_uint8], length: Union[c_size_t, int], placeholder: Union[c_bool, bool], /, -) -> Optional[mtmd_bitmap_p]: ... +) -> Optional[mtmd_bitmap_p]: + """Initialize an MTMD bitmap from a buffer.""" + return mtmd_helper_bitmap_init_from_buf_wrapper( + ctx, buf, length, placeholder + ).bitmap # MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); @@ -719,6 +836,94 @@ def mtmd_helper_decode_image_chunk( ... +# MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void); +@ctypes_function( + "mtmd_helper_video_init_params_default", [], mtmd_helper_video_init_params +) +def mtmd_helper_video_init_params_default() -> mtmd_helper_video_init_params: + """Return the default MTMD helper video initialization parameters.""" + ... + + +# MTMD_API mtmd_helper_video * mtmd_helper_video_init( +# struct mtmd_context * mctx, +# const char * path, +# struct mtmd_helper_video_init_params params); +@ctypes_function( + "mtmd_helper_video_init", + [mtmd_context_p_ctypes, c_char_p, mtmd_helper_video_init_params], + mtmd_helper_video_p_ctypes, +) +def mtmd_helper_video_init( + ctx: mtmd_context_p, + path: bytes, + params: mtmd_helper_video_init_params, + /, +) -> Optional[mtmd_helper_video_p]: + """Initialize an MTMD helper video stream from a file path.""" + ... + + +# MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf( +# struct mtmd_context * mctx, +# const unsigned char * buf, size_t len, +# struct mtmd_helper_video_init_params params); +@ctypes_function( + "mtmd_helper_video_init_from_buf", + [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t, mtmd_helper_video_init_params], + mtmd_helper_video_p_ctypes, +) +def mtmd_helper_video_init_from_buf( + ctx: mtmd_context_p, + buf: CtypesArray[c_uint8], + length: Union[c_size_t, int], + params: mtmd_helper_video_init_params, + /, +) -> Optional[mtmd_helper_video_p]: + """Initialize an MTMD helper video stream from a buffer.""" + ... + + +# MTMD_API void mtmd_helper_video_free(mtmd_helper_video * ctx); +@ctypes_function("mtmd_helper_video_free", [mtmd_helper_video_p_ctypes], None) +def mtmd_helper_video_free(ctx: mtmd_helper_video_p, /): + """Free an MTMD helper video stream.""" + ... + + +# MTMD_API struct mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx); +@ctypes_function( + "mtmd_helper_video_get_info", + [mtmd_helper_video_p_ctypes], + mtmd_helper_video_info, +) +def mtmd_helper_video_get_info(ctx: mtmd_helper_video_p, /) -> mtmd_helper_video_info: + """Get metadata for an MTMD helper video stream.""" + ... + + +# MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, +# mtmd_bitmap ** out_bitmap, +# char ** out_text); +@ctypes_function( + "mtmd_helper_video_read_next", + [ + mtmd_helper_video_p_ctypes, + POINTER(mtmd_bitmap_p_ctypes), + POINTER(c_char_p), + ], + c_int, +) +def mtmd_helper_video_read_next( + ctx: mtmd_helper_video_p, + out_bitmap: "_Pointer[mtmd_bitmap_p_ctypes]", + out_text: "_Pointer[c_char_p]", + /, +) -> int: + """Read the next bitmap or text chunk from an MTMD helper video stream.""" + ... + + # MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function( "mtmd_log_set", diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 9e3b928fd8..8f83d6c271 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 9e3b928fd8c9d14dbf15a8768b9fdd7e5c721d66 +Subproject commit 8f83d6c271d194bde2d410145a0ce73bc42e85cd From 051dda2515f240fe5e93ddbf6124f9c9dead3f73 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Mon, 8 Jun 2026 21:49:57 -0700 Subject: [PATCH 558/561] feat(example): support server video inputs and Gemma text tool calls (#2291) * feat(example): support server video inputs and Gemma text tool calls * fix(example): pass media count to MTMD tokenizer * fix(example): avoid MTMD video stdin double close * fix(example): parse Gemma text tool bracket calls --- CHANGELOG.md | 1 + .../server/configs/gemma-4-12b-it-qat.json | 6 +- examples/server/server.py | 410 +++++++++++++++--- 3 files changed, 354 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10209aacfb..e733f37e6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat(example): support server video inputs and Gemma text tool calls by @abetlen in #2291 - feat: update llama.cpp to ggml-org/llama.cpp@8f83d6c27 - fix(example): support multi-step Responses tool streaming by @abetlen in #2288 - fix(ci): Repair Linux accelerator wheels for manylinux publishing diff --git a/examples/server/configs/gemma-4-12b-it-qat.json b/examples/server/configs/gemma-4-12b-it-qat.json index fefeda8da0..60ffd2a987 100644 --- a/examples/server/configs/gemma-4-12b-it-qat.json +++ b/examples/server/configs/gemma-4-12b-it-qat.json @@ -44,7 +44,7 @@ }, "tool_calls": { "type": "array", - "x-regex-iterator": "<\\|tool_call>(call:[^\\{]+\\{.*?\\})<tool_call\\|>", + "x-regex-iterator": "<\\|tool_call>(call:[^\\{\\[]+[\\{\\[].*?[\\}\\]])<tool_call\\|>", "items": { "type": "object", "properties": { @@ -56,11 +56,11 @@ "properties": { "name": { "type": "string", - "x-regex": "^call:(\\w+)\\{" + "x-regex": "^call:(\\w+)[\\{\\[]" }, "arguments": { "type": "object", - "x-regex": "^call:\\w+(\\{.*\\})$", + "x-regex": "^call:\\w+(\\{.*\\}|\\[.*\\])$", "x-parser": "gemma4-tool-call", "additionalProperties": true } diff --git a/examples/server/server.py b/examples/server/server.py index fb00501cfe..e8034a2146 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -37,6 +37,7 @@ import shutil import inspect import sys +import tempfile import urllib.error import urllib.parse import urllib.request @@ -3224,6 +3225,7 @@ class MTMDOptions(BaseModel): allowed_local_media_path: Optional[str] = None image_max_bytes: int = Field(default=20 * 1024 * 1024, ge=1) audio_max_bytes: int = Field(default=100 * 1024 * 1024, ge=1) + video_max_bytes: int = Field(default=512 * 1024 * 1024, ge=1) image_timeout_seconds: float = Field(default=10.0, gt=0.0) @model_validator(mode="after") @@ -3350,7 +3352,7 @@ def load(cls, path: str) -> "ConfigFile": @dataclass(frozen=True) class MediaInput: - kind: Literal["image", "audio"] + kind: Literal["image", "audio", "video"] url: Optional[str] = None data: Optional[str] = None format: Optional[str] = None @@ -3426,6 +3428,33 @@ def media_inputs_from_messages( ) ) continue + if part_type in {"video_url", "input_video", "video"}: + input_video = part.get("input_video") + if isinstance(input_video, dict): + data = input_video.get("data") + video_url = input_video.get("video_url") or input_video.get("url") + video_format = input_video.get("format") + else: + data = part.get("data") + video_url = part.get("video_url") or part.get("url") + video_format = part.get("format") + if isinstance(video_url, dict): + video_url = video_url.get("url") + if isinstance(data, str): + if video_format is not None and not isinstance(video_format, str): + raise ValueError("input_video format must be a string") + media_inputs.append( + MediaInput( + kind="video", + data=data, + format=cast(Optional[str], video_format), + ) + ) + elif isinstance(video_url, str): + media_inputs.append(MediaInput(kind="video", url=video_url)) + else: + raise ValueError("video content part requires base64 data or a URL string") + continue return media_inputs @staticmethod @@ -3447,7 +3476,16 @@ def _content_part_for_template( if not isinstance(part, dict): raise ValueError("content parts must be strings or objects") part_type = part.get("type") - if part_type in {"image_url", "input_image", "image", "audio_url", "input_audio"}: + if part_type in { + "image_url", + "input_image", + "image", + "audio_url", + "input_audio", + "video_url", + "input_video", + "video", + }: if media_marker is None: raise ValueError("multimodal content requires model.mtmd") return media_marker @@ -3694,7 +3732,7 @@ class Media: positions: np.ndarray non_causal: bool = False - kind: Literal["text", "image", "audio"] + kind: Literal["text", "image", "audio", "video"] start_pos: int n_pos: int identity_tokens: List[int] @@ -4351,6 +4389,9 @@ def capture(match: re.Match[str]) -> str: strings.append(match.group(1)) return f"\x00{len(strings) - 1}\x00" + stripped = text.strip() + if stripped.startswith("[") and stripped.endswith("]"): + text = "{" + stripped[1:-1] + "}" text = re.sub(r'<\|"\|>(.*?)<\|"\|>', capture, text, flags=re.S) text = re.sub(r"(?<=[{,])(\w+):", r'"\1":', text) for index, value in enumerate(strings): @@ -5116,6 +5157,87 @@ def _raw_string_tool_arguments(self, tool_name: str, value: str) -> Optional[Dic return None return None + def _single_string_tool_argument_name(self, tool_name: str) -> Optional[str]: + if self._tools is None: + return None + for tool in self._tools: + if tool.get("type") != "function": + continue + function = tool.get("function", {}) + if function.get("name") != tool_name: + continue + parameters = function.get("parameters") + if not isinstance(parameters, dict): + return None + required = parameters.get("required") + if not isinstance(required, list) or len(required) != 1: + return None + argument_name = required[0] + if not isinstance(argument_name, str): + return None + properties = parameters.get("properties") + if not isinstance(properties, dict): + return None + argument_schema = properties.get(argument_name) + if not isinstance(argument_schema, dict): + return None + argument_type = argument_schema.get("type") + if argument_type == "string" or ( + isinstance(argument_type, list) and "string" in argument_type + ): + return argument_name + return None + return None + + def _text_tool_argument_from_object( + self, + tool_name: str, + arguments: Dict[str, Any], + ) -> Optional[str]: + input_value = arguments.get("input") + if isinstance(input_value, str): + return input_value + argument_name = self._single_string_tool_argument_name(tool_name) + if argument_name is not None: + argument_value = arguments.get(argument_name) + if isinstance(argument_value, str): + return argument_value + if len(arguments) == 1: + argument_value = next(iter(arguments.values())) + if isinstance(argument_value, str): + return argument_value + return None + + def _text_tool_arguments( + self, + tool_name: str, + arguments: Any, + *, + partial: bool, + ) -> Optional[str]: + if isinstance(arguments, str): + parsed_arguments = self._raw_object_tool_arguments(arguments) + if parsed_arguments is not None: + text = self._text_tool_argument_from_object(tool_name, parsed_arguments) + if text is not None: + return text + if partial: + return None + return json.dumps(parsed_arguments, ensure_ascii=False, separators=(",", ":")) + return arguments + if isinstance(arguments, ResponseParser.PartialJsonObject): + arguments = arguments.value + if isinstance(arguments, dict): + text = self._text_tool_argument_from_object(tool_name, arguments) + if text is not None: + return text + if partial: + return None + return json.dumps(arguments, ensure_ascii=False, separators=(",", ":")) + if partial: + return None + return str(arguments) + @classmethod def _raw_object_tool_arguments(cls, value: str) -> Optional[Dict[str, Any]]: candidates = [value] @@ -6950,13 +7072,13 @@ def _normalize_tool_call_item( "tool_calls function name must be a non-empty string" ) if self._tool_content_type(tool_name) == "text": - arguments = function.get("arguments", "") - if not isinstance(arguments, str): - if partial: - return None - raise CompletionResponseParsingError( - "tool_calls function arguments must be a string for text tools" - ) + arguments = self._text_tool_arguments( + tool_name, + function.get("arguments", ""), + partial=partial, + ) + if arguments is None: + return None return { "type": tool_call.get("type", "function"), "function": { @@ -7141,12 +7263,13 @@ def _parsed_chat_message( for tool_call_index, tool_call in enumerate(tool_calls): function = tool_call["function"] if self._tool_content_type(function["name"]) == "text": - raw_arguments = function["arguments"] - arguments = ( - raw_arguments - if isinstance(raw_arguments, str) - else str(raw_arguments) + arguments = self._text_tool_arguments( + function["name"], + function["arguments"], + partial=partial, ) + if arguments is None: + continue else: arguments = self._serialize_tool_arguments( function["arguments"], @@ -10195,7 +10318,7 @@ def _build_key( *, model_fingerprint: str, mmproj_fingerprint: str, - kind: Literal["image", "audio"], + kind: Literal["image", "audio", "video"], media_bytes: bytes, ) -> str: digest = hashlib.sha256(f"{kind}:".encode("utf-8") + media_bytes).hexdigest() @@ -10210,7 +10333,7 @@ def _build_key( ) return hashlib.sha256(payload.encode("utf-8")).hexdigest() - def key_for_media(self, kind: Literal["image", "audio"], media_bytes: bytes) -> str: + def key_for_media(self, kind: Literal["image", "audio", "video"], media_bytes: bytes) -> str: return self._build_key( model_fingerprint=self.model_fingerprint, mmproj_fingerprint=self.mmproj_fingerprint, @@ -10273,6 +10396,19 @@ def evict_if_needed(self) -> None: continue +@dataclass +class MTMDLoadedMedia: + media: MediaInput + media_bytes: bytes + key: str + bitmap: Any + video_ctx: Optional[Any] = None + video_temp_path: Optional[Path] = None + video_callback: Optional[Any] = None + video_frame_count: int = 0 + video_frames_used: int = 0 + + class MTMDProcessor: def __init__( self, @@ -10291,6 +10427,7 @@ def __init__( allowed_local_media_path: Optional[str], image_max_bytes: int, audio_max_bytes: int, + video_max_bytes: int, image_timeout_seconds: float, ) -> None: self.chat_formatter = chat_formatter @@ -10314,6 +10451,7 @@ def __init__( ) self.image_max_bytes = image_max_bytes self.audio_max_bytes = audio_max_bytes + self.video_max_bytes = video_max_bytes self.image_timeout_seconds = image_timeout_seconds self.lock = threading.Lock() params = mtmd_cpp.mtmd_context_params_default() @@ -10327,23 +10465,33 @@ def __init__( raise RuntimeError(f"failed to load MTMD context: {mmproj_path}") self.supports_vision = bool(mtmd_cpp.mtmd_support_vision(self.ctx)) self.supports_audio = bool(mtmd_cpp.mtmd_support_audio(self.ctx)) + self.supports_video = self.supports_vision and bool( + mtmd_cpp.mtmd_helper_support_video(self.ctx) + ) if not self.supports_vision and not self.supports_audio: mtmd_cpp.mtmd_free(self.ctx) self.ctx = None raise RuntimeError(f"MTMD projector does not support image or audio input: {mmproj_path}") - self.media_marker = mtmd_cpp.mtmd_default_marker().decode("utf-8") + media_marker = mtmd_cpp.mtmd_get_marker(self.ctx) + if media_marker is None: + mtmd_cpp.mtmd_free(self.ctx) + self.ctx = None + raise RuntimeError(f"MTMD projector does not expose a media marker: {mmproj_path}") + self.media_marker = media_marker.decode("utf-8") def close(self) -> None: if self.ctx is not None: mtmd_cpp.mtmd_free(self.ctx) self.ctx = None - def _max_bytes_for_media(self, kind: Literal["image", "audio"]) -> int: + def _max_bytes_for_media(self, kind: Literal["image", "audio", "video"]) -> int: if kind == "image": return self.image_max_bytes - return self.audio_max_bytes + if kind == "audio": + return self.audio_max_bytes + return self.video_max_bytes - def _load_media_file(self, kind: Literal["image", "audio"], media_url: str) -> bytes: + def _load_media_file(self, kind: Literal["image", "audio", "video"], media_url: str) -> bytes: if self.allowed_local_media_path is None: raise CompletionRequestValidationError("local media path is not allowed") parsed = urllib.parse.urlsplit(media_url) @@ -10391,7 +10539,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore opener = urllib.request.build_opener(NoRedirectHandler) return opener.open(request, timeout=timeout) - def _load_media_url(self, kind: Literal["image", "audio"], media_url: str) -> bytes: + def _load_media_url(self, kind: Literal["image", "audio", "video"], media_url: str) -> bytes: max_bytes = self._max_bytes_for_media(kind) if media_url.startswith("data:"): try: @@ -10423,29 +10571,134 @@ def _load_media_url(self, kind: Literal["image", "audio"], media_url: str) -> by def load_media(self, media: MediaInput) -> bytes: if media.url is not None: return self._load_media_url(media.kind, media.url) - if media.kind != "audio" or media.data is None: + if media.kind not in {"audio", "video"} or media.data is None: raise CompletionRequestValidationError(f"{media.kind} input requires a URL") try: data = base64.b64decode(media.data, validate=False) except (ValueError, binascii.Error) as exc: - raise CompletionRequestValidationError("input_audio data must be valid base64") from exc - if len(data) > self.audio_max_bytes: - raise CompletionRequestValidationError("audio exceeds model.mtmd.audio_max_bytes") + raise CompletionRequestValidationError(f"input_{media.kind} data must be valid base64") from exc + max_bytes = self._max_bytes_for_media(media.kind) + if len(data) > max_bytes: + raise CompletionRequestValidationError( + f"{media.kind} exceeds model.mtmd.{media.kind}_max_bytes" + ) return data - def _create_bitmap(self, media_bytes: bytes, kind: Literal["image", "audio"]) -> Any: + def _create_loaded_media( + self, + media: MediaInput, + media_bytes: bytes, + ) -> MTMDLoadedMedia: + key = ( + self.embedding_cache.key_for_media(media.kind, media_bytes) + if self.embedding_cache is not None + else MTMDEmbeddingCache._build_key( + model_fingerprint=self.model_fingerprint, + mmproj_fingerprint=self.mmproj_fingerprint, + kind=media.kind, + media_bytes=media_bytes, + ) + ) + if media.kind == "video": + return self._create_loaded_video_media(media, media_bytes, key) buffer = (ctypes.c_uint8 * len(media_bytes)).from_buffer_copy(media_bytes) - bitmap = mtmd_cpp.mtmd_helper_bitmap_init_from_buf( + wrapper = mtmd_cpp.mtmd_helper_bitmap_init_from_buf_wrapper( self.ctx, buffer, len(media_bytes), False, ) + bitmap = wrapper.bitmap if bitmap is None: - raise CompletionRequestValidationError(f"failed to create MTMD {kind} bitmap") - return bitmap + raise CompletionRequestValidationError(f"failed to create MTMD {media.kind} bitmap") + mtmd_cpp.mtmd_bitmap_set_id(bitmap, key.encode("utf-8")) + video_frame_count = 0 + video_ctx = wrapper.video_ctx + if video_ctx: + video_info = mtmd_cpp.mtmd_helper_video_get_info(video_ctx) + video_frame_count = max(0, int(video_info.n_frames)) + return MTMDLoadedMedia( + media=media, + media_bytes=media_bytes, + key=key, + bitmap=bitmap, + video_ctx=video_ctx, + video_frame_count=video_frame_count, + ) + + @staticmethod + def _video_temp_suffix(media: MediaInput) -> str: + extension = (media.format or "mp4").lstrip(".").lower() + if not extension or any(not char.isalnum() for char in extension): + extension = "video" + return f".{extension}" + + def _create_loaded_video_media( + self, + media: MediaInput, + media_bytes: bytes, + key: str, + ) -> MTMDLoadedMedia: + temp_file = tempfile.NamedTemporaryFile( + prefix="llama-cpp-python-mtmd-", + suffix=self._video_temp_suffix(media), + delete=False, + ) + temp_path = Path(temp_file.name) + try: + with temp_file: + temp_file.write(media_bytes) + params = mtmd_cpp.mtmd_helper_video_init_params_default() + video_ctx = mtmd_cpp.mtmd_helper_video_init( + self.ctx, + str(temp_path).encode("utf-8"), + params, + ) + if video_ctx is None: + raise CompletionRequestValidationError("failed to create MTMD video context") + + def read_next( + _chunk_index: int, + _user_data: Any, + out_bitmap: Any, + out_text: Any, + ) -> int: + return int(mtmd_cpp.mtmd_helper_video_read_next(video_ctx, out_bitmap, out_text)) + + callback = mtmd_cpp.mtmd_bitmap_lazy_callback(read_next) + bitmap = mtmd_cpp.mtmd_bitmap_init_lazy( + self.ctx, + key.encode("utf-8"), + ctypes.c_void_p(), + callback, + ) + if bitmap is None: + mtmd_cpp.mtmd_helper_video_free(video_ctx) + raise CompletionRequestValidationError("failed to create MTMD video bitmap") + video_info = mtmd_cpp.mtmd_helper_video_get_info(video_ctx) + return MTMDLoadedMedia( + media=media, + media_bytes=media_bytes, + key=key, + bitmap=bitmap, + video_ctx=video_ctx, + video_temp_path=temp_path, + video_callback=callback, + video_frame_count=max(0, int(video_info.n_frames)), + ) + except Exception: + try: + temp_path.unlink() + except OSError: + pass + raise - def _media_identity_tokens(self, kind: Literal["image", "audio"], key: str, n_pos: int) -> List[int]: + def _media_identity_tokens( + self, + kind: Literal["image", "audio", "video"], + key: str, + n_pos: int, + ) -> List[int]: tokens: List[int] = [] for index in range(n_pos): digest = hashlib.sha256(f"{kind}:{key}:{index}".encode("utf-8")).digest() @@ -10455,7 +10708,7 @@ def _media_identity_tokens(self, kind: Literal["image", "audio"], key: str, n_po def _encode_media_chunk( self, *, - kind: Literal["image", "audio"], + kind: Literal["image", "audio", "video"], key: str, chunk: Any, ) -> np.ndarray: @@ -10535,6 +10788,8 @@ def build_prompt_plan( raise CompletionRequestValidationError("MTMD projector does not support images") if any(media.kind == "audio" for media in media_inputs) and not self.supports_audio: raise CompletionRequestValidationError("MTMD projector does not support audio") + if any(media.kind == "video" for media in media_inputs) and not self.supports_video: + raise CompletionRequestValidationError("MTMD projector does not support video") with self.lock: return self._build_prompt_plan_locked( messages=messages, @@ -10567,13 +10822,19 @@ def _build_prompt_plan_locked( reasoning_effort=reasoning_effort, ) media_bytes_by_index = [self.load_media(media) for media in media_inputs] - bitmaps: List[Any] = [] + loaded_media: List[MTMDLoadedMedia] = [] chunks: Optional[Any] = None try: - bitmaps = [ - self._create_bitmap(media_bytes, media.kind) + loaded_media = [ + self._create_loaded_media(media, media_bytes) for media, media_bytes in zip(media_inputs, media_bytes_by_index) ] + loaded_media_by_key = {media.key: media for media in loaded_media} + video_media = [media for media in loaded_media if media.media.kind == "video"] + if len(video_media) > 1 and any(media.video_frame_count <= 0 for media in video_media): + raise CompletionRequestValidationError( + "multiple videos require MTMD to report frame counts" + ) input_text = mtmd_cpp.mtmd_input_text() input_text.text = prompt.encode("utf-8") input_text.add_special = False @@ -10581,14 +10842,16 @@ def _build_prompt_plan_locked( chunks = mtmd_cpp.mtmd_input_chunks_init() if chunks is None: raise CompletionRequestValidationError("failed to create MTMD input chunks") - bitmap_array = (mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps) + bitmap_array = (mtmd_cpp.mtmd_bitmap_p_ctypes * len(loaded_media))( + *(media.bitmap for media in loaded_media) + ) result = int( mtmd_cpp.mtmd_tokenize( self.ctx, chunks, ctypes.byref(input_text), bitmap_array, - len(bitmaps), + len(loaded_media), ) ) if result != 0: @@ -10601,7 +10864,8 @@ def _build_prompt_plan_locked( text_token_index_by_pos: Dict[int, int] = {} identity_pos = 0 decode_pos = 0 - media_index = 0 + video_index = 0 + used_media_keys = set() n_chunks = int(mtmd_cpp.mtmd_input_chunks_size(chunks)) for chunk_index in range(n_chunks): chunk = mtmd_cpp.mtmd_input_chunks_get(chunks, chunk_index) @@ -10640,32 +10904,50 @@ def _build_prompt_plan_locked( decode_pos += len(tokens) continue if chunk_type == mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_IMAGE: - kind: Literal["image", "audio"] = "image" + chunk_kind: Literal["image", "audio"] = "image" if not self.supports_vision: raise CompletionRequestValidationError("MTMD projector does not support images") elif chunk_type == mtmd_cpp.MTMD_INPUT_CHUNK_TYPE_AUDIO: - kind = "audio" + chunk_kind = "audio" if not self.supports_audio: raise CompletionRequestValidationError("MTMD projector does not support audio") else: raise CompletionRequestValidationError("unsupported MTMD input chunk type") - if media_index >= len(media_bytes_by_index): - raise CompletionRequestValidationError("MTMD media chunk count mismatch") - media = media_inputs[media_index] - media_bytes = media_bytes_by_index[media_index] - media_index += 1 - if media.kind != kind: - raise CompletionRequestValidationError("MTMD media chunk modality mismatch") - key = ( - self.embedding_cache.key_for_media(kind, media_bytes) - if self.embedding_cache is not None - else MTMDEmbeddingCache._build_key( - model_fingerprint=self.model_fingerprint, - mmproj_fingerprint=self.mmproj_fingerprint, - kind=kind, - media_bytes=media_bytes, - ) - ) + chunk_id_bytes = mtmd_cpp.mtmd_input_chunk_get_id(chunk) + chunk_id = chunk_id_bytes.decode("utf-8") if chunk_id_bytes else "" + media = loaded_media_by_key.get(chunk_id) + video_frame_index: Optional[int] = None + if media is None and chunk_kind == "image" and video_media: + while ( + video_index < len(video_media) + and video_media[video_index].video_frame_count > 0 + and video_media[video_index].video_frames_used + >= video_media[video_index].video_frame_count + ): + video_index += 1 + if video_index >= len(video_media): + raise CompletionRequestValidationError("MTMD video frame count mismatch") + media = video_media[video_index] + video_frame_index = media.video_frames_used + media.video_frames_used += 1 + if media is None: + raise CompletionRequestValidationError("MTMD media chunk identity mismatch") + if media.media.kind == "video": + if chunk_kind != "image": + raise CompletionRequestValidationError("MTMD video chunk modality mismatch") + kind: Literal["image", "audio", "video"] = "video" + if video_frame_index is None: + video_frame_index = media.video_frames_used + media.video_frames_used += 1 + key = hashlib.sha256( + f"{media.key}:frame:{video_frame_index}".encode("utf-8") + ).hexdigest() + else: + if media.media.kind != chunk_kind: + raise CompletionRequestValidationError("MTMD media chunk modality mismatch") + kind = media.media.kind + key = media.key + used_media_keys.add(media.key) decode_n_pos = int(mtmd_cpp.mtmd_input_chunk_get_n_pos(chunk)) if decode_n_pos <= 0: raise CompletionRequestValidationError("MTMD media chunk has no decoder positions") @@ -10698,7 +10980,7 @@ def _build_prompt_plan_locked( identity_tokens.extend(segment_identity) identity_pos += n_tokens decode_pos += decode_n_pos - if media_index != len(media_bytes_by_index): + if used_media_keys != {media.key for media in loaded_media}: raise CompletionRequestValidationError("not all media inputs were consumed by MTMD") return PromptPlan( text=prompt, @@ -10711,8 +10993,15 @@ def _build_prompt_plan_locked( finally: if chunks is not None: mtmd_cpp.mtmd_input_chunks_free(chunks) - for bitmap in bitmaps: - mtmd_cpp.mtmd_bitmap_free(bitmap) + for media in loaded_media: + mtmd_cpp.mtmd_bitmap_free(media.bitmap) + if media.video_ctx: + mtmd_cpp.mtmd_helper_video_free(media.video_ctx) + if media.video_temp_path is not None: + try: + media.video_temp_path.unlink() + except OSError: + pass class Model: @@ -15927,6 +16216,7 @@ def main() -> None: allowed_local_media_path=config.model.mtmd.allowed_local_media_path, image_max_bytes=config.model.mtmd.image_max_bytes, audio_max_bytes=config.model.mtmd.audio_max_bytes, + video_max_bytes=config.model.mtmd.video_max_bytes, image_timeout_seconds=config.model.mtmd.image_timeout_seconds, ) sequence_cache: Optional[SequenceCache] = None From 0edb5d8d2d68461e706f4b2ecfb69060e4e99486 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Tue, 9 Jun 2026 00:19:48 -0700 Subject: [PATCH 559/561] feat: update llama.cpp to ggml-org/llama.cpp@e3471b3e7 (#2294) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e733f37e6e..7cc732b935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat(example): support server video inputs and Gemma text tool calls by @abetlen in #2291 -- feat: update llama.cpp to ggml-org/llama.cpp@8f83d6c27 +- feat: update llama.cpp to ggml-org/llama.cpp@e3471b3e7 - fix(example): support multi-step Responses tool streaming by @abetlen in #2288 - fix(ci): Repair Linux accelerator wheels for manylinux publishing diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8f83d6c271..e3471b3e73 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8f83d6c271d194bde2d410145a0ce73bc42e85cd +Subproject commit e3471b3e7306fe120dc8f38a2263c1293fc2add7 From b5eefc82e0fd415d5547c81367c29b159c0268d3 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Wed, 10 Jun 2026 00:44:25 -0700 Subject: [PATCH 560/561] feat: update llama.cpp to ggml-org/llama.cpp@76da2450a (#2295) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cc732b935..1fc8750c87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat(example): support server video inputs and Gemma text tool calls by @abetlen in #2291 -- feat: update llama.cpp to ggml-org/llama.cpp@e3471b3e7 +- feat: update llama.cpp to ggml-org/llama.cpp@76da2450a - fix(example): support multi-step Responses tool streaming by @abetlen in #2288 - fix(ci): Repair Linux accelerator wheels for manylinux publishing diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e3471b3e73..76da2450a4 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e3471b3e7306fe120dc8f38a2263c1293fc2add7 +Subproject commit 76da2450a4f2cc9ce6c7fc8229e25dc0a4b41e5d From 19ea70cff57e2f60ee333fbe515640e53c83f994 Mon Sep 17 00:00:00 2001 From: Andrei <abetlen@gmail.com> Date: Wed, 10 Jun 2026 21:49:38 -0700 Subject: [PATCH 561/561] feat: update llama.cpp to ggml-org/llama.cpp@ac4cddeb0 (#2297) --- CHANGELOG.md | 2 +- vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fc8750c87..5c4c744d9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - feat(example): support server video inputs and Gemma text tool calls by @abetlen in #2291 -- feat: update llama.cpp to ggml-org/llama.cpp@76da2450a +- feat: update llama.cpp to ggml-org/llama.cpp@ac4cddeb0 - fix(example): support multi-step Responses tool streaming by @abetlen in #2288 - fix(ci): Repair Linux accelerator wheels for manylinux publishing diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 76da2450a4..ac4cddeb0d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 76da2450a4f2cc9ce6c7fc8229e25dc0a4b41e5d +Subproject commit ac4cddeb0dbd778f650bf568f6f08344a06abe3a