Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit aacdbd4

Browse filesBrowse files
authored
llama : fix params struct slignment (ggml-org#1936)
* Workaround struct misalignment during value-copy Signed-off-by: mudler <mudler@localai.io> * Move booleans at the bottom of the structure Signed-off-by: mudler <mudler@localai.io> * Add comment Signed-off-by: mudler <mudler@localai.io> --------- Signed-off-by: mudler <mudler@localai.io>
1 parent 20568fe commit aacdbd4
Copy full SHA for aacdbd4

File tree

Expand file treeCollapse file tree

2 files changed

+11
-12
lines changed
Filter options
Expand file treeCollapse file tree

2 files changed

+11
-12
lines changed

‎llama.cpp

Copy file name to clipboardExpand all lines: llama.cpp
+3-3Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -925,21 +925,21 @@ static bool kv_cache_init(
925925

926926
struct llama_context_params llama_context_default_params() {
927927
struct llama_context_params result = {
928+
/*.seed =*/ -1,
928929
/*.n_ctx =*/ 512,
929930
/*.n_batch =*/ 512,
930931
/*.gpu_layers =*/ 0,
931932
/*.main_gpu =*/ 0,
932933
/*.tensor_split =*/ {0},
934+
/*.progress_callback =*/ nullptr,
935+
/*.progress_callback_user_data =*/ nullptr,
933936
/*.low_vram =*/ false,
934-
/*.seed =*/ -1,
935937
/*.f16_kv =*/ true,
936938
/*.logits_all =*/ false,
937939
/*.vocab_only =*/ false,
938940
/*.use_mmap =*/ true,
939941
/*.use_mlock =*/ false,
940942
/*.embedding =*/ false,
941-
/*.progress_callback =*/ nullptr,
942-
/*.progress_callback_user_data =*/ nullptr,
943943
};
944944

945945
return result;

‎llama.h

Copy file name to clipboardExpand all lines: llama.h
+8-9Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -71,28 +71,27 @@ extern "C" {
7171

7272
typedef void (*llama_progress_callback)(float progress, void *ctx);
7373

74-
struct llama_context_params {
74+
struct llama_context_params {
75+
int seed; // RNG seed, -1 for random
7576
int n_ctx; // text context
7677
int n_batch; // prompt processing batch size
7778
int n_gpu_layers; // number of layers to store in VRAM
7879
int main_gpu; // the GPU that is used for scratch and small tensors
7980
float tensor_split[LLAMA_MAX_DEVICES]; // how to split layers across multiple GPUs
80-
bool low_vram; // if true, reduce VRAM usage at the cost of performance
81-
int seed; // RNG seed, -1 for random
81+
// called with a progress value between 0 and 1, pass NULL to disable
82+
llama_progress_callback progress_callback;
83+
// context pointer passed to the progress callback
84+
void * progress_callback_user_data;
8285

86+
// Keep the booleans together to avoid misalignment during copy-by-value.
87+
bool low_vram; // if true, reduce VRAM usage at the cost of performance
8388
bool f16_kv; // use fp16 for KV cache
8489
bool logits_all; // the llama_eval() call computes all logits, not just the last one
8590
bool vocab_only; // only load the vocabulary, no weights
8691
bool use_mmap; // use mmap if possible
8792
bool use_mlock; // force system to keep model in RAM
8893
bool embedding; // embedding mode only
89-
90-
// called with a progress value between 0 and 1, pass NULL to disable
91-
llama_progress_callback progress_callback;
92-
// context pointer passed to the progress callback
93-
void * progress_callback_user_data;
9494
};
95-
9695
// model file types
9796
enum llama_ftype {
9897
LLAMA_FTYPE_ALL_F32 = 0,

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.