lidanger
diff --git a/‎llama_cpp/server/__main__.py
Copy file name to clipboardExpand all lines: llama_cpp/server/__main__.py
+2-1Lines changed: 2 additions & 1 deletion b/‎llama_cpp/server/__main__.py
Copy file name to clipboardExpand all lines: llama_cpp/server/__main__.py
+2-1Lines changed: 2 additions & 1 deletion
diff --git a/‎llama_cpp/server/app.py
Copy file name to clipboardExpand all lines: llama_cpp/server/app.py
+23-20Lines changed: 23 additions & 20 deletions b/‎llama_cpp/server/app.py
Copy file name to clipboardExpand all lines: llama_cpp/server/app.py
+23-20Lines changed: 23 additions & 20 deletions
diff --git a/‎tests/test_llama.py
Copy file name to clipboardExpand all lines: tests/test_llama.py
+5-4Lines changed: 5 additions & 4 deletions b/‎tests/test_llama.py
Copy file name to clipboardExpand all lines: tests/test_llama.py
+5-4Lines changed: 5 additions & 4 deletions
@@ -24,9 +24,10 @@
 import os
 import uvicorn
 
-from llama_cpp.server.app import app
+from llama_cpp.server.app import app, init_llama
 
 if __name__ == "__main__":
+    init_llama()
 
     uvicorn.run(
         app, host=os.getenv("HOST", "localhost"), port=int(os.getenv("PORT", 8000))
 
@@ -13,7 +13,7 @@
 
 
 class Settings(BaseSettings):
-    model: str = os.environ["MODEL"]
+    model: str = os.environ.get("MODEL", "null")
     n_ctx: int = 2048
     n_batch: int = 512
     n_threads: int = max((os.cpu_count() or 2) // 2, 1)
@@ -38,31 +38,34 @@ class Settings(BaseSettings):
     allow_methods=["*"],
     allow_headers=["*"],
 )
-settings = Settings()
-llama = llama_cpp.Llama(
-    settings.model,
-    f16_kv=settings.f16_kv,
-    use_mlock=settings.use_mlock,
-    use_mmap=settings.use_mmap,
-    embedding=settings.embedding,
-    logits_all=settings.logits_all,
-    n_threads=settings.n_threads,
-    n_batch=settings.n_batch,
-    n_ctx=settings.n_ctx,
-    last_n_tokens_size=settings.last_n_tokens_size,
-    vocab_only=settings.vocab_only,
-)
-if settings.cache:
-    cache = llama_cpp.LlamaCache()
-    llama.set_cache(cache)
-llama_lock = Lock()
 
+llama: llama_cpp.Llama = None
+def init_llama(settings: Settings = None):
+    if settings is None:
+        settings = Settings()
+    global llama
+    llama = llama_cpp.Llama(
+        settings.model,
+        f16_kv=settings.f16_kv,
+        use_mlock=settings.use_mlock,
+        use_mmap=settings.use_mmap,
+        embedding=settings.embedding,
+        logits_all=settings.logits_all,
+        n_threads=settings.n_threads,
+        n_batch=settings.n_batch,
+        n_ctx=settings.n_ctx,
+        last_n_tokens_size=settings.last_n_tokens_size,
+        vocab_only=settings.vocab_only,
+    )
+    if settings.cache:
+        cache = llama_cpp.LlamaCache()
+        llama.set_cache(cache)
 
+llama_lock = Lock()
 def get_llama():
     with llama_lock:
         yield llama
 
-
 class CreateCompletionRequest(BaseModel):
     prompt: Union[str, List[str]]
     suffix: Optional[str] = Field(None)
 
@@ -132,10 +132,11 @@ def mock_sample(*args, **kwargs):
 
 def test_llama_server():
     from fastapi.testclient import TestClient
-    import os
-    os.environ["MODEL"] = MODEL
-    os.environ["VOCAB_ONLY"] = "true"
-    from llama_cpp.server.app import app
+    from llama_cpp.server.app import app, init_llama, Settings
+    s = Settings()
+    s.model = MODEL
+    s.vocab_only = True
+    init_llama(s)
     client = TestClient(app)
     response = client.get("/v1/models")
     assert response.json() == {