-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathserver.py
More file actions
44 lines (37 loc) 路 1.67 KB
/
server.py
File metadata and controls
44 lines (37 loc) 路 1.67 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
from http.server import BaseHTTPRequestHandler, HTTPServer
import json
import subprocess
from urllib.parse import urlparse, parse_qs
HOST = "0.0.0.0"
PORT = 8000
class Handler(BaseHTTPRequestHandler):
def _set_headers(self):
self.send_header("Content-Type", "application/json")
self.send_header("Access-Control-Allow-Origin", "*")
self.send_header("Access-Control-Allow-Methods", "POST, OPTIONS")
self.send_header("Access-Control-Allow-Headers", "Content-Type")
def do_OPTIONS(self):
self.send_response(200)
self._set_headers()
self.end_headers()
def do_POST(self):
if self.path != "/query":
self.send_response(404)
self.end_headers()
return
length = int(self.headers['Content-Length'])
body = self.rfile.read(length)
data = json.loads(body)
prompt = data.get("prompt","")
result = subprocess.run(
["llama.cpp/build/bin/llama-simple", "-m", "llama.cpp/models/tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf", "-p", prompt],
capture_output=True, text=True
)
response = {"response": result.stdout}
self.send_response(200)
self._set_headers()
self.end_headers()
self.wfile.write(json.dumps(response).encode("utf-8"))
httpd = HTTPServer((HOST,PORT), Handler)
print(f"Server running on http://{HOST}:{PORT}")
httpd.serve_forever()