Mastodon Feed: Post

Mastodon Feed

jonny@neuromatch.social ("jonny (nonvenomous)") wrote:

here's the MWE for plain inference as plaintext and highlighted picture of text:

import json, socket, struct, uuid
`SOCK = "/run/hatch/sandbox/space-inference.sock"`
def call(req, timeout=60):
payload = json.dumps(req).encode()
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.settimeout(timeout)
s.connect(SOCK)
s.sendall(struct.pack(">I", len(payload)) + payload)
(n,) = struct.unpack(">I", s.recv(4))
body = b""
while len(body) < n:
chunk = s.recv(n - len(body))
if not chunk:
raise RuntimeError("truncated response")
body += chunk
s.close()
return json.loads(body)
`resp = call({` ` "kind": "complete",` ` "request_id": f"mwe-{uuid.uuid4().hex[:8]}",` ` "slug": "__mwe__",` ` "system": "Always answer in exactly three words.", ` ` "prompt": "What is the capital of France?",` ` "json_schema": {` ` "type": "object",` ` "properties": {"reply": {"type": "string"}},` ` "required": ["reply"],` ` "additionalProperties": False,` ` },` `})`
print(resp["ok"])
print(resp["result"]["content"])
print(resp["result"].get("usage"))

[the code in the post, just colorful] import json, socket, struct, uuid SOCK = "/run/hatch/sandbox/space-inference.sock" def call(req, timeout=60): payload = json.dumps(req).encode() s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s.settimeout(timeout) s.connect(SOCK) s.sendall(struct.pack(">I", len(payload)) + payload) (n,) = struct.unpack(">I", s.recv(4)) body = b"" while len(body) < n: chunk = s.recv(n - len(body)) if not chunk: raise RuntimeError("truncated response") body += chunk s.close() return json.loads(body) resp = call({ "kind": "complete", "request_id": f"mwe-{uuid.uuid4().hex[:8]}", "slug": "mwe", "system": "Always answer in exactly three words.", "prompt": "What is the capital of France?", "json_schema": { "type": "object", "properties": {"reply": {"type": "string"}}, "required": ["reply"], "additionalProperties": False, }, }) print(resp["ok"]) print(resp["result"]["content"]) print(resp["result"].get("usage"))
[invoking the script, demonstrating that it shows expected output] True {'reply': 'Capital is Paris'} {'input_tokens': 970, 'output_tokens': 1226, 'total_tokens': 2196}