// app/lib/llm.ts
import 'server-only';
import OpenAI from 'openai';

export function llmClient(): OpenAI {
  return new OpenAI({
    baseURL: `${process.env.LLM_URL}/v1`,
    apiKey: process.env.LLM_API_KEY ?? 'dummy',
  });
}

export async function chat(prompt: string): Promise<string> {
  const r = await llmClient().chat.completions.create({
    model: process.env.LLM_MODEL ?? 'mlx-community/Qwen3-4B-Instruct-2507-4bit',
    messages: [{ role: 'user', content: prompt }],
    stream: false,
  });
  return r.choices[0]?.message?.content ?? '';
}

import type { NextRequest } from 'next/server';

export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';

export async function POST(req: NextRequest): Promise<Response> {
  const body = await req.json();
  const upstream = await fetch(`${process.env.LLM_URL}/v1/chat/completions`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'X-API-Key': process.env.LLM_API_KEY!,
    },
    body: JSON.stringify({ ...body, stream: true }),
    cache: 'no-store',
  });
  if (!upstream.ok || !upstream.body) {
    return new Response(`llm upstream ${upstream.status}`, { status: 502 });
  }
  return new Response(upstream.body, {
    status: 200,
    headers: {
      'Content-Type': 'text/event-stream',
      'Cache-Control': 'no-cache, no-transform',
      'Connection': 'keep-alive',
      'X-Accel-Buffering': 'no',
    },
  });
}

// src/modules/llm/llm.service.ts
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';

export interface ChatMessage {
  role: 'system' | 'user' | 'assistant' | 'tool';
  content: string;
  tool_call_id?: string;
}

export interface ChatRequest {
  messages: ChatMessage[];
  model?: string;
  temperature?: number;
  max_tokens?: number;
  tools?: ChatTool[];
  stream?: boolean;
}

export interface ChatTool {
  type: 'function';
  function: { name: string; description?: string; parameters: object };
}

export interface ChatResponse {
  id: string;
  choices: Array<{
    message: { role: string; content: string | null; tool_calls?: ToolCall[] };
    finish_reason: string;
  }>;
}

export interface ToolCall {
  id: string;
  type: 'function';
  function: { name: string; arguments: string };
}

@Injectable()
export class LlmService {
  constructor(private readonly cfg: ConfigService) {}

  async chat(req: ChatRequest): Promise<ChatResponse> {
    const r = await fetch(`${this.cfg.get('LLM_URL')}/v1/chat/completions`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'X-API-Key': this.cfg.getOrThrow('LLM_API_KEY'),
      },
      body: JSON.stringify({ ...req, stream: false }),
    });
    if (!r.ok) throw new Error(`llm ${r.status}`);
    return (await r.json()) as ChatResponse;
  }

  async stream(req: ChatRequest, sink: NodeJS.WritableStream): Promise<void> {
    const r = await fetch(`${this.cfg.get('LLM_URL')}/v1/chat/completions`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'X-API-Key': this.cfg.getOrThrow('LLM_API_KEY'),
      },
      body: JSON.stringify({ ...req, stream: true }),
    });
    if (!r.ok || !r.body) throw new Error(`llm ${r.status}`);
    for await (const chunk of r.body as unknown as AsyncIterable<Uint8Array>) {
      sink.write(chunk);
    }
    sink.end();
  }
}

from openai import AsyncOpenAI

client = AsyncOpenAI(
    base_url="http://127.0.0.1:4250/v1",
    api_key="dummy",
)

async def ask(prompt: str) -> str:
    r = await client.chat.completions.create(
        model="mlx-community/Qwen3-4B-Instruct-2507-4bit",
        messages=[{"role": "user", "content": prompt}],
    )
    return r.choices[0].message.content or ""

async def stream(prompt: str):
    async for chunk in await client.chat.completions.create(
        model="mlx-community/Qwen3-4B-Instruct-2507-4bit",
        messages=[{"role": "user", "content": prompt}],
        stream=True,
    ):
        delta = chunk.choices[0].delta.content
        if delta:
            yield delta

import httpx

async def swap_model(model_id: str) -> dict:
    async with httpx.AsyncClient(timeout=120) as cli:
        r = await cli.post(
            "http://127.0.0.1:4250/v1/models/load",
            headers={"X-API-Key": "dummy"},
            json={"model_id": model_id},
        )
        r.raise_for_status()
        return r.json()  # { swapped, load_ms, loaded }

{
  "choices": [{
    "message": {
      "role": "assistant",
      "content": null,
      "tool_calls": [{
        "id": "call_0",
        "type": "function",
        "function": { "name": "get_weather", "arguments": "{\"city\":\"Berlin\"}" }
      }]
    },
    "finish_reason": "tool_calls"
  }]
}