Simply Patrick Hopes can always go up, tears can only come down.

{
  "model": "gemma-4-E4B-it",
  "messages": [
    {
      "role": "user",
      "content": "請解釋什麼是 Rust 所有權。"
    }
  ]
}

{
  "model": "llava-v1.5-7b",
  "messages": [
    {
      "role": "user",
      "content": [
        {
          "type": "text",
          "text": "這張圖片中的水果叫什麼名字？"
        },
        {
          "type": "image_url",
          "image_url": {
            "url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEASABIAAD..."
          }
        }
      ]
    }
  ]
}

原始 Token 向量序列:
[ <bos>, <image>, "這", "張", "圖", "片", "中", "的", ... ]
    │        │
    │        └─ 被替換為由 ViT + Projector 生成的 576 個視覺特徵向量 (4096-dim)
    ▼
融合後的 Embedding 序列直接送入 LLM context 進行 evaluation

use rust_embed::RustEmbed;

#[derive(RustEmbed)]
#[folder = "ui/dist/"]
struct Asset;

use axum::{
    http::{header, StatusCode, Uri},
    response::IntoResponse,
};
use rust_embed::RustEmbed;

#[derive(RustEmbed)]
#[folder = "ui/dist/"]
struct Asset;

pub async fn static_handler(uri: Uri) -> impl IntoResponse {
    // 1. 去除請求路徑開頭的斜線，符合 rust-embed 的鍵值格式
    let mut path = uri.path().trim_start_matches('/').to_string();

    // 2. 如果是首頁請求，預設尋找 index.html
    if path.is_empty() {
        path = "index.html".to_string();
    }

    // 3. 嘗試從嵌入資源中取得檔案
    match Asset::get(path.as_str()) {
        Some(content) => {
            // 使用 mime_guess 根據副檔名判斷正確的 Content-Type (例如 text/css, application/javascript)
            let mime = mime_guess::from_path(path).first_or_octet_stream();
            (
                [(header::CONTENT_TYPE, mime.as_ref())],
                content.data.into_owned(),
            )
                .into_response()
        }
        None => {
            // 4. 找不到檔案時，Fallback 回傳 index.html（讓前端 SPA 處理路由）
            if let Some(index_content) = Asset::get("index.html") {
                (
                    [(header::CONTENT_TYPE, "text/html")],
                    index_content.data.into_owned(),
                )
                    .into_response()
            } else {
                // 如果連 index.html 都沒有（例如忘記先 build 前端），則回傳 404
                (StatusCode::NOT_FOUND, "404 Not Found").into_response()
            }
        }
    }
}

let app = Router::new()
    .route("/health", get(routes::health))
    .route("/v1/models", get(routes::list_models))
    .route("/v1/chat/completions", post(routes::chat_completions))
    // 當以上路由都沒配對成功時，交給靜態資源處理器
    .fallback(get(assets::static_handler))
    .layer(cors)
    .with_state(engine);

// 建立一個空的 AI 訊息元素，並附加閃爍的輸入游標 (cursor-blink)
const msgEl = document.createElement('div');
msgEl.className = 'message ai';
const contentEl = document.createElement('div');
contentEl.className = 'message-content';

const cursor = document.createElement('span');
cursor.className = 'cursor-blink';

msgEl.appendChild(contentEl);
msgEl.appendChild(cursor);
chatHistory.appendChild(msgEl);

let aiContent = '';

// 發送請求並讀取串流
const response = await fetch('/v1/chat/completions', {
  method: 'POST',
  headers: { 'Content-Type': 'application/json' },
  body: JSON.stringify({
    model: selectedModel || 'default',
    messages: messages,
    stream: true,
    max_tokens: 512
  })
});

const reader = response.body.getReader();
const decoder = new TextDecoder('utf-8');

while (true) {
  const { done, value } = await reader.read();
  if (done) break;

  // 解碼串流 byte 資料為文字
  const chunk = decoder.decode(value, { stream: true });
  const lines = chunk.split('\n');

  for (const line of lines) {
    if (line.startsWith('data: ')) {
      const dataStr = line.replace('data: ', '').trim();
      if (dataStr === '[DONE]') continue;
      
      if (dataStr) {
        try {
          const data = JSON.parse(dataStr);
          if (data.choices && data.choices[0].delta.content) {
            // 累加 token 片段並即時渲染到介面
            aiContent += data.choices[0].delta.content;
            contentEl.textContent = aiContent; // 使用 textContent 防範 XSS 攻擊
            chatHistory.scrollTop = chatHistory.scrollHeight; // 自動滾動到底部
          }
        } catch (e) {
          console.error('Error parsing SSE JSON:', e);
        }
      }
    }
  }
}

// 結束後移除閃爍游標
cursor.remove();
messages.push({ role: 'assistant', content: aiContent });

# 直接啟動服務（Rust 會自動接管 npm 前端編譯打包）
cargo run -- serve gemma-4-E4B-it --port 8080

Starting llm-local-studio server (axum)...
  Base URL: http://127.0.0.1:8080
  Endpoints:
    GET  /health
    GET  /v1/models
    POST /v1/chat/completions
    GET  / (Web UI)

cargo run --release -- serve gemma-4-e4b --port 8080

{
  "id": "chatcmpl-c513296e-7eb2-4f66-9af7-045fbae2fa36",
  "object": "chat.completion",
  "created": 1779608333,
  "model": "gemma-4-e4b",
  "choices": [
    {
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "Hello! I am a local assistant."
      },
      "finish_reason": "stop"
    }
  ],
  "usage": {
    "prompt_tokens": 12,
    "completion_tokens": 8,
    "total_tokens": 20
  }
}

data: {"id":"chatcmpl-...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}

data: {"id":"chatcmpl-...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}

data: {"id":"chatcmpl-...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}

pip install openai

from openai import OpenAI

# 指向本機伺服器的 Base URL，api_key 填入任意值即可
client = OpenAI(
    base_url="http://localhost:8080/v1",
    api_key="local-studio"
)

response = client.chat.completions.create(
    model="gemma-4-e4b",
    messages=[
        {"role": "user", "content": "Explain ownership in Rust programming"}
    ],
    max_tokens=80
)

print(response.choices[0].message.content)

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8080/v1",
    api_key="local-studio"
)

# 啟用 stream=True 進行打字機效果輸出
stream = client.chat.completions.create(
    model="gemma-4-e4b",
    messages=[
        {"role": "user", "content": "Explain ownership in Rust programming"}
    ],
    max_tokens=80,
    stream=True
)

for chunk in stream:
    # 讀取 delta 增量內容並即時印出
    content = chunk.choices[0].delta.content
    if content is not None:
        print(content, end="", flush=True)
print()

npm install openai

import OpenAI from "openai";

const openai = new OpenAI({
  baseURL: "http://localhost:8080/v1",
  apiKey: "local-studio",
});

async function main() {
  const stream = await openai.chat.completions.create({
    model: "gemma-4-e4b",
    messages: [
      { role: "user", content: "Explain ownership in Rust programming" }
    ],
    max_tokens: 80,
    stream: true,
  });

  for await (const chunk of stream) {
    const content = chunk.choices[0]?.delta?.content || "";
    process.stdout.write(content);
  }
  process.stdout.write("\n");
}

main().catch(console.error);

[dependencies]
async-openai = "0.26"
tokio = { version = "1", features = ["full"] }
futures = "0.3"

use async_openai::{
    config::OpenAIConfig,
    types::{CreateChatCompletionRequestArgs, ChatCompletionRequestMessage},
    Client,
};
use futures::StreamExt;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    // 1. 自訂配置，將 api_base 指向本機端點
    let config = OpenAIConfig::default()
        .with_api_base("http://localhost:8080/v1")
        .with_api_key("local-studio");

    let client = Client::with_config(config);

    // 2. 建立 Request 參數
    let request = CreateChatCompletionRequestArgs::default()
        .model("gemma-4-e4b")
        .max_tokens(80u32)
        .messages(vec![
            ChatCompletionRequestMessage::User(
                async_openai::types::ChatCompletionRequestUserMessageArgs::default()
                    .content("Explain ownership in Rust programming")
                    .build()?,
            )
        ])
        .stream(true)
        .build()?;

    // 3. 獲取非同步 Stream 並依序讀取 token
    let mut stream = client.chat().create_stream(request).await?;

    while let Some(result) = stream.next().await {
        match result {
            Ok(response) => {
                for choice in response.choices {
                    if let Some(content) = choice.delta.content {
                        print!("{content}");
                        std::io::Write::flush(&mut std::io::stdout())?;
                    }
                }
            }
            Err(err) => eprintln!("Error: {err}"),
        }
    }
    println!();

    Ok(())
}

curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma-4-e4b",
    "messages": [
      {"role": "user", "content": "Explain ownership in Rust programming"}
    ],
    "max_tokens": 80
  }'

curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma-4-e4b",
    "messages": [
      {"role": "user", "content": "Hello!"}
    ],
    "stream": true,
    "max_tokens": 30
  }'

{
  "models": [
    {
      "title": "Local Studio - Gemma",
      "provider": "openai",
      "model": "gemma-4-e4b",
      "apiBase": "http://localhost:8080/v1",
      "apiKey": "local-studio"
    }
  ],
  "tabAutocompleteModel": {
    "title": "Local Studio - Gemma Autocomplete",
    "provider": "openai",
    "model": "gemma-4-e4b",
    "apiBase": "http://localhost:8080/v1",
    "apiKey": "local-studio"
  }
}

/// 執行緒安全、對非同步友善的推理服務
#[derive(Clone)]
pub struct EngineService {
    inner: Arc<Mutex<LlamaCppEngine>>,
}

pub fn chat_streaming(
    &self,
    messages: Vec<ChatMessage>,
    max_tokens: u32,
    seed: Option<u32>,
) -> (mpsc::Receiver<String>, tokio::task::JoinHandle<Result<GenerateOutput>>) {
    // 1. 建立非同步多生產者單消費者管道，緩衝區設為 64
    let (tx, rx) = mpsc::channel::<String>(64);
    let engine = self.inner.clone();

    // 2. 在 blocking thread pool 中啟動推理任務
    let handle = tokio::task::spawn_blocking(move || {
        let mut engine = engine.lock().expect("engine mutex poisoned");
        let tx_clone = tx.clone();
        
        let result = engine.chat(ChatRequest {
            messages,
            max_tokens,
            seed,
            // 3. 當底層生成一個 token piece 時，觸發此 FFI 回呼
            stream_callback: Some(Box::new(move |piece: &str| {
                // 如果用戶端已斷開連接（Receiver 已 drop），我們忽略錯誤並繼續
                let _ = tx_clone.blocking_send(piece.to_owned());
            })),
        });
        
        // 4. 任務結束時主動 drop(tx)，使 Receiver 收到 None 訊號得知結束
        drop(tx);
        result
    });

    (rx, handle)
}

pub async fn start_server(config: ServerConfig, engine: EngineService) -> Result<()> {
    let bind_addr = format!("{}:{}", config.host, config.port);

    // 允許任何來源的 CORS 設定，方便第三方 Web UI 連接
    let cors = CorsLayer::new()
        .allow_origin(Any)
        .allow_methods(Any)
        .allow_headers(Any);

    let app = Router::new()
        .route("/health", get(routes::health))
        .route("/v1/models", get(routes::list_models))
        .route("/v1/chat/completions", post(routes::chat_completions))
        .layer(cors)
        .with_state(engine);

    let listener = tokio::net::TcpListener::bind(&bind_addr).await?;
    axum::serve(listener, app).await.context("HTTP server error")
}

pub async fn chat_completions(
    State(engine): State<EngineService>,
    Json(request): Json<ChatCompletionRequest>,
) -> Response {
    let max_tokens = request.max_tokens.unwrap_or(128);
    let seed = request.seed;
    let streaming = request.stream.unwrap_or(false);

    let model_id = engine
        .model_info()
        .map(|info| info.model_id)
        .unwrap_or_else(|| request.model.clone());

    if streaming {
        handle_streaming(engine, request.messages, max_tokens, seed, model_id).await
    } else {
        handle_non_streaming(engine, request.messages, max_tokens, seed, model_id).await
    }
}

async fn handle_streaming(
    engine: EngineService,
    messages: Vec<ChatMessage>,
    max_tokens: u32,
  seed: Option<u32>,
    model_id: String,
) -> Response {
    let completion_id = generate_completion_id();
    let created = unix_timestamp();

    // 1. 取得 mpsc 接收端與 blocking thread 的 JoinHandle
    let (rx, result_handle) = engine.chat_streaming(messages, max_tokens, seed);
    let token_stream = ReceiverStream::new(rx);

    // 2. 第一個事件：發送 Role
    let role_chunk = ChatCompletionChunk {
        id: completion_id.clone(),
        object: "chat.completion.chunk".to_string(),
        created,
        model: model_id.clone(),
        choices: vec![ChunkChoice {
            index: 0,
            delta: ChunkDelta { role: Some(Role::Assistant), content: None },
            finish_reason: None,
        }],
    };
    let role_event = Event::default().data(serde_json::to_string(&role_chunk).unwrap_or_default());
    let role_stream = stream::once(async move { Ok::<Event, std::convert::Infallible>(role_event) });

    // 3. 中間的 token 事件流
    let id_for_tokens = completion_id.clone();
    let model_for_tokens = model_id.clone();
    let token_events = token_stream.map(move |piece| {
        let chunk = ChatCompletionChunk {
            id: id_for_tokens.clone(),
            object: "chat.completion.chunk".to_string(),
            created,
            model: model_for_tokens.clone(),
            choices: vec![ChunkChoice {
                index: 0,
                delta: ChunkDelta { role: None, content: Some(piece) },
                finish_reason: None,
            }],
        };
        Ok::<Event, std::convert::Infallible>(
            Event::default().data(serde_json::to_string(&chunk).unwrap_or_default())
        )
    });

    // 4. 最後的結束事件：等 JoinHandle 結束取得 finish_reason，然後送出 [DONE]
    let id_for_final = completion_id;
    let model_for_final = model_id;
    let final_events = stream::once(async move {
        let finish_reason = match result_handle.await {
            Ok(Ok(output)) if output.generated_tokens >= max_tokens => "length",
            _ => "stop",
        };

        let done_chunk = ChatCompletionChunk {
            id: id_for_final,
            object: "chat.completion.chunk".to_string(),
            created,
            model: model_for_final,
            choices: vec![ChunkChoice {
                index: 0,
                delta: ChunkDelta { role: None, content: None },
                finish_reason: Some(finish_reason.to_string()),
            }],
        };

        let done_json = serde_json::to_string(&done_chunk).unwrap_or_default();
        let events = vec![
            Ok::<Event, std::convert::Infallible>(Event::default().data(done_json)),
            Ok::<Event, std::convert::Infallible>(Event::default().data("[DONE]")),
        ];
        stream::iter(events)
    }).flatten();

    // 5. 鏈接所有串流並包裝成 SSE 響應
    let full_stream = role_stream.chain(token_events).chain(final_events);
    let sse = Sse::new(full_stream).keep_alive(axum::response::sse::KeepAlive::default());

    // 6. 設定緩衝控制標頭，防止中介 Proxy 攔截阻礙實時輸出
    (
        [
            (axum::http::header::CACHE_CONTROL, "no-cache"),
            (axum::http::header::HeaderName::from_static("x-accel-buffering"), "no"),
        ],
        sse,
    ).into_response()
}

pub fn auto_detect(model_name: &str) -> Box<dyn ChatTemplate> {
    let lower = model_name.to_lowercase();

    if lower.contains("llama-3") || lower.contains("llama3") {
        Box::new(Llama3Template)
    } else if lower.contains("chatml") || lower.contains("im_start") {
        Box::new(ChatMLTemplate)
    } else {
        // 安全的安全預設值，因為大多數現代微調模型皆相容於 ChatML
        Box::new(ChatMLTemplate)
    }
}

/// 啟動與 OpenAI 相容的 HTTP API 伺服器
Serve {
    /// 本地 GGUF 路徑、模型 ID 或已下載檔案名稱的局部片段
    model: String,
    /// 搜尋與解析下載模型時的模型根資料夾
    #[arg(short, long, default_value = "models")]
    dir: PathBuf,
    /// 模型的上下文大小
    #[arg(long, default_value_t = 2048)]
    ctx_size: u32,
    /// 伺服器繫結的 Host 位址
    #[arg(long, default_value = "127.0.0.1")]
    host: String,
    /// 伺服器繫結的連接埠 (Port)
    #[arg(long, default_value_t = 8080)]
    port: u16,
}

async fn serve_model(model: String, dir: PathBuf, ctx_size: u32, host: String, port: u16) -> Result<()> {
    let model_record = resolve_model(&model, dir)?;
    println!("Loading model: {}", model_record.id);
    println!("  path: {}", model_record.path.display());
    println!();

    let engine = EngineService::new();
    engine.load_model(LoadModelRequest {
        model_id: model_record.id.clone(),
        path: model_record.path.clone(),
        context_size: Some(ctx_size),
    }).await?;

    let config = ServerConfig { host, port };
    start_server(config, engine).await
}

# 啟動 API 伺服器（系統會自動模糊搜尋 models/ 目錄下的 gemma-4-E4B-it 模型）
cargo run --release -- serve gemma-4-E4B-it --port 8080

curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma-4-E4B-it",
    "messages": [
      {"role": "user", "content": "Explain ownership in Rust programming"}
    ],
    "max_tokens": 80
  }'

{
  "id": "chatcmpl-a6bf02ed-3775-4dcd-a1b4-2492bec524d0",
  "object": "chat.completion",
  "created": 1779653484,
  "model": "gemma-4-E4B-it-Q4_K_M",
  "choices": [
    {
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "Ownership is one of the most fundamental and unique concepts in the Rust programming language. It is the core mechanism that allows Rust to guarantee **memory safety** (preventing issues like dangling pointers and data races) *without* needing a garbage collector (like in Java or Python).\n\nIn simple terms, **Ownership is a set of rules that governs how Rust manages memory and how long data lives in memory"
      },
      "finish_reason": "length"
    }
  ],
  "usage": {
    "prompt_tokens": 29,
    "completion_tokens": 80,
    "total_tokens": 109
  }
}

curl http://localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma-4-E4B-it",
    "messages": [
      {"role": "user", "content": "Explain ownership in Rust programming"}
    ],
    "stream": true,
    "max_tokens": 30
  }'

data: {"id":"chatcmpl-20b6d4b6-1ef3-45ed-9400-7bb1ce361ff0","object":"chat.completion.chunk","created":1779653484,"model":"gemma-4-E4B-it-Q4_K_M","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}

data: {"id":"chatcmpl-20b6d4b6-1ef3-45ed-9400-7bb1ce361ff0","object":"chat.completion.chunk","created":1779653484,"model":"gemma-4-E4B-it-Q4_K_M","choices":[{"index":0,"delta":{"content":"Ownership"},"finish_reason":null}]}

data: {"id":"chatcmpl-20b6d4b6-1ef3-45ed-9400-7bb1ce361ff0","object":"chat.completion.chunk","created":1779653484,"model":"gemma-4-E4B-it-Q4_K_M","choices":[{"index":0,"delta":{"content":" is"},"finish_reason":null}]}

data: {"id":"chatcmpl-20b6d4b6-1ef3-45ed-9400-7bb1ce361ff0","object":"chat.completion.chunk","created":1779653484,"model":"gemma-4-E4B-it-Q4_K_M","choices":[{"index":0,"delta":{"content":" one"},"finish_reason":null}]}

...

data: {"id":"chatcmpl-20b6d4b6-1ef3-45ed-9400-7bb1ce361ff0","object":"chat.completion.chunk","created":1779653484,"model":"gemma-4-E4B-it-Q4_K_M","choices":[{"index":0,"delta":{},"finish_reason":"length"}]}

data: [DONE]

#[derive(Debug, Subcommand)]
enum Command {
    /// 掃描本地資料夾裡的 GGUF 檔案
    Scan { dir: PathBuf },
    /// 搜尋 Hugging Face 上的 GGUF 模型
    HfSearch { query: String, limit: usize },
    /// 從 Hugging Face 下載 GGUF 模型
    HfDownload { name: String, filename: Option<String>, ... },
    /// 用 llama.cpp 跑本地模型推理
    Run { model: String, prompt: String, ... },
}

impl Command {
    fn run(self) -> Result<()> {
        match self {
            Self::Scan { dir } => scan_models(dir),
            Self::HfSearch { query, limit } => search_hugging_face(query, limit),
            Self::HfDownload { .. } => download_hugging_face_model(..),
            Self::Run { .. } => run_model(..),
        }
    }
}

fn resolve_model_path(model: &str, dir: PathBuf) -> Result<PathBuf> {
    let direct_path = PathBuf::from(model);
    if direct_path.try_exists()? {
        return Ok(direct_path);
    }
    let registry = ModelRegistry::scan_dir(&dir)?;
    Ok(registry.find(model)?.path.clone())
}

# 直接給路徑
llm-local-studio run models/Q4_K_M.gguf -p "Hello"

# 或者只給檔名的一部分
llm-local-studio run tinyllama -p "Hello"

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ModelRecord {
    pub id: String,
    pub path: PathBuf,
    pub size_bytes: u64,
    pub source: ModelSource,
    pub status: ModelLoadStatus,
}

fn scan_dir_recursive(dir: &Path, models: &mut Vec<ModelRecord>) -> Result<()> {
    if !dir.try_exists()? {
        return Ok(());
    }
    for entry in fs::read_dir(dir)? {
        let entry = entry?;
        let path = entry.path();
        let metadata = entry.metadata()?;
        if metadata.is_dir() {
            scan_dir_recursive(&path, models)?;
        } else if is_gguf_file(&path) {
            models.push(ModelRecord { ... });
        }
    }
    Ok(())
}

pub fn find(&self, query: &str) -> Result<&ModelRecord> {
    // 第一層：精確比對（不分大小寫）
    let mut matches = self.models.iter().filter(|model| {
        model.id.eq_ignore_ascii_case(query)
            || model.path.file_name()...eq_ignore_ascii_case(query)
    });
    if let Some(model) = matches.next() {
        return Ok(model);
    }

    // 第二層：部分比對（contains）
    let partial_matches = self.models.iter().filter(|model| {
        model.id.to_lowercase().contains(&query_lower)
            || model.path...to_lowercase().contains(&query_lower)
    }).collect::<Vec<_>>();

    match partial_matches.as_slice() {
        [model] => Ok(model),       // 唯一命中
        [] => bail!("no local model matched {query:?}"),
        matches => bail!("model name {query:?} is ambiguous; matches: {ids}"),
    }
}

# 先看下載 URL，不下載
llm-local-studio hf-download TheBloke/TinyLlama-1.1B-GGUF --print-url

# 確認無誤後再下載
llm-local-studio hf-download TheBloke/TinyLlama-1.1B-GGUF

const PREFERRED_QUANTS: &[&str] = &["Q4_K_M", "Q4_K_S", "Q5_K_M", "Q8_0", "Q4_0"];

pub fn choose_gguf_file<'a>(
    &self,
    files: &'a [HuggingFaceFile],
    requested: Option<&str>,
) -> Result<&'a HuggingFaceFile> {
    if let Some(requested) = requested {
        return files.iter()
            .find(|file| file.rfilename == requested)
            .with_context(|| ...);
    }
    for preferred in PREFERRED_QUANTS {
        if let Some(file) = files.iter().find(|file| file.rfilename.contains(preferred)) {
            return Ok(file);
        }
    }
    files.first().context("repository does not contain GGUF files")
}

let temp_path = plan.destination.with_extension("gguf.part");
let mut output = fs::File::create(&temp_path)?;
let bytes_written = io::copy(&mut response, &mut output)?;
drop(output); // 確保 flush

fs::rename(&temp_path, &plan.destination)?;

fn local_model_path(models_dir: &Path, model: &HuggingFaceModelRef, filename: &str) -> Result<PathBuf> {
    if filename.is_empty() || filename.contains("..") {
        anyhow::bail!("invalid Hugging Face filename {filename:?}");
    }
    for component in filename.split('/') {
        if component.is_empty() || component == "." || component == ".." {
            anyhow::bail!("invalid Hugging Face filename {filename:?}");
        }
        path.push(component);
    }
    Ok(path)
}

models/
  huggingface/
    TheBloke/
      TinyLlama-1.1B-GGUF/
        tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf

pub trait InferenceEngine {
    fn load_model(&mut self, request: LoadModelRequest) -> Result<ModelHandle>;
    fn unload_model(&mut self, model_id: &str) -> Result<()>;
    fn health(&self) -> EngineHealth;
}

pub enum EngineHealth {
    NotConfigured,
    Ready,
    Loaded { model_id: String },
    Error { message: String },
}

impl LlamaCppRunner {
    pub fn run(&self, request: &RunModelRequest) -> Result<RunModelOutput> {
        suppress_llama_logs();

        // 1. 初始化 llama.cpp 後端
        let backend = LlamaBackend::init()
            .context("failed to initialize llama.cpp backend")?;

        // 2. 載入 GGUF 模型
        let model = LlamaModel::load_from_file(
            &backend, &request.model_path, &LlamaModelParams::default()
        )?;

        // 3. 建立 context（設定 context window 大小）
        let context_size = NonZeroU32::new(request.context_size)?;
        let context_params = LlamaContextParams::default()
            .with_n_ctx(Some(context_size));
        let mut context = model.new_context(&backend, context_params)?;

        // 4. Tokenize prompt
        let prompt_tokens = model.str_to_token(&request.prompt, AddBos::Always)?;

        // 5. 檢查 context 容量
        let requested_tokens = prompt_tokens.len() + request.max_tokens as usize;
        if requested_tokens > context.n_ctx() as usize {
            anyhow::bail!("prompt plus generated tokens require {requested_tokens} ...");
        }

        // 6. 處理 prompt（一次性送入 KV cache）
        let mut batch = LlamaBatch::new(512, 1);
        for (position, token) in (0_i32..).zip(prompt_tokens.into_iter()) {
            batch.add(token, position, &[0], position == last_prompt_index)?;
        }
        context.decode(&mut batch)?;

        // 7. 逐 token 生成（streaming）
        let mut sampler = LlamaSampler::chain_simple([
            LlamaSampler::dist(1234),
            LlamaSampler::greedy(),
        ]);
        while generated_tokens < request.max_tokens {
            let token = sampler.sample(&context, batch.n_tokens() - 1);
            sampler.accept(token);
            if model.is_eog_token(token) { break; }

            let piece = model.token_to_piece(token, &mut decoder, true, None)?;
            print!("{piece}");        // 逐字印出
            std::io::stdout().flush()?;

            batch.clear();
            batch.add(token, position, &[0], true)?;
            context.decode(&mut batch)?;
            position += 1;
            generated_tokens += 1;
        }

        Ok(RunModelOutput { generated_tokens })
    }
}

if requested_tokens > context.n_ctx() as usize {
    anyhow::bail!("prompt plus generated tokens require {requested_tokens} ...");
}

// 概念上等價於 llama.cpp 內部的 e_model enum
enum ModelArch {
    Llama,
    Mistral,
    Gpt2,
    Phi2,
    Qwen2,
    Gemma,
    Mamba,
    // ... 60+ 種架構
}

# 1. 搜尋 Hugging Face 上的 GGUF 模型
llm-local-studio hf-search "tinyllama gguf"

# 2. 下載模型（自動選 Q4_K_M 量化）
llm-local-studio hf-download TheBloke/TinyLlama-1.1B-GGUF

# 3. 掃描本地已有的模型
llm-local-studio scan models

# 4. 跑推理！
llm-local-studio run tinyllama -p "Tell me a joke about Rust"

llm-local-studio run TheBloke/TinyLlama-1.1B-GGUF \
  -p "Explain ownership in Rust" \
  --ctx-size 2048 \
  -n 256

#[test]
fn parses_hugging_face_model_ref() {
    let model: HuggingFaceModelRef = "TheBloke/TinyLlama-1.1B-GGUF".parse().unwrap();
    assert_eq!(model.owner, "TheBloke");
    assert_eq!(model.repo, "TinyLlama-1.1B-GGUF");
}

#[test]
fn chooses_preferred_quant_when_filename_is_not_requested() {
    let files = vec![
        HuggingFaceFile { rfilename: "model.Q8_0.gguf".into(), size: None },
        HuggingFaceFile { rfilename: "model.Q4_K_M.gguf".into(), size: None },
    ];
    assert_eq!(
        client.choose_gguf_file(&files, None).unwrap().rfilename,
        "model.Q4_K_M.gguf"
    );
}

#[test]
fn builds_local_hugging_face_model_path() {
    let model: HuggingFaceModelRef = "owner/repo".parse().unwrap();
    let path = local_model_path(Path::new("models"), &model, "nested/model.gguf").unwrap();
    // models/huggingface/owner/repo/nested/model.gguf
}