Simply Patrick Hopes can always go up, tears can only come down.

[dependencies]
llama-cpp-4 = { version = "0.3.0", features = ["mtmd", "vulkan"] }

let model = LlamaModel::load_from_file(
    &backend, 
    &request.path, 
    &LlamaModelParams::default().with_n_gpu_layers(99) // 將 99 層（即所有層）模型載入 GPU 記憶體
)
.with_context(|| format!("failed to load model {}", request.path.display()))?;

// 1. 從音訊投影器檔案 (mmproj.gguf) 初始化多模態上下文
let params = MtmdContextParams::default();
let mtmd_ctx = MtmdContext::init_from_file(&mmproj_path, &session.model, params)?;

// 2. 將輸入的音訊檔案載入為 MtmdBitmap (此處 libmtmd 會處理對數梅爾頻譜圖轉換與特徵提取)
let bitmap = MtmdBitmap::from_file(mtmd_ctx, &request.audio_path)?;

// 3. 在文字 Prompt 中加入特殊的媒體預留標記 (Default Marker)
let marker = MtmdContext::default_marker();
let full_prompt = format!("{} {}", request.prompt, marker);

// 4. Tokenize：將文字與音訊資料進行融合，將預留標記替換為音訊向量
let text = MtmdInputText::new(&full_prompt, true, true);
let bitmaps = [&bitmap];
let mut chunks = MtmdInputChunks::new();
mtmd_ctx.tokenize(&text, &bitmaps, &mut chunks)?;

// 5. 評估（Evaluation）音訊與文字的混合向量
let mut lctx = session.model.new_context(&session.backend, self.loaded_context_params.clone())?;
let mut n_past = 0i32;
mtmd_ctx.eval_chunks(lctx.as_ptr(), &chunks, 0, 0, lctx.n_batch() as i32, true, &mut n_past)?;

// 6. 進入標準的 Token 採樣循環，即時輸出文字答案
let batch = LlamaBatch::new(512, 1);
let (output_text, generated_tokens) = sample_loop(
    session,
    &mut lctx,
    batch,
    n_past, // 評估音訊後的起始位置
    request.max_tokens,
    request.seed,
    &request.stream_callback,
)?;

fn audio_bytes_to_wav(audio_bytes: &[u8], _format_hint: &str) -> Result<std::path::PathBuf, String> {
    let tmp_dir = std::env::temp_dir();
    let wav_path = tmp_dir.join(format!("llm_audio_{}.wav", uuid::Uuid::new_v4()));

    // 啟動 ffmpeg 子行程
    let mut child = Command::new("ffmpeg")
        .args([
            "-y",            // 若檔案存在則覆寫
            "-i", "pipe:0",  // 從標準輸入 (stdin) 讀取 WebM 數據
            "-ar", "16000",  // 設定採樣率為 16000 Hz
            "-ac", "1",      // 單聲道 (Mono)
            "-f", "wav",     // 強制輸出格式為 wav
            wav_path.to_str().ok_or("temp path not UTF-8")?,
        ])
        .stdin(Stdio::piped())
        .stdout(Stdio::null())
        .stderr(Stdio::null())
        .spawn()
        .map_err(|e| format!("Failed to spawn ffmpeg: {e}"))?;

    // 將 WebM 二進位數據寫入 ffmpeg 的標準輸入
    {
        let mut stdin = child.stdin.take().ok_or("Failed to open ffmpeg stdin")?;
        stdin.write_all(audio_bytes).map_err(|e| format!("Failed to write audio to ffmpeg: {e}"))?;
    }

    // 等待轉換完成
    let status = child.wait().map_err(|e| format!("ffmpeg wait error: {e}"))?;
    if !status.success() {
        return Err("ffmpeg failed to convert audio to WAV".to_string());
    }

    Ok(wav_path)
}

// 1. 啟動錄音
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm' });
audioChunks = [];

mediaRecorder.ondataavailable = (event) => {
  if (event.data.size > 0) {
    audioChunks.push(event.data);
  }
};

mediaRecorder.onstop = async () => {
  // 將錄音數據打包為 Blob
  const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
  stream.getTracks().forEach(track => track.stop());
  await handleAudioMultimodal(audioBlob);
};

mediaRecorder.start();

// 將 Blob 轉成 Base64 字串
const base64Data = await new Promise<string>((resolve, reject) => {
  const reader = new FileReader();
  reader.onloadend = () => {
    const result = reader.result as string;
    const base64 = result.split(',')[1];
    resolve(base64);
  };
  reader.onerror = reject;
  reader.readAsDataURL(audioBlob);
});

// 建立多模態內容片段
const contentParts = [
  { type: 'text', text: chatInput.value.trim() || 'Listen and respond to this audio input.' },
  {
    type: 'input_audio',
    input_audio: {
      data: base64Data,
      format: 'webm' // 瀏覽器錄製的 webm 格式
    }
  }
];

// 推入對話歷史紀錄
messages.push({
  role: 'user',
  content: contentParts // 陣列格式的多模態訊息
});

// 呼叫 Axum 後端 (支援與純文字相同的 SSE 串流輸出)
await generateResponse();

# 啟動服務並加載 Vulkan GPU 加速與音訊投影器
cargo run --release -- serve gemma-4-E4B-it-Q4_K_M.gguf --mmproj models/huggingface/lmstudio-community/gemma-4-E4B-it-GGUF/mmproj-gemma-4-E4B-it-BF16.gguf --port 8080