fix: add timeout to LLM API calls to prevent hung requests.

This commit is contained in:
K
2026-05-03 17:26:20 +05:30
parent d2a75be7b6
commit 4c69ee4fc1
+27 -16
View File
@@ -21,26 +21,37 @@ const MODEL = "llama-3.1-8b-instant";
* @returns {Promise<string>} Trimmed text content from the first choice. * @returns {Promise<string>} Trimmed text content from the first choice.
* @throws {Error} If the API key is missing or the response is non-2xx. * @throws {Error} If the API key is missing or the response is non-2xx.
*/ */
const GROQ_TIMEOUT_MS = 8_000;
async function _groqCall({ systemPrompt, userMessage, maxTokens = 256, temperature = 0.2 }) { async function _groqCall({ systemPrompt, userMessage, maxTokens = 256, temperature = 0.2 }) {
const key = process.env.GROQ_API_KEY; const key = process.env.GROQ_API_KEY;
if (!key) throw new Error("GROQ_API_KEY not set"); if (!key) throw new Error("GROQ_API_KEY not set");
const res = await fetch(GROQ_API_URL, { const ctrl = new AbortController();
method: "POST", const tid = setTimeout(() => ctrl.abort(), GROQ_TIMEOUT_MS);
headers: {
"Content-Type": "application/json", let res;
Authorization: `Bearer ${key}`, try {
}, res = await fetch(GROQ_API_URL, {
body: JSON.stringify({ method: "POST",
model: MODEL, headers: {
messages: [ "Content-Type": "application/json",
{ role: "system", content: systemPrompt }, Authorization: `Bearer ${key}`,
{ role: "user", content: userMessage }, },
], body: JSON.stringify({
max_tokens: maxTokens, model: MODEL,
temperature, messages: [
}), { role: "system", content: systemPrompt },
}); { role: "user", content: userMessage },
],
max_tokens: maxTokens,
temperature,
}),
signal: ctrl.signal,
});
} finally {
clearTimeout(tid);
}
if (!res.ok) { if (!res.ok) {
const body = await res.json().catch(() => ({})); const body = await res.json().catch(() => ({}));