fix: add timeout to LLM API calls to prevent hung requests.

This commit is contained in:
K
2026-05-03 17:26:20 +05:30
parent d2a75be7b6
commit 4c69ee4fc1
+27 -16
View File
@@ -21,26 +21,37 @@ const MODEL = "llama-3.1-8b-instant";
* @returns {Promise<string>} Trimmed text content from the first choice.
* @throws {Error} If the API key is missing or the response is non-2xx.
*/
const GROQ_TIMEOUT_MS = 8_000;
async function _groqCall({ systemPrompt, userMessage, maxTokens = 256, temperature = 0.2 }) {
const key = process.env.GROQ_API_KEY;
if (!key) throw new Error("GROQ_API_KEY not set");
const res = await fetch(GROQ_API_URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${key}`,
},
body: JSON.stringify({
model: MODEL,
messages: [
{ role: "system", content: systemPrompt },
{ role: "user", content: userMessage },
],
max_tokens: maxTokens,
temperature,
}),
});
const ctrl = new AbortController();
const tid = setTimeout(() => ctrl.abort(), GROQ_TIMEOUT_MS);
let res;
try {
res = await fetch(GROQ_API_URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${key}`,
},
body: JSON.stringify({
model: MODEL,
messages: [
{ role: "system", content: systemPrompt },
{ role: "user", content: userMessage },
],
max_tokens: maxTokens,
temperature,
}),
signal: ctrl.signal,
});
} finally {
clearTimeout(tid);
}
if (!res.ok) {
const body = await res.json().catch(() => ({}));