fix: add timeout to LLM API calls to prevent hung requests.

This commit is contained in:
K
2026-05-03 17:26:20 +05:30
parent d2a75be7b6
commit 4c69ee4fc1
+12 -1
View File
@@ -21,11 +21,18 @@ const MODEL = "llama-3.1-8b-instant";
* @returns {Promise<string>} Trimmed text content from the first choice. * @returns {Promise<string>} Trimmed text content from the first choice.
* @throws {Error} If the API key is missing or the response is non-2xx. * @throws {Error} If the API key is missing or the response is non-2xx.
*/ */
const GROQ_TIMEOUT_MS = 8_000;
async function _groqCall({ systemPrompt, userMessage, maxTokens = 256, temperature = 0.2 }) { async function _groqCall({ systemPrompt, userMessage, maxTokens = 256, temperature = 0.2 }) {
const key = process.env.GROQ_API_KEY; const key = process.env.GROQ_API_KEY;
if (!key) throw new Error("GROQ_API_KEY not set"); if (!key) throw new Error("GROQ_API_KEY not set");
const res = await fetch(GROQ_API_URL, { const ctrl = new AbortController();
const tid = setTimeout(() => ctrl.abort(), GROQ_TIMEOUT_MS);
let res;
try {
res = await fetch(GROQ_API_URL, {
method: "POST", method: "POST",
headers: { headers: {
"Content-Type": "application/json", "Content-Type": "application/json",
@@ -40,7 +47,11 @@ async function _groqCall({ systemPrompt, userMessage, maxTokens = 256, temperatu
max_tokens: maxTokens, max_tokens: maxTokens,
temperature, temperature,
}), }),
signal: ctrl.signal,
}); });
} finally {
clearTimeout(tid);
}
if (!res.ok) { if (!res.ok) {
const body = await res.json().catch(() => ({})); const body = await res.json().catch(() => ({}));