fix: add timeout to LLM API calls to prevent hung requests.
This commit is contained in:
@@ -21,26 +21,37 @@ const MODEL = "llama-3.1-8b-instant";
|
|||||||
* @returns {Promise<string>} Trimmed text content from the first choice.
|
* @returns {Promise<string>} Trimmed text content from the first choice.
|
||||||
* @throws {Error} If the API key is missing or the response is non-2xx.
|
* @throws {Error} If the API key is missing or the response is non-2xx.
|
||||||
*/
|
*/
|
||||||
|
const GROQ_TIMEOUT_MS = 8_000;
|
||||||
|
|
||||||
async function _groqCall({ systemPrompt, userMessage, maxTokens = 256, temperature = 0.2 }) {
|
async function _groqCall({ systemPrompt, userMessage, maxTokens = 256, temperature = 0.2 }) {
|
||||||
const key = process.env.GROQ_API_KEY;
|
const key = process.env.GROQ_API_KEY;
|
||||||
if (!key) throw new Error("GROQ_API_KEY not set");
|
if (!key) throw new Error("GROQ_API_KEY not set");
|
||||||
|
|
||||||
const res = await fetch(GROQ_API_URL, {
|
const ctrl = new AbortController();
|
||||||
method: "POST",
|
const tid = setTimeout(() => ctrl.abort(), GROQ_TIMEOUT_MS);
|
||||||
headers: {
|
|
||||||
"Content-Type": "application/json",
|
let res;
|
||||||
Authorization: `Bearer ${key}`,
|
try {
|
||||||
},
|
res = await fetch(GROQ_API_URL, {
|
||||||
body: JSON.stringify({
|
method: "POST",
|
||||||
model: MODEL,
|
headers: {
|
||||||
messages: [
|
"Content-Type": "application/json",
|
||||||
{ role: "system", content: systemPrompt },
|
Authorization: `Bearer ${key}`,
|
||||||
{ role: "user", content: userMessage },
|
},
|
||||||
],
|
body: JSON.stringify({
|
||||||
max_tokens: maxTokens,
|
model: MODEL,
|
||||||
temperature,
|
messages: [
|
||||||
}),
|
{ role: "system", content: systemPrompt },
|
||||||
});
|
{ role: "user", content: userMessage },
|
||||||
|
],
|
||||||
|
max_tokens: maxTokens,
|
||||||
|
temperature,
|
||||||
|
}),
|
||||||
|
signal: ctrl.signal,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
clearTimeout(tid);
|
||||||
|
}
|
||||||
|
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const body = await res.json().catch(() => ({}));
|
const body = await res.json().catch(() => ({}));
|
||||||
|
|||||||
Reference in New Issue
Block a user