Concurrency

import asyncio, random
from openai import AsyncOpenAI, RateLimitError

client = AsyncOpenAI(base_url="https://api.flex.ai/v1")
# Bound concurrency to your tier: free=10/60≈1, elevated=60/60=1, paid=300/60=5.
sem = asyncio.Semaphore(5)

async def one(prompt: str, *, attempt: int = 0) -> str:
    async with sem:
        try:
            r = await client.chat.completions.create(
                model="Llama-3.3-70B-Instruct-FP8",
                messages=[{"role": "user", "content": prompt}],
            )
            return r.choices[0].message.content
        except RateLimitError as e:
            retry_after = int(e.response.headers.get("retry-after", 1))
            jitter = random.uniform(0, 1)
            await asyncio.sleep(retry_after + jitter)
            if attempt >= 5:
                raise
            return await one(prompt, attempt=attempt + 1)

import OpenAI from "openai";
const client = new OpenAI({ baseURL: "https://api.flex.ai/v1" });

async function withBackoff<T>(fn: () => Promise<T>, max = 5): Promise<T> {
  for (let attempt = 0; ; attempt++) {
    try {
      return await fn();
    } catch (err: any) {
      if (err?.status !== 429 || attempt >= max) throw err;
      const retryAfter = Number(err.headers?.["retry-after"] ?? 1);
      await new Promise((r) => setTimeout(r, (retryAfter + Math.random()) * 1000));
    }
  }
}