66 lines
2.6 KiB
Python
66 lines
2.6 KiB
Python
"""A per-request LLM-call budget — F-08.
|
|
|
|
`RagAgent.handle()` makes up to ~5 sequential Bedrock calls per turn
|
|
(understand, sufficiency, generate, up to 2 entailment retries) with no
|
|
aggregate deadline before this: each call is bounded only by its own fixed
|
|
provider timeout (`read_timeout=60` in `adapters/bedrock_converse.py`, times
|
|
up to 3 retries at "standard" backoff — worst case several minutes for one
|
|
stuck call, let alone five). Measured live 2026-08-07: a normal answerable
|
|
turn costs ~8-9s total; nothing bounds the pathological case.
|
|
|
|
Checked before each call, not wrapped around an already-running one — this
|
|
bounds how many MORE calls get a chance to start once time/calls run out. It
|
|
does not cancel a call already in flight past its own provider timeout; a
|
|
hard per-call cancellation would need cooperative cancellation support from
|
|
`adapters/bedrock_converse.py`'s boto3 client, a larger change than this
|
|
budget object alone. Still a real improvement: five calls each capable of
|
|
running to their own 60s+ limit, one after another, is the actual gap this
|
|
closes.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass
|
|
|
|
from .ports import AnswerGenerationUnavailable
|
|
|
|
|
|
class RequestBudgetExhausted(AnswerGenerationUnavailable):
|
|
"""The per-request budget ran out before a call could be attempted.
|
|
|
|
Subclasses `AnswerGenerationUnavailable` deliberately: every existing
|
|
`except AnswerGenerationUnavailable:` fail-closed/fail-open handler
|
|
already does the right thing for this with no changes — a caller that
|
|
wants to log a distinct reason (budget vs. genuine outage) catches this
|
|
subclass specifically before the general one.
|
|
"""
|
|
|
|
|
|
@dataclass
|
|
class RequestBudget:
|
|
deadline: float
|
|
calls_remaining: int
|
|
|
|
@classmethod
|
|
def start(cls, max_wall_clock_ms: int, max_calls: int) -> "RequestBudget":
|
|
return cls(
|
|
deadline=time.monotonic() + max_wall_clock_ms / 1000,
|
|
calls_remaining=max_calls,
|
|
)
|
|
|
|
def has_budget(self) -> bool:
|
|
return self.calls_remaining > 0 and time.monotonic() < self.deadline
|
|
|
|
def spend(self) -> None:
|
|
self.calls_remaining -= 1
|
|
|
|
def require(self) -> None:
|
|
"""Raise if there's no budget for one more call, else spend it.
|
|
The single call site every LLM-call wrapper below should make
|
|
immediately before its actual provider call."""
|
|
if not self.has_budget():
|
|
raise RequestBudgetExhausted(
|
|
"request budget exhausted (calls or wall-clock deadline)"
|
|
)
|
|
self.spend()
|