65 lines
2.5 KiB
Python
65 lines
2.5 KiB
Python
"""A per-request LLM-call budget — F-08.
|
|
|
|
`RagAgent.handle()` makes up to ~5 sequential Bedrock calls per turn
|
|
(understand, sufficiency, generate, entailment) with no aggregate deadline
|
|
before this. Each call now has at most two bounded provider attempts
|
|
(`read_timeout=20` in `adapters/bedrock_converse.py`) because this budget is
|
|
checked between calls and cannot cancel boto3 while it is already in flight.
|
|
Measured live 2026-08-07: a normal answerable
|
|
turn costs ~8-9s total; nothing bounds the pathological case.
|
|
|
|
Checked before each call, not wrapped around an already-running one — this
|
|
bounds how many MORE calls get a chance to start once time/calls run out. It
|
|
does not cancel a call already in flight past its own provider timeout; a
|
|
hard per-call cancellation would need cooperative cancellation support from
|
|
`adapters/bedrock_converse.py`'s boto3 client. The adapter-level timeout bounds
|
|
that residual gap; this object prevents any later call from starting after
|
|
the aggregate deadline.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass
|
|
|
|
from .ports import AnswerGenerationUnavailable
|
|
|
|
|
|
class RequestBudgetExhausted(AnswerGenerationUnavailable):
|
|
"""The per-request budget ran out before a call could be attempted.
|
|
|
|
Subclasses `AnswerGenerationUnavailable` deliberately: every existing
|
|
`except AnswerGenerationUnavailable:` fail-closed/fail-open handler
|
|
already does the right thing for this with no changes — a caller that
|
|
wants to log a distinct reason (budget vs. genuine outage) catches this
|
|
subclass specifically before the general one.
|
|
"""
|
|
|
|
|
|
@dataclass
|
|
class RequestBudget:
|
|
deadline: float
|
|
calls_remaining: int
|
|
|
|
@classmethod
|
|
def start(cls, max_wall_clock_ms: int, max_calls: int) -> "RequestBudget":
|
|
return cls(
|
|
deadline=time.monotonic() + max_wall_clock_ms / 1000,
|
|
calls_remaining=max_calls,
|
|
)
|
|
|
|
def has_budget(self) -> bool:
|
|
return self.calls_remaining > 0 and time.monotonic() < self.deadline
|
|
|
|
def spend(self) -> None:
|
|
self.calls_remaining -= 1
|
|
|
|
def require(self) -> None:
|
|
"""Raise if there's no budget for one more call, else spend it.
|
|
The single call site every LLM-call wrapper below should make
|
|
immediately before its actual provider call."""
|
|
if not self.has_budget():
|
|
raise RequestBudgetExhausted(
|
|
"request budget exhausted (calls or wall-clock deadline)"
|
|
)
|
|
self.spend()
|