Fix ai-service Dockerfile: bake in drug_entities.json, override its path
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
"""A per-request LLM-call budget — F-08.
|
||||
|
||||
`RagAgent.handle()` makes up to ~5 sequential Bedrock calls per turn
|
||||
(understand, sufficiency, generate, up to 2 entailment retries) with no
|
||||
aggregate deadline before this: each call is bounded only by its own fixed
|
||||
provider timeout (`read_timeout=60` in `adapters/bedrock_converse.py`, times
|
||||
up to 3 retries at "standard" backoff — worst case several minutes for one
|
||||
stuck call, let alone five). Measured live 2026-08-07: a normal answerable
|
||||
turn costs ~8-9s total; nothing bounds the pathological case.
|
||||
|
||||
Checked before each call, not wrapped around an already-running one — this
|
||||
bounds how many MORE calls get a chance to start once time/calls run out. It
|
||||
does not cancel a call already in flight past its own provider timeout; a
|
||||
hard per-call cancellation would need cooperative cancellation support from
|
||||
`adapters/bedrock_converse.py`'s boto3 client, a larger change than this
|
||||
budget object alone. Still a real improvement: five calls each capable of
|
||||
running to their own 60s+ limit, one after another, is the actual gap this
|
||||
closes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .ports import AnswerGenerationUnavailable
|
||||
|
||||
|
||||
class RequestBudgetExhausted(AnswerGenerationUnavailable):
|
||||
"""The per-request budget ran out before a call could be attempted.
|
||||
|
||||
Subclasses `AnswerGenerationUnavailable` deliberately: every existing
|
||||
`except AnswerGenerationUnavailable:` fail-closed/fail-open handler
|
||||
already does the right thing for this with no changes — a caller that
|
||||
wants to log a distinct reason (budget vs. genuine outage) catches this
|
||||
subclass specifically before the general one.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestBudget:
|
||||
deadline: float
|
||||
calls_remaining: int
|
||||
|
||||
@classmethod
|
||||
def start(cls, max_wall_clock_ms: int, max_calls: int) -> "RequestBudget":
|
||||
return cls(
|
||||
deadline=time.monotonic() + max_wall_clock_ms / 1000,
|
||||
calls_remaining=max_calls,
|
||||
)
|
||||
|
||||
def has_budget(self) -> bool:
|
||||
return self.calls_remaining > 0 and time.monotonic() < self.deadline
|
||||
|
||||
def spend(self) -> None:
|
||||
self.calls_remaining -= 1
|
||||
|
||||
def require(self) -> None:
|
||||
"""Raise if there's no budget for one more call, else spend it.
|
||||
The single call site every LLM-call wrapper below should make
|
||||
immediately before its actual provider call."""
|
||||
if not self.has_budget():
|
||||
raise RequestBudgetExhausted(
|
||||
"request budget exhausted (calls or wall-clock deadline)"
|
||||
)
|
||||
self.spend()
|
||||
Reference in New Issue
Block a user