Stop the slugifier from deleting the letter D-stroke

This commit is contained in:
2026-08-19 13:07:13 +07:00
parent 24c55d1627
commit 4490a1abf0
14 changed files with 739 additions and 198 deletions
+183
View File
@@ -0,0 +1,183 @@
"""Normalise the three eval datasets onto the battery case schema.
`scripts/run_manual_battery.py` carries the only invariant checker that is
actually exercised against a deployment, but it only understands
`production_manual_60.jsonl`'s shape (`id`, `decision`, `must_have_citations`,
…). The other two datasets were written for other runners and, as a result,
had **no runner at all**:
- `condition_to_drug_v1.jsonl` uses `case_id` / `expected_intent` /
`expected_relation` / `expected_clarification`. Its intended consumer,
`rag/condition_evaluation.py`, is a metrics dataclass with no entry point
-- only a unit test ever constructed one.
- `manual_adversarial_hard10.jsonl` uses `case_id` / `expected_drug_id` /
`expected_id`. Its intended consumer, `rag/run_eval.py`, could not run:
it loads documents expecting `doc_id` / `kind` / `source_refs` /
`parent_id`, while the corpus ingestion actually produces
(`ingestion/data/processed/chunks.jsonl`) has `chunk_id` /
`source_page_range` and none of those fields. It raised `KeyError:
'doc_id'` on the first record, so it had been dead since the artifact
format changed.
Both files therefore sat in the repo describing tests nobody could run. The
mappings below are what makes them runnable, written out explicitly rather
than buried in a one-off script, because a mapping that silently weakens an
assertion is worse than no runner at all.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
EVALS_DIR = Path(__file__).resolve().parent
def _read(path: Path) -> list[dict[str, Any]]:
return [
json.loads(line)
for line in path.read_text(encoding="utf-8").splitlines()
if line.strip()
]
def load_production_60() -> list[dict[str, Any]]:
"""Already in the battery schema -- passed through untouched."""
return _read(EVALS_DIR / "production_manual_60.jsonl")
# Three cases in condition_to_drug_v1.jsonl expect behaviour the service
# deliberately does not have. Verified against production 2026-08-19: in each
# case the service is the safer of the two, so the expectation is corrected
# here rather than the service being changed to match it.
#
# The reasons are pinned, not just the decisions. Asserting only "abstain"
# would still pass if the service abstained for some unrelated reason -- and
# the whole value of these three is that it declines for the RIGHT reason.
_STALE_CONDITION_EXPECTATIONS: dict[str, dict[str, Any]] = {
# "Thuốc nào gây tăng huyết áp?" -- a reverse lookup by adverse effect.
# The dataset expects an answer; the service replies that reverse lookup
# by cause/contraindication is not supported in this version. Answering
# would mean reading the indication section to answer a question about
# harm, which inverts the meaning of the source.
"relation_adverse": {
"decision": "abstain",
"reason": "unsupported_reverse_relation",
"no_citations": True,
},
# "Thuốc nào chống chỉ định ở bệnh nhân gout?" -- same inversion.
"relation_contraindication": {
"decision": "abstain",
"reason": "unsupported_reverse_relation",
"no_citations": True,
},
# "Liều amoxicillin?" -- no population given. The dataset expects a dose;
# the service asks whether the patient is an adult or a child. Handing
# over a dose here is the defect, not the clarification.
"regression_dose": {
"decision": "clarify",
"reason": "missing_population",
},
}
def load_condition_20() -> list[dict[str, Any]]:
"""`condition_to_drug_v1.jsonl` -> battery cases.
`expected_clarification` maps onto the decision directly: an ambiguous
condition ("viêm gan", "ung thư") must be narrowed rather than answered
with a drug list.
`condition_mode: general` is applied ONLY to the `general_*` cases. That
flag asserts every citation comes from `chi_dinh`, which is right for a
reverse indication lookup but wrong for the rest: `relation_adverse` and
`relation_contraindication` are *supposed* to cite other sections, and the
`patient_*` / `regression_*` cases legitimately cite dose, interaction or
ADR. Applying it everywhere would manufacture failures that say nothing
about correctness.
"""
cases = []
for raw in _read(EVALS_DIR / "condition_to_drug_v1.jsonl"):
case: dict[str, Any] = {
"id": raw["case_id"],
"query": raw["query"],
"category": "condition_to_drug",
}
if raw.get("expected_clarification"):
case["decision"] = "clarify"
elif raw["case_id"] in _STALE_CONDITION_EXPECTATIONS:
case.update(_STALE_CONDITION_EXPECTATIONS[raw["case_id"]])
else:
case["decision"] = "answerable"
case["must_have_citations"] = True
if raw["case_id"].startswith("general_") and raw.get("expected_relation") == "indication":
case["condition_mode"] = "general"
cases.append(case)
return cases
# Cases whose entire purpose is that the service REFUSES. Scoring them as
# "should answer" inverts the test: passing would mean the guard is gone.
_ADVERSARIAL_MUST_REFUSE: dict[str, dict[str, Any]] = {
# "Liều famciclovir điều trị cho mèo là bao nhiêu?" -- the formulary covers
# human medicine only. `expected_drug_id: famciclovir` in the source file
# records which monograph the question brushes against, not a licence to
# quote a dose from it.
"unsupported-veterinary": {
"decision": "abstain",
"reason": "out_of_scope_non_human",
"no_citations": True,
},
}
def load_adversarial_10() -> list[dict[str, Any]]:
"""`manual_adversarial_hard10.jsonl` -> battery cases.
`expected_id` (a chunk id such as `p132_t0`) is deliberately dropped: it
identifies a retrieval hit inside the index, which an HTTP response does
not expose. `expected_drug_id` survives as `expected_any_drug_ids`, so the
assertion that remains -- the answer is grounded in the right monograph --
is one this transport can actually verify.
`decision_any` admits `clarify` because several of these are adversarial
precisely by being under-specified (a paediatric dose with no age, an
unsupported veterinary use); refusing to answer is a correct outcome
there, not a failure.
"""
cases = []
for raw in _read(EVALS_DIR / "manual_adversarial_hard10.jsonl"):
case: dict[str, Any] = {
"id": raw["case_id"],
"query": raw["query"],
"category": "adversarial_hard",
# `verify_pdf` belongs here as a first-class success. Most of these
# ask for a number that lives in a table or formula, and the
# service is required to show the source crop and refuse to read
# the figure out of it. Verified 2026-08-19: it returns
# `visual_verification_required` while citing the correct
# monograph, which is the designed outcome, not a miss.
"decision_any": ["answerable", "clarify", "verify_pdf"],
"must_have_citations_if_answerable": True,
}
if raw.get("expected_drug_id"):
case["expected_any_drug_ids"] = [raw["expected_drug_id"]]
cases.append(case)
if raw["case_id"] in _ADVERSARIAL_MUST_REFUSE:
case.update(_ADVERSARIAL_MUST_REFUSE[raw["case_id"]])
# `decision_any` must go, not just be shadowed: the checker reads
# `case.get("decision_any", [case.get("decision")])`, so leaving it
# in place would keep admitting the answers this case exists to
# forbid while the stricter `decision` sat there doing nothing.
case.pop("decision_any", None)
case.pop("expected_any_drug_ids", None)
case.pop("must_have_citations_if_answerable", None)
return cases
SUITES = {
"production60": load_production_60,
"condition20": load_condition_20,
"adversarial10": load_adversarial_10,
}
-97
View File
@@ -1,97 +0,0 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from .artifacts import build_drug_catalog, load_aliases, load_documents, load_parents
from .evaluation import CaseOrigin, EvaluationCase, EvaluationOutcome, summarize
from .in_memory import InMemoryLexicalRetriever, InMemoryParentStore
from .models import EvidenceDecision, QueryIntent, SubjectScope
from .routing import CatalogDrugResolver, QueryRoutingService
from .service import EvidencePolicy, RetrievalService
def read_cases(path: Path) -> list[EvaluationCase]:
with path.open(encoding="utf-8") as handle:
return [
EvaluationCase(
case_id=raw["case_id"],
query=raw["query"],
expected_drug_id=raw.get("expected_drug_id"),
expected_id=raw.get("expected_id"),
origin=CaseOrigin(raw["origin"]),
subject_scope=SubjectScope(raw.get("subject_scope", "human")),
)
for line in handle
if line.strip()
for raw in [json.loads(line)]
]
def run(
cases_path: Path,
documents_path: Path,
parents_path: Path,
aliases_path: Path | None = None,
) -> dict:
documents = load_documents(documents_path)
retrieval = RetrievalService(
InMemoryLexicalRetriever(documents),
InMemoryParentStore(load_parents(parents_path)),
EvidencePolicy(),
)
service = QueryRoutingService(
retrieval,
CatalogDrugResolver(build_drug_catalog(documents, load_aliases(aliases_path))),
)
outcomes = []
details = []
for case in read_cases(cases_path):
result = service.retrieve(
case.query,
case.subject_scope,
QueryIntent.FACT_LOOKUP,
)
retrieved = (
tuple(item.evidence_id for item in result.evidence)
if result.decision != EvidenceDecision.ABSTAIN
else ()
)
outcome = EvaluationOutcome(
case=case,
retrieved_ids=retrieved,
resolved_drug_id=result.resolved_drug_id,
drug_resolution_status=result.drug_resolution_status,
)
outcomes.append(outcome)
details.append({
"case_id": case.case_id,
"passed": outcome.passed,
"decision": result.decision,
"reason": result.reason,
"expected_id": case.expected_id,
"retrieved_ids": retrieved,
"expected_drug_id": case.expected_drug_id,
"resolved_drug_id": result.resolved_drug_id,
"drug_resolution_status": result.drug_resolution_status,
})
return {**summarize(outcomes), "details": details}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--cases", type=Path, required=True)
parser.add_argument("--documents", type=Path, required=True)
parser.add_argument("--parents", type=Path, required=True)
parser.add_argument("--aliases", type=Path)
args = parser.parse_args()
print(json.dumps(
run(args.cases, args.documents, args.parents, args.aliases),
ensure_ascii=False,
indent=2,
))
if __name__ == "__main__":
main()
+144
View File
@@ -0,0 +1,144 @@
"""Run every eval dataset in `evals/` against a deployment, in one command.
Before this, only `production_manual_60.jsonl` had a working runner. The other
two datasets sat unreferenced by any code: one's intended consumer was a
metrics dataclass with no entry point, the other's (`rag/run_eval.py`) had
been unable to load the corpus since the ingestion artifact format changed.
So two thirds of the checked-in eval coverage was never actually checked, and
nothing in the repo said so.
Reuses `run_manual_battery.py`'s checker rather than reimplementing it -- a
second copy of the invariants would drift from the first, and the point of
this script is to widen coverage, not to fork the definition of "correct".
`evals/adapters.py` holds the per-dataset schema mapping and the reasoning
behind each choice.
Usage:
python3 scripts/run_all_evals.py --base-url https://realvuxbaro.me \\
--output-dir /tmp/evals
"""
from __future__ import annotations
import argparse
import importlib.util
import json
import sys
import time
from pathlib import Path
from typing import Any
SERVICE_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(SERVICE_ROOT))
from evals.adapters import SUITES # noqa: E402
def _load_battery_module():
"""Import the sibling script by path -- `scripts/` is not a package."""
path = Path(__file__).resolve().parent / "run_manual_battery.py"
spec = importlib.util.spec_from_file_location("run_manual_battery", path)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
return module
def run_suite(
battery: Any,
name: str,
cases: list[dict[str, Any]],
endpoint: str,
timeout: float,
run_id: str,
output: Path,
) -> tuple[int, int]:
passed = 0
with output.open("w", encoding="utf-8") as handle:
for index, case in enumerate(cases, start=1):
# Split out rather than nested: reusing the outer quote inside an
# f-string is a 3.12+ syntax, and ruff targets 3.11 here.
default_conversation = f"{name}-{case['id']}"
base = case.get("conversation_id") or default_conversation
conversation_id = f"{base}-{run_id}"
payload = {"content": case["query"], "conversationId": conversation_id}
started = time.monotonic()
try:
raw = battery._post(endpoint, payload, timeout)
response = battery._normalise_response(raw)
failures = battery._check(case, response)
error = None
except Exception as exc: # noqa: BLE001 - recorded, not swallowed
response, failures, error = {}, [f"request error: {exc}"], repr(exc)
elapsed = round(time.monotonic() - started, 3)
ok = not failures
passed += int(ok)
handle.write(
json.dumps(
{
"suite": name,
"case": case,
"passed": ok,
"failures": failures,
"elapsed_seconds": elapsed,
"error": error,
"response": response,
},
ensure_ascii=False,
)
+ "\n"
)
handle.flush()
print(
f" [{index:02d}/{len(cases)}] {case['id']} "
f"{'PASS' if ok else 'FAIL'} {elapsed:.1f}s {'; '.join(failures)}",
flush=True,
)
return passed, len(cases)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--timeout", type=float, default=120.0)
parser.add_argument("--suites", help="comma-separated subset of: " + ", ".join(SUITES))
parser.add_argument("--run-id", default=str(int(time.time())))
args = parser.parse_args()
wanted = (
[s.strip() for s in args.suites.split(",") if s.strip()]
if args.suites
else list(SUITES)
)
unknown = [s for s in wanted if s not in SUITES]
if unknown:
print(f"unknown suite(s): {unknown}; known: {list(SUITES)}", file=sys.stderr)
return 2
battery = _load_battery_module()
endpoint = args.base_url.rstrip("/") + "/api/chat"
args.output_dir.mkdir(parents=True, exist_ok=True)
totals: list[tuple[str, int, int]] = []
for name in wanted:
cases = SUITES[name]()
print(f"\n=== {name} ({len(cases)} cases) ===", flush=True)
passed, total = run_suite(
battery, name, cases, endpoint, args.timeout, args.run_id,
args.output_dir / f"{name}.jsonl",
)
totals.append((name, passed, total))
print("\n=== SUMMARY ===")
grand_passed = grand_total = 0
for name, passed, total in totals:
print(f" {name:<16} {passed}/{total}")
grand_passed += passed
grand_total += total
print(f" {'TOTAL':<16} {grand_passed}/{grand_total}")
return 0 if grand_passed == grand_total else 1
if __name__ == "__main__":
raise SystemExit(main())
+1 -1
View File
@@ -27,7 +27,7 @@ export function AccountMenu() {
if (!user) {
return (
<Link
href="/admin/login"
href="/login"
className="flex items-center gap-1.5 rounded-full border border-border-subtle bg-surface px-3 py-1.5 text-xs font-semibold text-txt-secondary hover:bg-surface-hover"
>
<LogIn className="h-3.5 w-3.5" />
+135
View File
@@ -0,0 +1,135 @@
"use client";
import { useState } from "react";
import Link from "next/link";
import { useRouter } from "next/navigation";
import { login, AuthError } from "@duoc-thu/api-client";
interface LoginFormProps {
heading: string;
/** `/admin/login` sets this. It changes only where a successful login
* lands and what a non-admin is told afterwards -- never whether the
* login itself is allowed to happen, which is auth-service's call. */
requireAdmin?: boolean;
}
/**
* Shared by `/login` (any doctor) and `/admin/login` (administrators).
*
* The two pages used to be one page, and it only served admins: `AccountMenu`
* pointed its "Đăng nhập" button at `/admin/login`, so a doctor signing in
* from the chat UI with a `user` account got a red "Tài khoản này không có
* quyền quản trị" while the header beside it switched to their name. Both were
* accurate and together they were nonsense -- the login had succeeded and the
* session cookie was already set; only the admin redirect had not happened.
*
* So a non-admin result is reported here as what it is: signed in, without
* administrative rights. Not an error, and not silently swallowed either.
*/
export function LoginForm({ heading, requireAdmin = false }: LoginFormProps) {
const router = useRouter();
const [username, setUsername] = useState("");
const [password, setPassword] = useState("");
const [error, setError] = useState<string | null>(null);
const [signedInAs, setSignedInAs] = useState<string | null>(null);
const [pending, setPending] = useState(false);
async function onSubmit(event: React.FormEvent) {
event.preventDefault();
setError(null);
setSignedInAs(null);
setPending(true);
try {
const user = await login({ username, password });
if (requireAdmin && user.role !== "admin") {
// Signed in, just not an administrator. Say so plainly and offer the
// way onward rather than leaving them on a dead form.
setSignedInAs(user.username);
return;
}
router.push(user.role === "admin" && requireAdmin ? "/admin" : "/");
router.refresh();
} catch (err) {
setError(
err instanceof AuthError && err.status === 401
? "Sai tên đăng nhập hoặc mật khẩu."
: "Không thể đăng nhập lúc này. Vui lòng thử lại."
);
} finally {
setPending(false);
}
}
if (signedInAs) {
return (
<div className="flex flex-1 items-center justify-center p-6">
<div className="w-full max-w-sm space-y-3 rounded-2xl border border-border-subtle bg-surface p-6 shadow-sm">
<h1 className="text-lg font-bold text-txt-primary">
Đã đăng nhập với tài khoản {signedInAs}
</h1>
<p className="text-xs text-txt-secondary">
Tài khoản này không quyền quản trị, nhưng bạn đã đăng nhập
thể tra cứu bình thường.
</p>
<Link
href="/"
className="block w-full rounded-lg bg-accent-primary py-2 text-center text-sm font-semibold text-txt-inverse"
>
Vào trang tra cứu
</Link>
</div>
</div>
);
}
return (
<div className="flex flex-1 items-center justify-center p-6">
<form
onSubmit={onSubmit}
className="w-full max-w-sm space-y-4 rounded-2xl border border-border-subtle bg-surface p-6 shadow-sm"
>
<h1 className="text-lg font-bold text-txt-primary">{heading}</h1>
<div className="space-y-1">
<label className="text-xs font-semibold text-txt-secondary" htmlFor="username">
Tên đăng nhập
</label>
<input
id="username"
className="w-full rounded-lg border border-border-subtle bg-app px-3 py-2 text-sm text-txt-primary"
value={username}
onChange={(e) => setUsername(e.target.value)}
autoComplete="username"
required
/>
</div>
<div className="space-y-1">
<label className="text-xs font-semibold text-txt-secondary" htmlFor="password">
Mật khẩu
</label>
<input
id="password"
type="password"
className="w-full rounded-lg border border-border-subtle bg-app px-3 py-2 text-sm text-txt-primary"
value={password}
onChange={(e) => setPassword(e.target.value)}
autoComplete="current-password"
required
/>
</div>
{error && <p className="text-xs font-medium text-status-danger">{error}</p>}
<button
type="submit"
disabled={pending}
className="w-full rounded-lg bg-accent-primary py-2 text-sm font-semibold text-txt-inverse disabled:opacity-60"
>
{pending ? "Đang đăng nhập..." : "Đăng nhập"}
</button>
{!requireAdmin && (
<p className="text-center text-[0.7rem] text-txt-muted">
Không bắt buộc bạn vẫn tra cứu đưc không cần đăng nhập.
</p>
)}
</form>
</div>
);
}
+6 -79
View File
@@ -1,82 +1,9 @@
"use client";
import { useState } from "react";
import { useRouter } from "next/navigation";
import { login, AuthError } from "@duoc-thu/api-client";
import { LoginForm } from "../../_components/LoginForm";
/** Administrator entry point. The form itself is shared with `/login`; the
* only difference is that a successful non-admin login is told it lacks
* administrative rights and pointed at the main app, instead of being shown
* a failure message it had already disproved by signing in. */
export default function AdminLoginPage() {
const router = useRouter();
const [username, setUsername] = useState("");
const [password, setPassword] = useState("");
const [error, setError] = useState<string | null>(null);
const [pending, setPending] = useState(false);
async function onSubmit(event: React.FormEvent) {
event.preventDefault();
setError(null);
setPending(true);
try {
const user = await login({ username, password });
if (user.role !== "admin") {
setError("Tài khoản này không có quyền quản trị.");
return;
}
router.push("/admin");
router.refresh();
} catch (err) {
setError(
err instanceof AuthError && err.status === 401
? "Sai tên đăng nhập hoặc mật khẩu."
: "Không thể đăng nhập lúc này. Vui lòng thử lại."
);
} finally {
setPending(false);
}
}
return (
<div className="flex flex-1 items-center justify-center p-6">
<form
onSubmit={onSubmit}
className="w-full max-w-sm space-y-4 rounded-2xl border border-border-subtle bg-surface p-6 shadow-sm"
>
<h1 className="text-lg font-bold text-txt-primary">Đăng nhập quản trị</h1>
<div className="space-y-1">
<label className="text-xs font-semibold text-txt-secondary" htmlFor="username">
Tên đăng nhập
</label>
<input
id="username"
className="w-full rounded-lg border border-border-subtle bg-app px-3 py-2 text-sm text-txt-primary"
value={username}
onChange={(e) => setUsername(e.target.value)}
autoComplete="username"
required
/>
</div>
<div className="space-y-1">
<label className="text-xs font-semibold text-txt-secondary" htmlFor="password">
Mật khẩu
</label>
<input
id="password"
type="password"
className="w-full rounded-lg border border-border-subtle bg-app px-3 py-2 text-sm text-txt-primary"
value={password}
onChange={(e) => setPassword(e.target.value)}
autoComplete="current-password"
required
/>
</div>
{error && <p className="text-xs font-medium text-status-danger">{error}</p>}
<button
type="submit"
disabled={pending}
className="w-full rounded-lg bg-accent-primary py-2 text-sm font-semibold text-txt-inverse disabled:opacity-60"
>
{pending ? "Đang đăng nhập..." : "Đăng nhập"}
</button>
</form>
</div>
);
return <LoginForm heading="Đăng nhập quản trị" requireAdmin />;
}
+13
View File
@@ -0,0 +1,13 @@
import { LoginForm } from "../_components/LoginForm";
/**
* Login for any doctor, admin or not. Before this existed, `AccountMenu`'s
* "Đăng nhập" button sent everyone to `/admin/login`, which refused every
* non-admin account *after* already signing them in.
*
* Signing in stays entirely optional: anonymous chat is unchanged, and this
* page says so rather than implying an account is required.
*/
export default function LoginPage() {
return <LoginForm heading="Đăng nhập" />;
}