Mdmem Agent
8fd3e37b67
- Delete PLAN.md (LLM agent development brief) - Delete SECURITY_ANALYSIS.md (internal audit report) - Delete deploy/ (environment-specific K8s manifests with hardcoded infra) - Remove eval/corpus/, eval/results/, eval/queries.synthetic.yaml, eval/README.md - Move queries.example.yaml to root, update all path references
272 lines
9.2 KiB
Python
272 lines
9.2 KiB
Python
import asyncio
|
|
import json
|
|
from dataclasses import replace
|
|
|
|
import pytest
|
|
|
|
from mdmem.embed import FakeEmbedder
|
|
from mdmem.evalharness import (
|
|
EvalQuery,
|
|
Variant,
|
|
bootstrap_ci,
|
|
calibrate_min_score,
|
|
load_queries,
|
|
mrr_at,
|
|
ndcg_at,
|
|
pick_default,
|
|
recall_at,
|
|
render_table,
|
|
run_eval_cli,
|
|
run_matrix,
|
|
run_variant,
|
|
)
|
|
from mdmem.service import MemoryService
|
|
|
|
|
|
def test_recall_mrr_ndcg():
|
|
ranked = ["a", "b", "c"]
|
|
assert recall_at(ranked, ["a", "z"], 5) == 0.5
|
|
assert recall_at(ranked, ["a"], 1) == 1.0
|
|
assert mrr_at(ranked, ["b"]) == 0.5
|
|
assert mrr_at(ranked, ["z"]) == 0.0
|
|
assert ndcg_at(ranked, ["a"]) == 1.0
|
|
assert 0 < ndcg_at(ranked, ["c"]) < 1
|
|
|
|
|
|
def test_metrics_are_nan_for_abstain_queries():
|
|
import math
|
|
|
|
assert math.isnan(recall_at(["a"], [], 5))
|
|
assert math.isnan(mrr_at(["a"], []))
|
|
|
|
|
|
def test_bootstrap_ci_brackets_the_mean():
|
|
values = [0.2, 0.4, 0.6, 0.8, 1.0]
|
|
lo, hi = bootstrap_ci(values, rounds=500)
|
|
assert lo < 0.6 < hi
|
|
|
|
|
|
def test_load_queries_validates(tmp_path):
|
|
p = tmp_path / "q.yaml"
|
|
p.write_text("- q: hallo\n relevant: [01J]\n category: single\n", encoding="utf-8")
|
|
qs = load_queries(p)
|
|
assert qs[0].q == "hallo" and qs[0].relevant == ["01J"]
|
|
|
|
p.write_text("- relevant: []\n", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="no 'q'"):
|
|
load_queries(p)
|
|
|
|
p.write_text("- q: x\n category: nonsense\n", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="unknown category"):
|
|
load_queries(p)
|
|
|
|
|
|
def test_the_shipped_example_query_set_parses():
|
|
qs = load_queries("queries.example.yaml")
|
|
assert len(qs) >= 5
|
|
assert {q.category for q in qs} >= {"single", "temporal", "update", "abstain"}
|
|
|
|
|
|
@pytest.fixture()
|
|
async def seeded(settings):
|
|
svc = MemoryService.build(settings, embedder=FakeEmbedder(settings.embed_dim))
|
|
await svc.start(watch=False)
|
|
ids = {}
|
|
for title, body, kw in [
|
|
("llama-swap routet Modelle", "Bifrost fronts llama-swap auf evo-x2", ["router"]),
|
|
("Kaffeemaschine entkalken", "alle drei Monate mit Essig entkalken", ["kaffee"]),
|
|
("Longhorn Storage", "Longhorn liefert RWO Volumes im Cluster", ["storage"]),
|
|
]:
|
|
r = await svc.write_note(title=title, body=body, type="fact", summary=title, keywords=kw)
|
|
await svc.set_state(r["id"], "active")
|
|
ids[title] = r["id"]
|
|
yield svc, ids
|
|
await svc.stop()
|
|
|
|
|
|
async def test_run_variant_scores_a_query_set(seeded):
|
|
svc, ids = seeded
|
|
queries = [
|
|
EvalQuery(q="llama-swap", relevant=[ids["llama-swap routet Modelle"]], category="single"),
|
|
EvalQuery(q="entkalken", relevant=[ids["Kaffeemaschine entkalken"]], category="single"),
|
|
EvalQuery(q="Zahnarzttermin nächste Woche", relevant=[], category="abstain"),
|
|
]
|
|
report = await run_variant(svc, queries, Variant("rrf", fusion="rrf"), max_chars=6000)
|
|
assert report["n_queries"] == 3
|
|
assert report["overall"]["recall@5"] == 1.0
|
|
assert report["overall"]["recall@5_ci95"]
|
|
assert report["abstention"]["n_abstain_queries"] == 1
|
|
assert report["latency_ms_p95"] > 0
|
|
assert report["mean_chars"] > 0
|
|
|
|
|
|
async def test_every_variant_runs_at_the_same_budget(seeded):
|
|
svc, ids = seeded
|
|
queries = [EvalQuery(q="llama-swap Modelle", relevant=[ids["llama-swap routet Modelle"]])]
|
|
reports = []
|
|
for v in (
|
|
Variant("rrf", fusion="rrf"),
|
|
Variant("lex", fusion="rrf", channels=("lex",)),
|
|
Variant("sem", fusion="rrf", channels=("sem",)),
|
|
Variant("cc", fusion="cc", cc_alpha=0.5),
|
|
):
|
|
reports.append(await run_variant(svc, queries, v, max_chars=500))
|
|
assert all(r["mean_chars"] <= 800 for r in reports)
|
|
assert {r["variant"] for r in reports} == {"rrf", "lex", "sem", "cc"}
|
|
table = render_table(reports)
|
|
assert "| rrf |" in table and "R@5" in table
|
|
|
|
|
|
async def test_sem_only_still_ranks_without_lexical_overlap(seeded):
|
|
svc, ids = seeded
|
|
q = [EvalQuery(q="alle drei Monate mit Essig entkalken", relevant=[ids["Kaffeemaschine entkalken"]])]
|
|
report = await run_variant(svc, q, Variant("sem", fusion="rrf", channels=("sem",)), max_chars=6000)
|
|
assert report["overall"]["recall@5"] == 1.0
|
|
|
|
|
|
async def test_matrix_rebuilds_for_a_dim_variant_and_cleans_up(settings, seeded):
|
|
svc, ids = seeded
|
|
queries = [EvalQuery(q="Longhorn", relevant=[ids["Longhorn Storage"]])]
|
|
reports = await run_matrix(
|
|
settings,
|
|
queries,
|
|
[Variant("rrf", fusion="rrf"), Variant("dim-8", fusion="rrf", embed_dim=8)],
|
|
embedder_factory=lambda s: FakeEmbedder(s.embed_dim),
|
|
max_chars=6000,
|
|
)
|
|
assert len(reports) == 2
|
|
assert reports[1]["config"]["embed_dim"] == 8
|
|
assert not settings.index_db.with_name("eval-dim8.db").exists()
|
|
|
|
|
|
async def test_matrix_never_mutates_the_configured_live_index(settings, seeded):
|
|
svc, ids = seeded
|
|
before = settings.index_db.read_bytes()
|
|
reports = await run_matrix(
|
|
settings,
|
|
[EvalQuery(q="Longhorn", relevant=[ids["Longhorn Storage"]])],
|
|
[Variant("rrf-no-trigram", fusion="rrf", trigram=False)],
|
|
embedder_factory=lambda s: FakeEmbedder(s.embed_dim),
|
|
)
|
|
assert reports
|
|
assert settings.index_db.read_bytes() == before
|
|
assert svc.db.read().execute("SELECT 1 FROM sqlite_master WHERE name='chunks_tri'").fetchone()
|
|
|
|
|
|
def test_pick_default_keeps_the_baseline_when_nothing_clears_the_noise():
|
|
reports = [
|
|
{"variant": "rrf", "overall": {"ndcg@10": 0.70, "ndcg@10_ci95": [0.60, 0.80]}},
|
|
{"variant": "cc-a0.5", "overall": {"ndcg@10": 0.74, "ndcg@10_ci95": [0.64, 0.84]}},
|
|
]
|
|
out = pick_default(reports)
|
|
assert out["recommendation"] == "rrf" and "overlap" in out["reason"]
|
|
|
|
|
|
def test_pick_default_switches_on_a_clear_win():
|
|
reports = [
|
|
{"variant": "rrf", "overall": {"ndcg@10": 0.50, "ndcg@10_ci95": [0.45, 0.55]}},
|
|
{"variant": "cc-a0.5", "overall": {"ndcg@10": 0.80, "ndcg@10_ci95": [0.75, 0.85]}},
|
|
]
|
|
assert pick_default(reports)["recommendation"] == "cc-a0.5"
|
|
|
|
|
|
def test_pick_default_rejects_category_regression():
|
|
reports = [
|
|
{
|
|
"variant": "rrf",
|
|
"overall": {"ndcg@10": 0.50, "ndcg@10_ci95": [0.45, 0.55]},
|
|
"per_category": {
|
|
"single": {"ndcg@10": 0.50},
|
|
"temporal": {"ndcg@10": 0.90},
|
|
},
|
|
},
|
|
{
|
|
"variant": "cc-a0.5",
|
|
"overall": {"ndcg@10": 0.80, "ndcg@10_ci95": [0.75, 0.85]},
|
|
"per_category": {
|
|
"single": {"ndcg@10": 1.00},
|
|
"temporal": {"ndcg@10": 0.40},
|
|
},
|
|
},
|
|
]
|
|
out = pick_default(reports)
|
|
assert out["recommendation"] == "rrf"
|
|
|
|
|
|
def test_min_score_calibration_finds_a_separating_threshold():
|
|
report = {
|
|
"queries": [
|
|
{"is_abstain_query": False, "top_score": 0.9, "ranked": ["a"], "recall@10": 1.0},
|
|
{"is_abstain_query": False, "top_score": 0.8, "ranked": ["b"], "recall@10": 1.0},
|
|
{"is_abstain_query": True, "top_score": 0.2, "ranked": ["c"], "recall@10": float("nan")},
|
|
{"is_abstain_query": True, "top_score": 0.1, "ranked": ["d"], "recall@10": float("nan")},
|
|
]
|
|
}
|
|
best = calibrate_min_score(report)
|
|
assert best["f1"] == 1.0 and 0.2 < best["min_score"] <= 0.8
|
|
|
|
|
|
async def test_cli_reports_and_writes_a_json_file(settings, seeded, tmp_path, capsys):
|
|
svc, ids = seeded
|
|
qfile = tmp_path / "q.yaml"
|
|
qfile.write_text(
|
|
f"- q: llama-swap\n relevant: [{ids['llama-swap routet Modelle']}]\n category: single\n",
|
|
encoding="utf-8",
|
|
)
|
|
out = tmp_path / "report.json"
|
|
|
|
args = type(
|
|
"Args",
|
|
(),
|
|
{
|
|
"queries": str(qfile),
|
|
"config": "rrf",
|
|
"out": str(out),
|
|
"ablate": False,
|
|
"fake_embeddings": True,
|
|
},
|
|
)()
|
|
# run_eval_cli calls asyncio.run() internally, which is illegal from inside a
|
|
# running loop -- drive the real CLI off the loop instead of reimplementing it.
|
|
assert await asyncio.to_thread(run_eval_cli, replace(settings), args) == 0
|
|
text = out.read_text(encoding="utf-8")
|
|
parsed = json.loads(text, parse_constant=lambda constant: (_ for _ in ()).throw(ValueError(constant)))
|
|
assert parsed["reports"][0]["overall"]["recall@5"] == 1.0
|
|
assert "NaN" not in text
|
|
|
|
|
|
async def test_matrix_uses_fair_per_variant_embedding_caches(settings, monkeypatch):
|
|
seen = []
|
|
|
|
class DummyService:
|
|
def __init__(self, run_settings, embedder=None):
|
|
self.settings = run_settings
|
|
seen.append(run_settings.cache_db)
|
|
|
|
async def start(self, **_kw):
|
|
return None
|
|
|
|
async def reindex(self, **_kw):
|
|
return {}
|
|
|
|
async def stop(self):
|
|
return None
|
|
|
|
async def search(self, *_args, **_kw):
|
|
from mdmem.search import SearchResponse
|
|
|
|
return SearchResponse()
|
|
|
|
monkeypatch.setattr(
|
|
"mdmem.evalharness.MemoryService.build",
|
|
lambda run_settings, embedder=None: DummyService(run_settings, embedder),
|
|
)
|
|
await run_matrix(
|
|
settings,
|
|
[EvalQuery(q="nothing", relevant=[])],
|
|
[Variant("a"), Variant("b")],
|
|
embedder_factory=lambda s: FakeEmbedder(s.embed_dim),
|
|
)
|
|
assert len(seen) == 2
|
|
assert len(set(seen)) == 2
|