MarkusandClaude Opus 5
f5dce6ce96
Closes SECURITY_ANALYSIS.md M-14, M-15 and M-16.
M-15: unavailable metrics stayed float("nan") and json.dumps emitted bare NaN,
which RFC 8259 does not permit — the committed report carried 120 of them. They
now serialize as null, and the test named as a CLI writer test invokes the real
CLI and parses with a strict parser instead of hand-writing a different object.
M-14: pick_default compared only overall nDCG and confidence intervals, so a
variant with a catastrophic temporal or update regression could still be
recommended. M-16: each variant got an isolated index but shared the configured
embedding cache, so the first real-model variant paid query embedding latency
that later variants read from cache, making p95 incomparable.
The synthetic smoke report is regenerated against the current code and renamed to
match the date it actually carries. rrf, rrf-inbox-1.0 and rrf-bm25-flat now
score identically, which is expected rather than the shared-state corruption an
earlier report showed: no seeded note is in inbox, and lexical subchannels fuse
by ordinal rank, so a BM25 weight change only moves the score if it reorders.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0139vHnNpS2sLNELjrAHX5Cu
8281 lines
246 KiB
JSON
8281 lines
246 KiB
JSON
{
|
|
"date": "2026-09-13T07:16:53Z",
|
|
"embed_model": "llamaswap/qwen3-embedding-4b",
|
|
"embed_dim": 2560,
|
|
"max_chars": 6000,
|
|
"n_queries": 18,
|
|
"decision": {
|
|
"recommendation": "rrf",
|
|
"reason": "no variant beat the baseline with non-overlapping 95 % CIs",
|
|
"baseline_ndcg@10": 0.8012
|
|
},
|
|
"min_score_calibration": {
|
|
"min_score": 0.032787,
|
|
"f1": 0.19999999999999998
|
|
},
|
|
"reports": [
|
|
{
|
|
"variant": "rrf",
|
|
"config": {
|
|
"fusion": "rrf",
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.7537,
|
|
"mrr@10_ci95": [
|
|
0.5923,
|
|
0.9062
|
|
],
|
|
"ndcg@10": 0.8012,
|
|
"ndcg@10_ci95": [
|
|
0.6777,
|
|
0.9183
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.6716,
|
|
"mrr@10_ci95": [
|
|
0.4752,
|
|
0.875
|
|
],
|
|
"ndcg@10": 0.7511,
|
|
"ndcg@10_ci95": [
|
|
0.6023,
|
|
0.9077
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 773.9,
|
|
"latency_ms_p95": 3.65,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.032018442622950824
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03177805800756621
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.0315136476426799
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.14285714285714285,
|
|
"ndcg@10": 0.3333333333333333,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03047794966520434
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.03149801587301587
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.25,
|
|
"ndcg@10": 0.43067655807339306,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03225806451612903
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "lex-only",
|
|
"config": {
|
|
"fusion": "rrf",
|
|
"channels": [
|
|
"lex"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 0.875,
|
|
"recall@10_ci95": [
|
|
0.6875,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.875,
|
|
"mrr@10_ci95": [
|
|
0.6875,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8629,
|
|
"ndcg@10_ci95": [
|
|
0.6754,
|
|
1.0
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 0.8333,
|
|
"recall@10_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8333,
|
|
"mrr@10_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8333,
|
|
"ndcg@10_ci95": [
|
|
0.5833,
|
|
1.0
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 320.2,
|
|
"latency_ms_p95": 1.37,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 200,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 267,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 267,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 411,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 211,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 145,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 526,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 0.0,
|
|
"mrr@10": 0.0,
|
|
"ndcg@10": 0.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 211,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 105,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 0.0,
|
|
"mrr@10": 0.0,
|
|
"ndcg@10": 0.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 295,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 395,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 159,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 492,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 404,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 598,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 230,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 467,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 381,
|
|
"top_score": 0.01639344262295082
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "sem-only",
|
|
"config": {
|
|
"fusion": "rrf",
|
|
"channels": [
|
|
"sem"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.7188,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
0.9375
|
|
],
|
|
"recall@10": 0.9375,
|
|
"recall@10_ci95": [
|
|
0.8125,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.4689,
|
|
"mrr@10_ci95": [
|
|
0.3037,
|
|
0.6418
|
|
],
|
|
"ndcg@10": 0.5684,
|
|
"ndcg@10_ci95": [
|
|
0.4268,
|
|
0.7027
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.6667,
|
|
"recall@5_ci95": [
|
|
0.4167,
|
|
0.9167
|
|
],
|
|
"recall@10": 0.9167,
|
|
"recall@10_ci95": [
|
|
0.75,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.3891,
|
|
"mrr@10_ci95": [
|
|
0.2319,
|
|
0.5736
|
|
],
|
|
"ndcg@10": 0.5138,
|
|
"ndcg@10_ci95": [
|
|
0.3693,
|
|
0.6703
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.6667,
|
|
"mrr@10_ci95": [
|
|
0.3333,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.6489,
|
|
"ndcg@10_ci95": [
|
|
0.5,
|
|
0.7977
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 768.7,
|
|
"latency_ms_p95": 2.43,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.25,
|
|
"ndcg@10": 0.43067655807339306,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.2,
|
|
"ndcg@10": 0.38685280723454163,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.1,
|
|
"ndcg@10": 0.2890648263178879,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.3333333333333333,
|
|
"ndcg@10": 0.5,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 791,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.14285714285714285,
|
|
"ndcg@10": 0.3333333333333333,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 0.0,
|
|
"mrr@10": 0.0,
|
|
"ndcg@10": 0.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.14285714285714285,
|
|
"ndcg@10": 0.3333333333333333,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.3333333333333333,
|
|
"ndcg@10": 0.5,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.7977228895450266,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.01639344262295082
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "cc-a0.3",
|
|
"config": {
|
|
"fusion": "cc",
|
|
"cc_alpha": 0.3,
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8917,
|
|
"mrr@10_ci95": [
|
|
0.7271,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9032,
|
|
"ndcg@10_ci95": [
|
|
0.7783,
|
|
1.0
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8556,
|
|
"mrr@10_ci95": [
|
|
0.6417,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8871,
|
|
"ndcg@10_ci95": [
|
|
0.7206,
|
|
1.0
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 773.9,
|
|
"latency_ms_p95": 3.17,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9912579277069259
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9904500744082242
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9913959472096767
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9914383857278162
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.9987526142988801
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.1,
|
|
"ndcg@10": 0.2890648263178879,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.9983702072989499
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9810573521010515
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.9825565327747925
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9799881435001647
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9998781241313157
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9841474814185607
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9913960454610302
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9986076663495764
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "cc-a0.4",
|
|
"config": {
|
|
"fusion": "cc",
|
|
"cc_alpha": 0.4,
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8917,
|
|
"mrr@10_ci95": [
|
|
0.7271,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9032,
|
|
"ndcg@10_ci95": [
|
|
0.7783,
|
|
1.0
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8556,
|
|
"mrr@10_ci95": [
|
|
0.6417,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8871,
|
|
"ndcg@10_ci95": [
|
|
0.7206,
|
|
1.0
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 773.9,
|
|
"latency_ms_p95": 3.25,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9883439036092346
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9872667658776323
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9885279296129024
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9885845143037548
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.9983368190651736
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.1,
|
|
"ndcg@10": 0.2890648263178879,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.9978269430652666
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9747431361347353
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.9767420436997234
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9733175246668864
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9998374988417544
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9788633085580809
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9885280606147069
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9981435551327686
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "cc-a0.5",
|
|
"config": {
|
|
"fusion": "cc",
|
|
"cc_alpha": 0.5,
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8917,
|
|
"mrr@10_ci95": [
|
|
0.7271,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9032,
|
|
"ndcg@10_ci95": [
|
|
0.7783,
|
|
1.0
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8556,
|
|
"mrr@10_ci95": [
|
|
0.6417,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8871,
|
|
"ndcg@10_ci95": [
|
|
0.7206,
|
|
1.0
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 773.9,
|
|
"latency_ms_p95": 3.14,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9854298795115433
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9840834573470404
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.985659912016128
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9857306428796936
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.9979210238314669
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.1,
|
|
"ndcg@10": 0.2890648263178879,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.9972836788315833
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9684289201684191
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.9709275546246543
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.966646905833608
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.999796873552193
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9735791356976011
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9856600757683837
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9976794439159608
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "cc-a0.6",
|
|
"config": {
|
|
"fusion": "cc",
|
|
"cc_alpha": 0.6,
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8917,
|
|
"mrr@10_ci95": [
|
|
0.7271,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9032,
|
|
"ndcg@10_ci95": [
|
|
0.7783,
|
|
1.0
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8556,
|
|
"mrr@10_ci95": [
|
|
0.6417,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8871,
|
|
"ndcg@10_ci95": [
|
|
0.7206,
|
|
1.0
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 773.9,
|
|
"latency_ms_p95": 2.92,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9825158554138519
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9809001488164485
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9827918944193536
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9828767714556323
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.9975052285977603
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.1,
|
|
"ndcg@10": 0.2890648263178879,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.9967404145978999
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.962114704202103
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.9651130655495851
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9599762870003296
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9997562482626317
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9682949628371215
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9827920909220604
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9972153326991529
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "cc-a0.7",
|
|
"config": {
|
|
"fusion": "cc",
|
|
"cc_alpha": 0.7,
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8917,
|
|
"mrr@10_ci95": [
|
|
0.7271,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9032,
|
|
"ndcg@10_ci95": [
|
|
0.7783,
|
|
1.0
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8556,
|
|
"mrr@10_ci95": [
|
|
0.6417,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8871,
|
|
"ndcg@10_ci95": [
|
|
0.7206,
|
|
1.0
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 773.9,
|
|
"latency_ms_p95": 3.03,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9796018313161605
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9777168402858566
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9799238768225792
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.980022900031571
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.9970894333640536
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.1,
|
|
"ndcg@10": 0.2890648263178879,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.9961971503642165
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9558004882357868
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.959298576474516
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9533056681670512
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9997156229730703
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9630107899766417
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9799241060757371
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9967512214823451
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "cc-a0.8",
|
|
"config": {
|
|
"fusion": "cc",
|
|
"cc_alpha": 0.8,
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8917,
|
|
"mrr@10_ci95": [
|
|
0.7271,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9032,
|
|
"ndcg@10_ci95": [
|
|
0.7783,
|
|
1.0
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8556,
|
|
"mrr@10_ci95": [
|
|
0.6417,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8871,
|
|
"ndcg@10_ci95": [
|
|
0.7206,
|
|
1.0
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 773.7,
|
|
"latency_ms_p95": 3.0,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9766878072184693
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9745335317552646
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9770558592258047
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9771690286075098
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 791,
|
|
"top_score": 0.996673638130347
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.1,
|
|
"ndcg@10": 0.2890648263178879,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.9956538861305332
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9494862722694706
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.9534840873994468
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9466350493337727
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9996749976835089
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9577266171161619
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9770561212294139
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9962871102655372
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "cc-a0.9",
|
|
"config": {
|
|
"fusion": "cc",
|
|
"cc_alpha": 0.9,
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8604,
|
|
"mrr@10_ci95": [
|
|
0.6958,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8802,
|
|
"ndcg@10_ci95": [
|
|
0.7573,
|
|
0.9879
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8139,
|
|
"mrr@10_ci95": [
|
|
0.6056,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8564,
|
|
"ndcg@10_ci95": [
|
|
0.6954,
|
|
1.0
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 770.9,
|
|
"latency_ms_p95": 3.19,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9737737831207779
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9713502232246727
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9741878416290303
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9743151571834484
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 791,
|
|
"top_score": 0.9962578428966404
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.1,
|
|
"ndcg@10": 0.2890648263178879,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.9951106218968498
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9431720563031545
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.9476695983243776
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9451106218968499
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9996343723939475
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.9524424442556821
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9741881363830905
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 1.0
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.9958229990487294
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "rrf-no-trigram",
|
|
"config": {
|
|
"fusion": "rrf",
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
],
|
|
"trigram": false
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.9062,
|
|
"recall@5_ci95": [
|
|
0.75,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.7746,
|
|
"mrr@10_ci95": [
|
|
0.6272,
|
|
0.9062
|
|
],
|
|
"ndcg@10": 0.8178,
|
|
"ndcg@10_ci95": [
|
|
0.707,
|
|
0.9226
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.9167,
|
|
"recall@5_ci95": [
|
|
0.75,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.6994,
|
|
"mrr@10_ci95": [
|
|
0.5149,
|
|
0.875
|
|
],
|
|
"ndcg@10": 0.774,
|
|
"ndcg@10_ci95": [
|
|
0.6343,
|
|
0.9077
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.8989,
|
|
"ndcg@10_ci95": [
|
|
0.7977,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 771.4,
|
|
"latency_ms_p95": 2.86,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.032018442622950824
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03177805800756621
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.0315136476426799
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 791,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.14285714285714285,
|
|
"ndcg@10": 0.3333333333333333,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03047794966520434
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.01639344262295082
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.25,
|
|
"ndcg@10": 0.43067655807339306,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03225806451612903
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.7977228895450266,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "rrf-inbox-1.0",
|
|
"config": {
|
|
"fusion": "rrf",
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
],
|
|
"inbox_weight": 1.0
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.7537,
|
|
"mrr@10_ci95": [
|
|
0.5923,
|
|
0.9062
|
|
],
|
|
"ndcg@10": 0.8012,
|
|
"ndcg@10_ci95": [
|
|
0.6777,
|
|
0.9183
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.6716,
|
|
"mrr@10_ci95": [
|
|
0.4752,
|
|
0.875
|
|
],
|
|
"ndcg@10": 0.7511,
|
|
"ndcg@10_ci95": [
|
|
0.6023,
|
|
0.9077
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 773.9,
|
|
"latency_ms_p95": 3.03,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.032018442622950824
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03177805800756621
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.0315136476426799
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.14285714285714285,
|
|
"ndcg@10": 0.3333333333333333,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03047794966520434
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.03149801587301587
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.25,
|
|
"ndcg@10": 0.43067655807339306,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03225806451612903
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "rrf-bm25-flat",
|
|
"config": {
|
|
"fusion": "rrf",
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
],
|
|
"bm25_weights": [
|
|
1.0,
|
|
1.0,
|
|
1.0,
|
|
1.0
|
|
]
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.7537,
|
|
"mrr@10_ci95": [
|
|
0.5923,
|
|
0.9062
|
|
],
|
|
"ndcg@10": 0.8012,
|
|
"ndcg@10_ci95": [
|
|
0.6777,
|
|
0.9183
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.6716,
|
|
"mrr@10_ci95": [
|
|
0.4752,
|
|
0.875
|
|
],
|
|
"ndcg@10": 0.7511,
|
|
"ndcg@10_ci95": [
|
|
0.6023,
|
|
0.9077
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 773.9,
|
|
"latency_ms_p95": 3.02,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.032018442622950824
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03177805800756621
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.0315136476426799
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.14285714285714285,
|
|
"ndcg@10": 0.3333333333333333,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03047794966520434
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.03149801587301587
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.25,
|
|
"ndcg@10": 0.43067655807339306,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "dim-1024",
|
|
"config": {
|
|
"fusion": "rrf",
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
],
|
|
"embed_dim": 1024
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.9062,
|
|
"recall@5_ci95": [
|
|
0.75,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.8042,
|
|
"mrr@10_ci95": [
|
|
0.6479,
|
|
0.9375
|
|
],
|
|
"ndcg@10": 0.8402,
|
|
"ndcg@10_ci95": [
|
|
0.7205,
|
|
0.9418
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.9167,
|
|
"recall@5_ci95": [
|
|
0.75,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.7389,
|
|
"mrr@10_ci95": [
|
|
0.5444,
|
|
0.9167
|
|
],
|
|
"ndcg@10": 0.803,
|
|
"ndcg@10_ci95": [
|
|
0.655,
|
|
0.9385
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9033,
|
|
"ndcg@10_ci95": [
|
|
0.8066,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 779.1,
|
|
"latency_ms_p95": 1.85,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.030886196246139225
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.031054405392392875
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.030798389007344232
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 751,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 751,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.2,
|
|
"ndcg@10": 0.38685280723454163,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.16666666666666666,
|
|
"ndcg@10": 0.3562071871080222,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 751,
|
|
"top_score": 0.032018442622950824
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 800,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000HRB"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000VSY"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8065735963827293,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 800,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 782,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 783,
|
|
"top_score": 0.03278688524590164
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"variant": "dim-512",
|
|
"config": {
|
|
"fusion": "rrf",
|
|
"channels": [
|
|
"lex",
|
|
"sem"
|
|
],
|
|
"embed_dim": 512
|
|
},
|
|
"n_queries": 18,
|
|
"overall": {
|
|
"recall@5": 0.8438,
|
|
"recall@5_ci95": [
|
|
0.6562,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.7648,
|
|
"mrr@10_ci95": [
|
|
0.5998,
|
|
0.9141
|
|
],
|
|
"ndcg@10": 0.8107,
|
|
"ndcg@10_ci95": [
|
|
0.6878,
|
|
0.9236
|
|
]
|
|
},
|
|
"per_category": {
|
|
"single": {
|
|
"recall@5": 0.8333,
|
|
"recall@5_ci95": [
|
|
0.5833,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 0.728,
|
|
"mrr@10_ci95": [
|
|
0.537,
|
|
0.9167
|
|
],
|
|
"ndcg@10": 0.7924,
|
|
"ndcg@10_ci95": [
|
|
0.6443,
|
|
0.9385
|
|
]
|
|
},
|
|
"multi": {
|
|
"recall@5": 0.75,
|
|
"recall@5_ci95": [
|
|
0.5,
|
|
1.0
|
|
],
|
|
"recall@10": 1.0,
|
|
"recall@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"mrr@10": 1.0,
|
|
"mrr@10_ci95": [
|
|
1.0,
|
|
1.0
|
|
],
|
|
"ndcg@10": 0.9158,
|
|
"ndcg@10_ci95": [
|
|
0.8316,
|
|
1.0
|
|
]
|
|
},
|
|
"temporal": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309
|
|
},
|
|
"update": {
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0
|
|
},
|
|
"abstain": {
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null
|
|
}
|
|
},
|
|
"abstention": {
|
|
"n_abstain_queries": 2,
|
|
"precision": 0.0,
|
|
"recall": 0.0,
|
|
"f1": 0.0
|
|
},
|
|
"mean_chars": 775.1,
|
|
"latency_ms_p95": 1.97,
|
|
"queries": [
|
|
{
|
|
"q": "Welcher Router verteilt meine lokalen Modelle?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03131881575727918
|
|
},
|
|
{
|
|
"q": "llama-swap",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03131881575727918
|
|
},
|
|
{
|
|
"q": "Which gateway sits in front of llama-swap?",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.030679156908665108
|
|
},
|
|
{
|
|
"q": "Wo hält Bifrost die virtuellen Schlüssel?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Wie heißt die GPU-Box?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "storage class for persistent volumes",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03278688524590164
|
|
},
|
|
{
|
|
"q": "Welche StorageClass nutzt der Cluster?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wie oft laufen die Backups?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.1111111111111111,
|
|
"ndcg@10": 0.3010299956639812,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.032018442622950824
|
|
},
|
|
{
|
|
"q": "restic snapshot schedule",
|
|
"category": "single",
|
|
"lang": "en",
|
|
"ranked": [
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.03047794966520434
|
|
},
|
|
{
|
|
"q": "Wie melde ich mich bei den Web-UIs an?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000SWP"
|
|
],
|
|
"recall@5": 0.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.125,
|
|
"ndcg@10": 0.31546487678572877,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.031754032258064516
|
|
},
|
|
{
|
|
"q": "Wie entkalke ich die Kaffeemaschine?",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 808,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Pegel am Traunsee",
|
|
"category": "single",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.03177805800756621
|
|
},
|
|
{
|
|
"q": "Wie heißt der Modellrouter jetzt?",
|
|
"category": "update",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Welches Embedding-Modell und welche Quantisierung?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 1.0,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.032266458495966696
|
|
},
|
|
{
|
|
"q": "Woher kommen die Container-Images und wer rollt sie aus?",
|
|
"category": "multi",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT"
|
|
],
|
|
"recall@5": 0.5,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 1.0,
|
|
"ndcg@10": 0.8315546295836225,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 757,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Was wurde am 11. September 2026 entschieden?",
|
|
"category": "temporal",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000EVX"
|
|
],
|
|
"recall@5": 1.0,
|
|
"recall@10": 1.0,
|
|
"mrr@10": 0.5,
|
|
"ndcg@10": 0.6309297535714575,
|
|
"is_abstain_query": false,
|
|
"abstained": false,
|
|
"chars": 773,
|
|
"top_score": 0.031746031746031744
|
|
},
|
|
{
|
|
"q": "Wie ist die Telefonnummer meines Zahnarztes?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000EVX",
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000HRB",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000TRN"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 751,
|
|
"top_score": 0.03252247488101534
|
|
},
|
|
{
|
|
"q": "Wann fährt der letzte Zug nach Wien?",
|
|
"category": "abstain",
|
|
"lang": "de",
|
|
"ranked": [
|
|
"01M00000000000000000000FXC",
|
|
"01M00000000000000000000VSY",
|
|
"01M00000000000000000000QWN",
|
|
"01M00000000000000000000TRN",
|
|
"01M00000000000000000000BFR",
|
|
"01M00000000000000000000KAF",
|
|
"01M00000000000000000000STG",
|
|
"01M00000000000000000000PKT",
|
|
"01M00000000000000000000SWP",
|
|
"01M00000000000000000000HRB"
|
|
],
|
|
"recall@5": null,
|
|
"recall@10": null,
|
|
"mrr@10": null,
|
|
"ndcg@10": null,
|
|
"is_abstain_query": true,
|
|
"abstained": false,
|
|
"chars": 795,
|
|
"top_score": 0.03252247488101534
|
|
}
|
|
]
|
|
}
|
|
]
|
|
} |