feat(p0): repo skeleton — pyproject, config, JSON logging, CLI, Containerfile

Non-root, read-only-rootfs image with HOME/XDG_CACHE_HOME on /tmp so nothing
writes outside /data and /tmp (§13a.3, §14.16).

Containerfile

@@ -0,0 +1,38 @@
+# mdmem — build: podman build -t harbor.cloud.traunseenet.com/library/mdmem:0.1.0 -f Containerfile .
+FROM docker.io/library/python:3.12-slim-bookworm AS build
+
+COPY --from=ghcr.io/astral-sh/uv:0.5.18 /uv /usr/local/bin/uv
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PYTHON_DOWNLOADS=never
+WORKDIR /app
+
+# Dependencies first, so a source-only change does not re-resolve the world.
+COPY pyproject.toml uv.lock README.md ./
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-install-project --no-dev
+
+COPY src ./src
+RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev
+
+# ---------------------------------------------------------------------------
+FROM docker.io/library/python:3.12-slim-bookworm
+
+# readOnlyRootFilesystem: true (§13a.3) — nothing may write outside /data and /tmp.
+ENV HOME=/tmp \
+    XDG_CACHE_HOME=/tmp \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PATH="/app/.venv/bin:$PATH"
+
+WORKDIR /app
+COPY --from=build /app/.venv /app/.venv
+COPY --from=build /app/src /app/src
+
+# The sqlite-vec shared object ships inside the image and is loaded read-only (§14.16).
+RUN /app/.venv/bin/python -c "import sqlite_vec, os; p = sqlite_vec.loadable_path(); \
+    assert os.path.exists(p + '.so') or os.path.exists(p), p; print('sqlite-vec at', p)"
+
+USER 1000:1000
+EXPOSE 8080 8081
+VOLUME ["/data"]
+ENTRYPOINT ["/app/.venv/bin/mdmem"]
+CMD ["serve"]

README.md

@@ -0,0 +1,3 @@
+# mdmem
+
+Markdown-canonical agent memory service. See `PLAN.md` for the full design.

pyproject.toml

@@ -0,0 +1,57 @@
+[project]
+name = "mdmem"
+version = "0.1.0"
+description = "Markdown-canonical agent memory service: hybrid retrieval over a markdown vault, MCP + web UI"
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "MIT" }
+dependencies = [
+    "fastapi==0.115.6",
+    "uvicorn[standard]==0.34.0",
+    "mcp==1.2.0",
+    "sqlite-vec==0.1.6",
+    "ruamel.yaml==0.18.6",
+    "markdown-it-py==3.0.0",
+    "watchfiles==1.0.3",
+    "httpx==0.28.1",
+    "numpy==2.2.1",
+    "python-ulid==3.0.0",
+    "jinja2==3.1.5",
+    "authlib==1.4.0",
+    "itsdangerous==2.2.0",
+    "prometheus-client==0.21.1",
+    "pyyaml==6.0.2",
+    "python-multipart==0.0.20",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest==8.3.4",
+    "pytest-asyncio==0.25.2",
+    "hypothesis==6.123.7",
+    "ruff==0.9.2",
+]
+
+[project.scripts]
+mdmem = "mdmem.cli:main"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/mdmem"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
+markers = ["live: hits real Bifrost/llama-server; excluded from the default run"]
+addopts = "-m 'not live' -q"
+
+[tool.ruff]
+line-length = 100
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]

src/mdmem/__init__.py

@@ -0,0 +1,3 @@
+"""mdmem — markdown-canonical agent memory service."""
+
+__version__ = "0.1.0"

src/mdmem/__main__.py

@@ -0,0 +1,4 @@
+from mdmem.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/mdmem/cli.py

@@ -0,0 +1,68 @@
+"""Command line entry point."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+from mdmem import __version__
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="mdmem",
+        description="Markdown-canonical agent memory service (MCP + web UI).",
+    )
+    p.add_argument("--version", action="version", version=f"mdmem {__version__}")
+    sub = p.add_subparsers(dest="command", required=False, metavar="{serve,reindex,eval,doctor}")
+
+    serve = sub.add_parser("serve", help="run the MCP listener and the web UI")
+    serve.add_argument("--mcp-only", action="store_true", help="only start the MCP listener")
+    serve.add_argument("--ui-only", action="store_true", help="only start the web UI")
+    serve.add_argument("--no-watch", action="store_true", help="do not watch the vault for changes")
+
+    reindex = sub.add_parser("reindex", help="rebuild the index from the vault")
+    reindex.add_argument("--full", action="store_true", help="drop the index first")
+    reindex.add_argument("--no-embed", action="store_true", help="lexical index only")
+
+    ev = sub.add_parser("eval", help="run the retrieval eval harness (§11)")
+    ev.add_argument("--queries", default="eval/queries.yaml")
+    ev.add_argument("--config", default="default", help="name for this run's config")
+    ev.add_argument("--out", default=None, help="write the JSON report here")
+    ev.add_argument("--ablate", action="store_true", help="run the ablation matrix")
+
+    sub.add_parser("doctor", help="check config, index and embedding backend")
+    return p
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.command is None:
+        build_parser().print_help()
+        return 0
+
+    from mdmem import logging as mdlog
+    from mdmem.config import Settings
+
+    settings = Settings.from_env()
+    mdlog.setup(settings.log_level, settings.log_style)
+
+    if args.command == "serve":
+        from mdmem.server import run_serve
+
+        return run_serve(settings, args)
+    if args.command == "reindex":
+        from mdmem.server import run_reindex
+
+        return run_reindex(settings, args)
+    if args.command == "eval":
+        from mdmem.evalharness import run_eval_cli
+
+        return run_eval_cli(settings, args)
+    if args.command == "doctor":
+        from mdmem.server import run_doctor
+
+        return run_doctor(settings)
+    print(f"unknown command {args.command}", file=sys.stderr)
+    return 2

src/mdmem/config.py

@@ -0,0 +1,153 @@
+"""Configuration. Every value comes from the environment (§12)."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+FUSIONS = ("rrf", "cc")
+NOTE_TYPES = ("fact", "preference", "decision", "procedure", "entity", "event", "note")
+STATES = ("inbox", "active", "archive")
+
+
+def _bool(name: str, default: bool) -> bool:
+    raw = os.environ.get(name)
+    if raw is None or raw == "":
+        return default
+    return raw.strip().lower() in ("1", "true", "yes", "on")
+
+
+def _int(name: str, default: int) -> int:
+    raw = os.environ.get(name, "").strip()
+    return int(raw) if raw else default
+
+
+def _float(name: str, default: float) -> float:
+    raw = os.environ.get(name, "").strip()
+    return float(raw) if raw else default
+
+
+def _opt_float(name: str) -> float | None:
+    raw = os.environ.get(name, "").strip()
+    return float(raw) if raw else None
+
+
+def _addr(name: str, default: str) -> tuple[str, int]:
+    raw = os.environ.get(name, "").strip() or default
+    host, _, port = raw.rpartition(":")
+    if not host or not port.isdigit():
+        raise ValueError(f"{name} must be host:port, got {raw!r}")
+    return host, int(port)
+
+
+@dataclass(frozen=True, slots=True)
+class Settings:
+    # storage
+    vault_dir: Path
+    index_db: Path
+    cache_db: Path
+    history_dir: Path
+    # embeddings
+    bifrost_url: str
+    bifrost_vk: str
+    embed_model: str
+    embed_dim: int
+    # retrieval
+    fusion: str
+    cc_alpha: float
+    fts_trigram: bool
+    inbox_weight: float
+    bm25_weights: tuple[float, float, float, float]
+    include_inbox_default: bool
+    min_score: float | None
+    max_chars_default: int
+    dup_threshold: float
+    max_note_chars: int
+    # listeners
+    mcp_host: str
+    mcp_port: int
+    ui_host: str
+    ui_port: int
+    mcp_token: str
+    # ui / auth
+    ui_base_url: str
+    ui_allowed_origins: tuple[str, ...]
+    oidc_issuer: str
+    oidc_scope: str
+    oidc_client_id: str
+    oidc_client_secret: str
+    oidc_groups_claim: str
+    oidc_required_group: str
+    session_secret: str
+    session_ttl_hours: int
+    ui_auth_disabled: bool
+    # misc
+    log_level: str
+    log_style: str
+    tz: str
+
+    @classmethod
+    def from_env(cls) -> Settings:
+        vault = Path(os.environ.get("VAULT_DIR", "/data/vault")).resolve()
+        data = vault.parent
+        fusion = (os.environ.get("FUSION", "").strip() or "rrf").lower()
+        if fusion not in FUSIONS:
+            raise ValueError(f"FUSION must be one of {FUSIONS}, got {fusion!r}")
+        raw_w = os.environ.get("BM25_WEIGHTS", "").strip() or "3,2,1.5,1"
+        parts = [p for p in raw_w.replace(" ", "").split(",") if p]
+        if len(parts) != 4:
+            raise ValueError(f"BM25_WEIGHTS needs 4 comma-separated numbers, got {raw_w!r}")
+        try:
+            weights = tuple(float(p) for p in parts)
+        except ValueError as exc:
+            raise ValueError(f"BM25_WEIGHTS not numeric: {raw_w!r}") from exc
+
+        mcp_host, mcp_port = _addr("MCP_ADDR", "0.0.0.0:8080")
+        ui_host, ui_port = _addr("UI_ADDR", "0.0.0.0:8081")
+        origins = tuple(
+            o.strip()
+            for o in os.environ.get("UI_ALLOWED_ORIGINS", "").split(",")
+            if o.strip()
+        )
+        return cls(
+            vault_dir=vault,
+            index_db=Path(os.environ.get("INDEX_DB") or data / "index.db"),
+            cache_db=Path(os.environ.get("CACHE_DB") or data / "emb_cache.db"),
+            history_dir=Path(os.environ.get("HISTORY_DIR") or data / ".history"),
+            bifrost_url=(
+                os.environ.get("BIFROST_URL", "http://bifrost.apps.svc.cluster.local:8080")
+            ).rstrip("/"),
+            bifrost_vk=os.environ.get("BIFROST_VK", ""),
+            embed_model=os.environ.get("EMBED_MODEL", "llamaswap/qwen3-embedding-4b"),
+            embed_dim=_int("EMBED_DIM", 2560),
+            fusion=fusion,
+            cc_alpha=_float("CC_ALPHA", 0.5),
+            fts_trigram=_bool("FTS_TRIGRAM", True),
+            inbox_weight=_float("INBOX_WEIGHT", 0.8),
+            bm25_weights=weights,  # type: ignore[arg-type]
+            include_inbox_default=_bool("INCLUDE_INBOX_DEFAULT", True),
+            min_score=_opt_float("MIN_SCORE"),
+            max_chars_default=_int("MAX_CHARS_DEFAULT", 6000),
+            dup_threshold=_float("DUP_THRESHOLD", 0.90),
+            max_note_chars=_int("MAX_NOTE_CHARS", 8000),
+            mcp_host=mcp_host,
+            mcp_port=mcp_port,
+            ui_host=ui_host,
+            ui_port=ui_port,
+            mcp_token=os.environ.get("MCP_TOKEN", ""),
+            ui_base_url=os.environ.get("UI_BASE_URL", "").rstrip("/"),
+            ui_allowed_origins=origins,
+            oidc_issuer=os.environ.get("OIDC_ISSUER", "").rstrip("/"),
+            oidc_scope=os.environ.get("OIDC_SCOPE", "openid email profile groups"),
+            oidc_client_id=os.environ.get("OIDC_CLIENT_ID", ""),
+            oidc_client_secret=os.environ.get("OIDC_CLIENT_SECRET", ""),
+            oidc_groups_claim=os.environ.get("OIDC_GROUPS_CLAIM", "groups"),
+            oidc_required_group=os.environ.get("OIDC_REQUIRED_GROUP", "mdmem_admins"),
+            session_secret=os.environ.get("SESSION_SECRET", ""),
+            session_ttl_hours=_int("SESSION_TTL_HOURS", 12),
+            ui_auth_disabled=_bool("UI_AUTH_DISABLED", False),
+            log_level=os.environ.get("LOG_LEVEL", "info"),
+            log_style=os.environ.get("LOG_STYLE", "json"),
+            tz=os.environ.get("TZ", "Europe/Vienna"),
+        )

src/mdmem/logging.py

@@ -0,0 +1,94 @@
+"""Structured logging. JSON lines by default (§12), plus the MCP audit log (§9)."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+_AUDIT_NAME = "mdmem.audit"
+
+
+class JsonFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created))
+            + f".{int(record.msecs):03d}Z",
+            "level": record.levelname.lower(),
+            "logger": record.name,
+            "msg": record.getMessage(),
+        }
+        extra = getattr(record, "fields", None)
+        if isinstance(extra, dict):
+            payload.update(extra)
+        if record.exc_info:
+            payload["exc"] = self.formatException(record.exc_info)
+        return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
+
+
+def setup(level: str = "info", style: str = "json") -> None:
+    root = logging.getLogger()
+    root.handlers.clear()
+    handler = logging.StreamHandler(sys.stderr)
+    if style == "json":
+        handler.setFormatter(JsonFormatter())
+    else:
+        handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s %(name)s %(message)s"))
+    root.addHandler(handler)
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))
+    logging.getLogger("watchfiles").setLevel(logging.WARNING)
+
+
+def log(logger: logging.Logger, level: int, msg: str, **fields: Any) -> None:
+    logger.log(level, msg, extra={"fields": fields})
+
+
+class AuditLog:
+    """Append-only JSON-lines log of tool calls. Never records note bodies (§9)."""
+
+    def __init__(self, path: Path | None = None, keep: int = 1000) -> None:
+        self.path = path
+        self.keep = keep
+        self._recent: list[dict[str, Any]] = []
+        self._log = logging.getLogger(_AUDIT_NAME)
+
+    def record(self, **fields: Any) -> None:
+        entry = {
+            "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            **fields,
+        }
+        self._recent.append(entry)
+        if len(self._recent) > self.keep:
+            del self._recent[: len(self._recent) - self.keep]
+        line = json.dumps(entry, ensure_ascii=False, separators=(",", ":"))
+        if self.path is not None:
+            try:
+                self.path.parent.mkdir(parents=True, exist_ok=True)
+                with self.path.open("a", encoding="utf-8") as fh:
+                    fh.write(line + "\n")
+            except OSError:
+                self._log.warning("audit log write failed", extra={"fields": {"path": str(self.path)}})
+        self._log.info("tool_call", extra={"fields": fields})
+
+    def recent(self, limit: int = 100) -> list[dict[str, Any]]:
+        out = self._recent[-limit:]
+        if not out and self.path is not None and self.path.exists():
+            try:
+                lines = self.path.read_text(encoding="utf-8").splitlines()[-limit:]
+            except OSError:
+                return []
+            for line in lines:
+                try:
+                    out.append(json.loads(line))
+                except ValueError:
+                    continue
+        return list(reversed(out))
+
+
+def audit_path_from_env() -> Path | None:
+    raw = os.environ.get("AUDIT_LOG", "").strip()
+    return Path(raw) if raw else None

tests/test_cli.py

@@ -0,0 +1,21 @@
+import subprocess
+import sys
+
+
+def _run(*args):
+    return subprocess.run(
+        [sys.executable, "-m", "mdmem", *args], capture_output=True, text=True
+    )
+
+
+def test_help_exits_zero():
+    r = _run("--help")
+    assert r.returncode == 0
+    for cmd in ("serve", "reindex", "eval", "doctor"):
+        assert cmd in r.stdout
+
+
+def test_version():
+    r = _run("--version")
+    assert r.returncode == 0
+    assert "0.1.0" in r.stdout

tests/test_config.py

@@ -0,0 +1,61 @@
+import pytest
+
+from mdmem.config import Settings
+
+
+def test_defaults_match_plan(tmp_path, monkeypatch):
+    monkeypatch.setenv("VAULT_DIR", str(tmp_path / "vault"))
+    s = Settings.from_env()
+    assert s.embed_dim == 2560
+    assert s.fusion == "rrf"
+    assert s.inbox_weight == pytest.approx(0.8)
+    assert s.bm25_weights == (3.0, 2.0, 1.5, 1.0)
+    assert s.dup_threshold == pytest.approx(0.90)
+    assert s.max_note_chars == 8000
+    assert s.max_chars_default == 6000
+    assert s.min_score is None
+    assert s.include_inbox_default is True
+    assert s.fts_trigram is True
+
+
+def test_paths_derive_from_vault_parent(tmp_path, monkeypatch):
+    monkeypatch.setenv("VAULT_DIR", str(tmp_path / "vault"))
+    monkeypatch.setenv("INDEX_DB", str(tmp_path / "index.db"))
+    s = Settings.from_env()
+    assert s.index_db == tmp_path / "index.db"
+    assert s.vault_dir == tmp_path / "vault"
+
+
+def test_bm25_weights_parsed(monkeypatch, tmp_path):
+    monkeypatch.setenv("VAULT_DIR", str(tmp_path))
+    monkeypatch.setenv("BM25_WEIGHTS", "1,2,3,4")
+    assert Settings.from_env().bm25_weights == (1.0, 2.0, 3.0, 4.0)
+
+
+def test_bad_bm25_weights_rejected(monkeypatch, tmp_path):
+    monkeypatch.setenv("VAULT_DIR", str(tmp_path))
+    monkeypatch.setenv("BM25_WEIGHTS", "1,2,3")
+    with pytest.raises(ValueError, match="BM25_WEIGHTS"):
+        Settings.from_env()
+
+
+def test_bad_fusion_rejected(monkeypatch, tmp_path):
+    monkeypatch.setenv("VAULT_DIR", str(tmp_path))
+    monkeypatch.setenv("FUSION", "magic")
+    with pytest.raises(ValueError, match="FUSION"):
+        Settings.from_env()
+
+
+def test_addr_parsing(monkeypatch, tmp_path):
+    monkeypatch.setenv("VAULT_DIR", str(tmp_path))
+    monkeypatch.setenv("MCP_ADDR", "0.0.0.0:9999")
+    s = Settings.from_env()
+    assert s.mcp_host == "0.0.0.0" and s.mcp_port == 9999
+
+
+def test_empty_min_score_is_none(monkeypatch, tmp_path):
+    monkeypatch.setenv("VAULT_DIR", str(tmp_path))
+    monkeypatch.setenv("MIN_SCORE", "")
+    assert Settings.from_env().min_score is None
+    monkeypatch.setenv("MIN_SCORE", "0.2")
+    assert Settings.from_env().min_score == pytest.approx(0.2)