fix(chunker): drop empty H2/H3 sections instead of chunking them

_split_long("") returned [""], so an H2/H3 heading immediately followed by
another heading (or by the end of the body) produced a real chunk with an
empty body: a wasted FTS row, trigram row and embedding call. ord stays
contiguous since the loop only increments when a chunk is actually appended.

Test: tests/test_chunker.py::test_empty_h2_sections_do_not_produce_empty_chunks
This commit is contained in:
Markus committed 2026-09-12 18:30:41 +00:00
1 parent 0d53c854d8
commit 8c4e90d27b
2 files changed
+20 -1

No files matched your search

+5 -1
View File
@@ -49,7 +49,11 @@ def _sections(body: str) -> list[tuple[str, str]]:
def _split_long(text: str, limit: int = MAX_CHUNK_CHARS) -> list[str]:
if len(text) <= limit:
return [text]
# An empty/whitespace-only section (an H2/H3 immediately followed by
# another heading, or by the end of the body) must not become a real
# chunk: an empty body still gets an FTS row, a trigram row and a
# wasted embedding call.
return [text] if text.strip() else []
parts: list[str] = []
buf: list[str] = []
size = 0
+15
View File
@@ -49,6 +49,21 @@ def test_embed_text_merges_keys_at_index_time():
assert t.endswith("Körper")
def test_empty_h2_sections_do_not_produce_empty_chunks():
"""An empty H2/H3 section (heading immediately followed by another heading,
or by the end of the body) used to become a real chunk with an empty body
-- a wasted FTS row, trigram row and embedding call for nothing."""
body = "## A\n\n## B\n\n" + "wort " * 1400 + "\n\n## C\n\n"
n = Note.new(title="T", body=body, type="note")
chunks = chunk_note(n)
assert all(c.body.strip() for c in chunks), [c.body for c in chunks]
assert [c.ord for c in chunks] == list(range(len(chunks)))
headings = {c.heading for c in chunks}
assert "A" not in headings
assert "C" not in headings
assert "B" in headings
def test_empty_body_yields_one_chunk_with_title_text():
n = Note.new(title="Nur Titel", body="", type="fact")
chunks = chunk_note(n)