fix(chunker): drop empty H2/H3 sections instead of chunking them
_split_long("") returned [""], so an H2/H3 heading immediately followed by
another heading (or by the end of the body) produced a real chunk with an
empty body: a wasted FTS row, trigram row and embedding call. ord stays
contiguous since the loop only increments when a chunk is actually appended.
Test: tests/test_chunker.py::test_empty_h2_sections_do_not_produce_empty_chunks
This commit is contained in:
1 parent
0d53c854d8
commit
8c4e90d27b
2 files changed
+20
-1
No files matched your search
@@ -49,7 +49,11 @@ def _sections(body: str) -> list[tuple[str, str]]:
|
||||
|
||||
def _split_long(text: str, limit: int = MAX_CHUNK_CHARS) -> list[str]:
|
||||
if len(text) <= limit:
|
||||
return [text]
|
||||
# An empty/whitespace-only section (an H2/H3 immediately followed by
|
||||
# another heading, or by the end of the body) must not become a real
|
||||
# chunk: an empty body still gets an FTS row, a trigram row and a
|
||||
# wasted embedding call.
|
||||
return [text] if text.strip() else []
|
||||
parts: list[str] = []
|
||||
buf: list[str] = []
|
||||
size = 0
|
||||
|
||||
@@ -49,6 +49,21 @@ def test_embed_text_merges_keys_at_index_time():
|
||||
assert t.endswith("Körper")
|
||||
|
||||
|
||||
def test_empty_h2_sections_do_not_produce_empty_chunks():
|
||||
"""An empty H2/H3 section (heading immediately followed by another heading,
|
||||
or by the end of the body) used to become a real chunk with an empty body
|
||||
-- a wasted FTS row, trigram row and embedding call for nothing."""
|
||||
body = "## A\n\n## B\n\n" + "wort " * 1400 + "\n\n## C\n\n"
|
||||
n = Note.new(title="T", body=body, type="note")
|
||||
chunks = chunk_note(n)
|
||||
assert all(c.body.strip() for c in chunks), [c.body for c in chunks]
|
||||
assert [c.ord for c in chunks] == list(range(len(chunks)))
|
||||
headings = {c.heading for c in chunks}
|
||||
assert "A" not in headings
|
||||
assert "C" not in headings
|
||||
assert "B" in headings
|
||||
|
||||
|
||||
def test_empty_body_yields_one_chunk_with_title_text():
|
||||
n = Note.new(title="Nur Titel", body="", type="fact")
|
||||
chunks = chunk_note(n)
|
||||
|
||||
Reference in new issue
Block a user