fix(index): dedupe ScanStats.pending by chunk_id

scan() extended stats.pending from each index_note() return and then again
from pending_embeds() (which re-reads every vec_pending=1 chunk in the db,
including the ones just indexed), so every freshly pending chunk was listed
twice and would be embedded twice.

Test: tests/test_index.py::test_scan_pending_has_no_duplicate_chunk_ids
This commit is contained in:
Markus committed 2026-09-12 18:30:20 +00:00
1 parent 82a16a129f
commit 0d53c854d8
2 files changed
+23 -1

No files matched your search

+8 -1
View File
@@ -245,7 +245,14 @@ class Indexer:
for note_id in set(known) - seen:
self.remove(note_id)
stats.removed += 1
stats.pending.extend(self.pending_embeds())
# pending_embeds() returns every vec_pending=1 chunk in the whole db,
# which includes the ones index_note() just appended above -- dedupe
# by chunk_id so each pending chunk is listed once, not twice.
seen_chunk_ids = {p.chunk_id for p in stats.pending}
for p in self.pending_embeds():
if p.chunk_id not in seen_chunk_ids:
seen_chunk_ids.add(p.chunk_id)
stats.pending.append(p)
stats.seconds = time.monotonic() - started
return stats
+15
View File
@@ -209,6 +209,21 @@ def test_broken_yaml_is_skipped_not_fatal(idx):
assert db.read().execute("SELECT count(*) FROM notes").fetchone()[0] == 1
def test_scan_pending_has_no_duplicate_chunk_ids(idx):
"""scan() extends ScanStats.pending from each index_note() return AND again
from pending_embeds(), so every pending chunk used to be listed twice."""
indexer, vault, db = idx
a = Note.new(title="A", body="body a", type="fact")
b = Note.new(title="B", body="body b", type="fact")
vault.write(a)
vault.write(b)
stats = indexer.scan()
assert stats.indexed == 2
chunk_ids = [p.chunk_id for p in stats.pending]
assert len(chunk_ids) == len(set(chunk_ids)), f"duplicate pending chunk ids: {chunk_ids}"
assert len(stats.pending) == 2
def test_unchanged_embed_text_keeps_its_vector(idx):
indexer, vault, db = idx
n = Note.new(title="T", body="body", type="fact")