fix(index): dedupe ScanStats.pending by chunk_id
scan() extended stats.pending from each index_note() return and then again from pending_embeds() (which re-reads every vec_pending=1 chunk in the db, including the ones just indexed), so every freshly pending chunk was listed twice and would be embedded twice. Test: tests/test_index.py::test_scan_pending_has_no_duplicate_chunk_ids
This commit is contained in:
1 parent
82a16a129f
commit
0d53c854d8
2 files changed
+23
-1
No files matched your search
+8
-1
@@ -245,7 +245,14 @@ class Indexer:
|
||||
for note_id in set(known) - seen:
|
||||
self.remove(note_id)
|
||||
stats.removed += 1
|
||||
stats.pending.extend(self.pending_embeds())
|
||||
# pending_embeds() returns every vec_pending=1 chunk in the whole db,
|
||||
# which includes the ones index_note() just appended above -- dedupe
|
||||
# by chunk_id so each pending chunk is listed once, not twice.
|
||||
seen_chunk_ids = {p.chunk_id for p in stats.pending}
|
||||
for p in self.pending_embeds():
|
||||
if p.chunk_id not in seen_chunk_ids:
|
||||
seen_chunk_ids.add(p.chunk_id)
|
||||
stats.pending.append(p)
|
||||
stats.seconds = time.monotonic() - started
|
||||
return stats
|
||||
|
||||
|
||||
@@ -209,6 +209,21 @@ def test_broken_yaml_is_skipped_not_fatal(idx):
|
||||
assert db.read().execute("SELECT count(*) FROM notes").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_scan_pending_has_no_duplicate_chunk_ids(idx):
|
||||
"""scan() extends ScanStats.pending from each index_note() return AND again
|
||||
from pending_embeds(), so every pending chunk used to be listed twice."""
|
||||
indexer, vault, db = idx
|
||||
a = Note.new(title="A", body="body a", type="fact")
|
||||
b = Note.new(title="B", body="body b", type="fact")
|
||||
vault.write(a)
|
||||
vault.write(b)
|
||||
stats = indexer.scan()
|
||||
assert stats.indexed == 2
|
||||
chunk_ids = [p.chunk_id for p in stats.pending]
|
||||
assert len(chunk_ids) == len(set(chunk_ids)), f"duplicate pending chunk ids: {chunk_ids}"
|
||||
assert len(stats.pending) == 2
|
||||
|
||||
|
||||
def test_unchanged_embed_text_keeps_its_vector(idx):
|
||||
indexer, vault, db = idx
|
||||
n = Note.new(title="T", body="body", type="fact")
|
||||
|
||||
Reference in new issue
Block a user