From 7d80eb4f83c3e4fbd206835363b970e9fe0b776a Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 1 Jun 2026 07:29:51 -0400 Subject: [PATCH 1/2] Fix discover() crash when file is deleted during stat() A file deleted between os.walk() and path.stat() raises FileNotFoundError, which propagated uncaught and failed the entire discover() sweep. With enough failures this trips the circuit breaker, silencing the poller. Catch FileNotFoundError around the stat() call and skip the file. The next sweep (or watchfiles) will emit the DELETE event. --- .../haiku/rag/ingester/sources/fs.py | 7 ++++++- tests/ingester/test_fs_source.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/haiku_rag_slim/haiku/rag/ingester/sources/fs.py b/haiku_rag_slim/haiku/rag/ingester/sources/fs.py index 8b944e04..8de87469 100644 --- a/haiku_rag_slim/haiku/rag/ingester/sources/fs.py +++ b/haiku_rag_slim/haiku/rag/ingester/sources/fs.py @@ -139,7 +139,12 @@ class FSSource: if not self.filter.include_file(str(path)): continue uri = path.as_uri() - revision = str(path.stat().st_mtime_ns) + try: + revision = str(path.stat().st_mtime_ns) + except FileNotFoundError: + # File was deleted between the os.walk() and stat() call. + # Skip it — the next sweep (or watchfiles) will emit DELETE. + continue seen.add(uri) previous = snapshot.get(uri) kind = ( diff --git a/tests/ingester/test_fs_source.py b/tests/ingester/test_fs_source.py index df32304f..c827bf0e 100644 --- a/tests/ingester/test_fs_source.py +++ b/tests/ingester/test_fs_source.py @@ -131,6 +131,25 @@ async def test_fs_source_discover_respects_extension_filter(fs_root: Path): assert (fs_root / "b.txt").as_uri() not in uris +@pytest.mark.asyncio +async def test_fs_source_discover_skips_file_deleted_during_stat(fs_root: Path): + """A file deleted between os.walk() and stat() should be silently + skipped instead of crashing the entire discover() sweep.""" + src = FSSource(root=fs_root, supported_extensions=[".md", ".txt"]) + events = [] + async for event in src.discover(since=None): + events.append(event) + # Delete a file mid-iteration so the next stat() hits a missing file. + victim = fs_root / "b.txt" + if victim.exists(): + victim.unlink() + uris = {e.uri for e in events} + # a.md and sub/c.md should still appear; b.txt may or may not depending + # on iteration order, but the key assertion is no exception was raised. + assert (fs_root / "a.md").as_uri() in uris + assert (fs_root / "sub" / "c.md").as_uri() in uris + + @pytest.mark.asyncio async def test_fs_source_discover_respects_ignore_patterns(fs_root: Path): src = FSSource( From 2a06421e7a946befbc0526b2228bb26dc86c4902 Mon Sep 17 00:00:00 2001 From: Chris McDonough Date: Mon, 1 Jun 2026 09:28:26 -0400 Subject: [PATCH 2/2] Improve discover stat race test to actually exercise the try/except MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous test deleted a file mid-iteration, but is_file() caught it before stat() ran — so the new try/except never executed. Monkeypatch Path.stat to raise FileNotFoundError on the third call for the victim path (after is_symlink and is_file pass), simulating the exact TOCTOU window between is_file() and stat(). --- tests/ingester/test_fs_source.py | 34 +++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/ingester/test_fs_source.py b/tests/ingester/test_fs_source.py index c827bf0e..f5e088cf 100644 --- a/tests/ingester/test_fs_source.py +++ b/tests/ingester/test_fs_source.py @@ -132,22 +132,34 @@ async def test_fs_source_discover_respects_extension_filter(fs_root: Path): @pytest.mark.asyncio -async def test_fs_source_discover_skips_file_deleted_during_stat(fs_root: Path): - """A file deleted between os.walk() and stat() should be silently +async def test_fs_source_discover_skips_file_deleted_during_stat( + fs_root: Path, monkeypatch +): + """A file deleted between is_file() and stat() should be silently skipped instead of crashing the entire discover() sweep.""" + victim = fs_root / "b.txt" + original_stat = Path.stat + victim_calls = 0 + + def _stat_that_fails_on_second_call(self, *args, **kwargs): + nonlocal victim_calls + if self == victim: + victim_calls += 1 + # First calls are from is_symlink/is_file; the later call + # is the explicit stat().st_mtime_ns we want to fail. + if victim_calls > 2: + raise FileNotFoundError(f"[Errno 2] No such file: '{self}'") + return original_stat(self, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", _stat_that_fails_on_second_call) + src = FSSource(root=fs_root, supported_extensions=[".md", ".txt"]) - events = [] - async for event in src.discover(since=None): - events.append(event) - # Delete a file mid-iteration so the next stat() hits a missing file. - victim = fs_root / "b.txt" - if victim.exists(): - victim.unlink() + events = [e async for e in src.discover(since=None)] uris = {e.uri for e in events} - # a.md and sub/c.md should still appear; b.txt may or may not depending - # on iteration order, but the key assertion is no exception was raised. + # b.txt was skipped due to the simulated race; a.md and sub/c.md are fine. assert (fs_root / "a.md").as_uri() in uris assert (fs_root / "sub" / "c.md").as_uri() in uris + assert victim.as_uri() not in uris @pytest.mark.asyncio