diff --git a/haiku_rag_slim/haiku/rag/ingester/sources/fs.py b/haiku_rag_slim/haiku/rag/ingester/sources/fs.py index 7159b950..42e6ca94 100644 --- a/haiku_rag_slim/haiku/rag/ingester/sources/fs.py +++ b/haiku_rag_slim/haiku/rag/ingester/sources/fs.py @@ -109,16 +109,23 @@ class FSSource: seen: set[str] = set() # os.walk with followlinks=False so symlinked directories aren't - # traversed. Then per-file: skip individual file-symlinks too, since - # they could point outside root and reading them would leak data. - # Operators wanting to ingest content from outside root should - # bind-mount it in or configure a second source. + # traversed (avoids cycles and unbounded recursion). File symlinks + # are followed iff their target resolves inside root, matching + # supports/head/fetch's resolve-then-check behaviour. Out-of-root + # targets stay skipped so a stray link can't exfiltrate data the + # operator didn't intend to expose. candidates: list[Path] = [] for dirpath, _dirnames, filenames in os.walk(self.root, followlinks=False): for filename in filenames: path = Path(dirpath) / filename if path.is_symlink(): - continue + try: + resolved = path.resolve(strict=False) + except OSError: + continue + if not resolved.is_relative_to(self.root): + continue + path = resolved candidates.append(path) candidates.sort() @@ -128,6 +135,10 @@ class FSSource: if not self.filter.include_file(str(path)): continue uri = path.as_uri() + if uri in seen: + # A symlink and its target (or two symlinks to the same file) + # both walked through; the first wins so the queue sees one job. + continue revision = str(path.stat().st_mtime_ns) seen.add(uri) previous = snapshot.get(uri) diff --git a/tests/ingester/test_fs_source.py b/tests/ingester/test_fs_source.py index 816b2fbb..baa57977 100644 --- a/tests/ingester/test_fs_source.py +++ b/tests/ingester/test_fs_source.py @@ -195,11 +195,13 @@ async def test_fs_source_fetch_rejects_symlink_to_outside_file( @pytest.mark.asyncio -async def test_fs_source_discover_skips_symlinks(fs_root: Path, tmp_path: Path): - """rglob (and os.walk by default) follows symlinks. We use - followlinks=False AND an explicit per-file is_symlink() filter so a - malicious symlink under root can't be discovered, can't be queued, and - can't be fetched even if its URI ends up enqueued some other way.""" +async def test_fs_source_discover_skips_symlinks_pointing_outside_root( + fs_root: Path, tmp_path: Path +): + """A symlink under root whose target resolves outside root must not be + discovered — otherwise a stray link could exfiltrate files the operator + didn't intend to expose. Mirrors the resolve-then-check guard that + supports/head/fetch use.""" secret = tmp_path.parent / "secret.md" secret.write_text("sensitive") link = fs_root / "evil.md" @@ -207,10 +209,31 @@ async def test_fs_source_discover_skips_symlinks(fs_root: Path, tmp_path: Path): src = FSSource(root=fs_root, supported_extensions=[".md"]) uris = {e.uri async for e in src.discover(since=None)} assert link.as_uri() not in uris + assert secret.as_uri() not in uris # And the legitimate files in fs_root still come through. assert (fs_root / "a.md").as_uri() in uris +@pytest.mark.asyncio +async def test_fs_source_discover_follows_within_root_symlinks(fs_root: Path): + """A symlink whose target lives inside root is legitimate — supports/ + head/fetch all accept it (resolve-then-check), so discover() must too, + otherwise an ad-hoc add-src on a link works but the poller never picks + it up. The emitted URI is the resolved target's, and the seen-set + dedupes when both the link and its target are walked.""" + target = fs_root / "real.md" + target.write_text("real content") + (fs_root / "alias.md").symlink_to(target) + src = FSSource(root=fs_root, supported_extensions=[".md"]) + events = [e async for e in src.discover(since=None)] + uris = [e.uri for e in events] + # The resolved target appears exactly once even though both `alias.md` + # and `real.md` are encountered during the walk. + assert uris.count(target.as_uri()) == 1 + # The alias's own URI is not emitted — we normalise to the target. + assert (fs_root / "alias.md").as_uri() not in uris + + @pytest.mark.asyncio async def test_fs_source_discover_skips_symlinked_directories( fs_root: Path, tmp_path: Path