Skip to content

Navigation Menu

Sign in
Appearance settings
Sign in
Sign up
Appearance settings

Commit 85a43d0

Browse filesBrowse the repository at this point in the historyBrowse files
GoodbyePlanetclaude
GoodbyePlanet
and
claude
committed
fix: paginate GitHub trees on truncation to avoid silently dropping files
list_github_files fetched the full repo tree in a single recursive Trees API call. On very large repos GitHub sets truncated:true and returns a partial tree; semcode logged a warning and indexed the partial set, so an unknown subset of files was silently absent from the index. Now the truncated case falls back to a recursive per-subtree walk using non-recursive git/trees/<sha> calls (each listing stays well under the size limit), pruning subtrees outside root and fetching siblings concurrently. Shared filter logic is extracted so both paths behave identically. Closes #55 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 9baa063 commit 85a43d0
Copy full SHA for 85a43d0

3 files changed

+268-27Lines changed: 268 additions & 27 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎docs/ingestion.md‎

Copy file name to clipboardExpand all lines: docs/ingestion.md
+1-1Lines changed: 1 addition & 1 deletion
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ This returns the full file tree in one round-trip. Each entry is filtered by thr
3737

3838
Surviving files become `GitHubFile(rel_path, blob_sha)` objects. The `blob_sha` is the git blob SHA — a content fingerprint that drives incremental indexing in the next stage.
3939

40-
> **Large-repo warning:** The GitHub Trees API has an undocumented response size limit. When a repo's tree is truncated, a warning is logged but affected files are silently absent from the index run. There is no automatic retry or fallback.
40+
> **Large-repo fallback:** The GitHub Trees API has an undocumented response size limit. When a repo's tree exceeds it, GitHub sets `truncated: true` and returns only a partial tree. In that case `list_github_files` falls back to a recursive per-subtree walk — fetching `git/trees/<sha>` for each directory (non-recursive, so each listing stays well under the limit), pruning subtrees outside `root`, and fetching siblings concurrently. This guarantees no files are silently dropped from large repos.
4141
4242
### 2. Incremental Check
4343

Collapse file

‎server/indexer/github_source.py‎

Copy file name to clipboardExpand all lines: server/indexer/github_source.py
+114-26Lines changed: 114 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
_GITHUB_API = "https://api.github.com"
1818
_DIFF_CONCURRENCY = 10
19+
_TREE_WALK_CONCURRENCY = 10
1920

2021
logger = logging.getLogger(__name__)
2122

@@ -100,6 +101,96 @@ async def _gh_get(
100101
return r.json() # unreachable
101102

102103

104+
def _filter_tree_blobs(
105+
blobs: list[tuple[str, str]],
106+
root_prefix: str | None,
107+
exclude: list[str],
108+
) -> list[GitHubFile]:
109+
"""Apply the discovery filters (root prefix, supported extension, exclude) to blobs.
110+
111+
*blobs* is an iterable of ``(full_path, blob_sha)`` tuples. Returns the surviving
112+
files with paths made relative to *root_prefix* when set.
113+
"""
114+
files: list[GitHubFile] = []
115+
for path, sha in blobs:
116+
if root_prefix and not path.startswith(root_prefix):
117+
continue
118+
if not is_supported_path(path):
119+
continue
120+
if exclude and _matches_any(path, exclude):
121+
continue
122+
rel_path = path[len(root_prefix) :] if root_prefix else path
123+
files.append(GitHubFile(rel_path=rel_path, blob_sha=sha))
124+
return files
125+
126+
127+
def _subtree_is_relevant(dir_path: str, root_prefix: str | None) -> bool:
128+
"""Whether a directory could contain files under *root_prefix* and is worth fetching.
129+
130+
A directory is relevant when it lives under the root prefix, or is an ancestor of it.
131+
"""
132+
if not root_prefix:
133+
return True
134+
dir_prefix = dir_path + "/"
135+
return dir_prefix.startswith(root_prefix) or root_prefix.startswith(dir_prefix)
136+
137+
138+
async def _walk_tree_recursive(
139+
client: httpx.AsyncClient,
140+
repo: str,
141+
token: str,
142+
tree_sha: str,
143+
prefix: str,
144+
root_prefix: str | None,
145+
sem: asyncio.Semaphore,
146+
) -> list[tuple[str, str]]:
147+
"""Recursively list all blobs under *tree_sha* via non-recursive trees calls.
148+
149+
Each single-directory listing is small and effectively never hits the Trees API size
150+
limit. Subtrees that cannot contain files under *root_prefix* are pruned. Sibling
151+
subtree fetches run concurrently, bounded by *sem*. Returns ``(full_path, blob_sha)``
152+
tuples.
153+
"""
154+
async with sem:
155+
tree = await _gh_get(
156+
client,
157+
f"{_GITHUB_API}/repos/{repo}/git/trees/{tree_sha}",
158+
token,
159+
timeout=30,
160+
)
161+
162+
if tree.get("truncated"):
163+
logger.warning(
164+
"GitHub trees response still truncated for subtree %s under %s — directory has "
165+
"too many entries; some files may be missing from the index.",
166+
tree_sha,
167+
prefix or "<root>",
168+
)
169+
170+
blobs: list[tuple[str, str]] = []
171+
subtree_tasks = []
172+
for item in tree.get("tree", []):
173+
full_path = f"{prefix}{item['path']}"
174+
if item["type"] == "blob":
175+
blobs.append((full_path, item["sha"]))
176+
elif item["type"] == "tree" and _subtree_is_relevant(full_path, root_prefix):
177+
subtree_tasks.append(
178+
_walk_tree_recursive(
179+
client,
180+
repo,
181+
token,
182+
item["sha"],
183+
f"{full_path}/",
184+
root_prefix,
185+
sem,
186+
)
187+
)
188+
189+
for sub_blobs in await asyncio.gather(*subtree_tasks):
190+
blobs.extend(sub_blobs)
191+
return blobs
192+
193+
103194
async def list_github_files(
104195
token: str,
105196
repo: str,
@@ -109,12 +200,18 @@ async def list_github_files(
109200
root: str | None = None,
110201
client: httpx.AsyncClient | None = None,
111202
) -> list[GitHubFile]:
112-
"""List matching files via the git trees API (single request for the full tree).
203+
"""List matching files via the git trees API.
204+
205+
Uses a single recursive request for the full tree. If GitHub truncates that response
206+
(undocumented size limit on very large repos), falls back to a recursive per-subtree
207+
walk so no files are silently dropped.
113208
114209
All files whose extension or basename is recognised by the parser registry are
115210
indexed. If *root* is set, only files under that path prefix are considered.
116211
Paths matching *exclude* patterns are skipped.
117212
"""
213+
root_prefix = root.rstrip("/") + "/" if root else None
214+
118215
async with _client_ctx(client) as c:
119216
tree = await _gh_get(
120217
c,
@@ -124,35 +221,26 @@ async def list_github_files(
124221
timeout=30,
125222
)
126223

127-
if tree.get("truncated"):
128-
logger.warning(
129-
"GitHub trees response truncated for %s@%s — repo is very large; "
130-
"some files may be missing from the index.",
224+
if not tree.get("truncated"):
225+
blobs = [
226+
(item["path"], item["sha"])
227+
for item in tree.get("tree", [])
228+
if item["type"] == "blob"
229+
]
230+
return _filter_tree_blobs(blobs, root_prefix, exclude)
231+
232+
logger.info(
233+
"GitHub trees response truncated for %s@%s — falling back to recursive "
234+
"per-subtree walk.",
131235
repo,
132236
ref,
133237
)
134-
135-
root_prefix = root.rstrip("/") + "/" if root else None
136-
137-
files: list[GitHubFile] = []
138-
for item in tree.get("tree", []):
139-
if item["type"] != "blob":
140-
continue
141-
path = item["path"]
142-
if root_prefix and not path.startswith(root_prefix):
143-
continue
144-
if not is_supported_path(path):
145-
continue
146-
if exclude and _matches_any(path, exclude):
147-
continue
148-
rel_path = path[len(root_prefix) :] if root_prefix else path
149-
files.append(
150-
GitHubFile(
151-
rel_path=rel_path,
152-
blob_sha=item["sha"],
153-
)
238+
sem = asyncio.Semaphore(_TREE_WALK_CONCURRENCY)
239+
blobs = await _walk_tree_recursive(
240+
c, repo, token, tree["sha"], "", root_prefix, sem
154241
)
155-
return files
242+
243+
return _filter_tree_blobs(blobs, root_prefix, exclude)
156244

157245

158246
async def list_commits(
Collapse file

‎tests/test_github_source.py‎

Copy file name to clipboard
+153Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
from __future__ import annotations
2+
3+
import httpx
4+
import respx
5+
6+
from server.indexer.github_source import list_github_files
7+
8+
_API = "https://api.github.com"
9+
_REPO = "owner/repo"
10+
_REF = "main"
11+
12+
13+
def _tree_url(sha: str) -> str:
14+
return f"{_API}/repos/{_REPO}/git/trees/{sha}"
15+
16+
17+
@respx.mock
18+
async def test_non_truncated_filters_blobs() -> None:
19+
respx.get(_tree_url(_REF)).mock(
20+
return_value=httpx.Response(
21+
200,
22+
json={
23+
"sha": "roottree",
24+
"truncated": False,
25+
"tree": [
26+
{"path": "src/main.py", "type": "blob", "sha": "a"},
27+
{"path": "README.md", "type": "blob", "sha": "b"},
28+
{"path": "notes.txt", "type": "blob", "sha": "c"},
29+
{"path": "src", "type": "tree", "sha": "t1"},
30+
],
31+
},
32+
)
33+
)
34+
35+
files = await list_github_files("tok", _REPO, _REF, "svc", exclude=[])
36+
37+
by_path = {f.rel_path: f.blob_sha for f in files}
38+
assert by_path == {"src/main.py": "a", "README.md": "b"}
39+
40+
41+
@respx.mock
42+
async def test_non_truncated_applies_root_and_exclude() -> None:
43+
respx.get(_tree_url(_REF)).mock(
44+
return_value=httpx.Response(
45+
200,
46+
json={
47+
"sha": "roottree",
48+
"truncated": False,
49+
"tree": [
50+
{"path": "src/app.py", "type": "blob", "sha": "a"},
51+
{"path": "src/test_app.py", "type": "blob", "sha": "b"},
52+
{"path": "docs/guide.py", "type": "blob", "sha": "c"},
53+
],
54+
},
55+
)
56+
)
57+
58+
files = await list_github_files(
59+
"tok", _REPO, _REF, "svc", exclude=["test_*.py"], root="src"
60+
)
61+
62+
by_path = {f.rel_path: f.blob_sha for f in files}
63+
assert by_path == {"app.py": "a"}
64+
65+
66+
@respx.mock
67+
async def test_truncated_falls_back_to_recursive_walk() -> None:
68+
# Recursive response is truncated and omits src/util.py entirely.
69+
respx.get(_tree_url(_REF)).mock(
70+
return_value=httpx.Response(
71+
200,
72+
json={
73+
"sha": "roottree",
74+
"truncated": True,
75+
"tree": [{"path": "main.py", "type": "blob", "sha": "m"}],
76+
},
77+
)
78+
)
79+
respx.get(_tree_url("roottree")).mock(
80+
return_value=httpx.Response(
81+
200,
82+
json={
83+
"truncated": False,
84+
"tree": [
85+
{"path": "main.py", "type": "blob", "sha": "m"},
86+
{"path": "src", "type": "tree", "sha": "t_src"},
87+
],
88+
},
89+
)
90+
)
91+
respx.get(_tree_url("t_src")).mock(
92+
return_value=httpx.Response(
93+
200,
94+
json={
95+
"truncated": False,
96+
"tree": [
97+
{"path": "app.py", "type": "blob", "sha": "app"},
98+
{"path": "util.py", "type": "blob", "sha": "u"},
99+
],
100+
},
101+
)
102+
)
103+
104+
files = await list_github_files("tok", _REPO, _REF, "svc", exclude=[])
105+
106+
by_path = {f.rel_path: f.blob_sha for f in files}
107+
assert by_path == {"main.py": "m", "src/app.py": "app", "src/util.py": "u"}
108+
109+
110+
@respx.mock
111+
async def test_truncated_walk_prunes_subtrees_outside_root() -> None:
112+
respx.get(_tree_url(_REF)).mock(
113+
return_value=httpx.Response(
114+
200, json={"sha": "roottree", "truncated": True, "tree": []}
115+
)
116+
)
117+
respx.get(_tree_url("roottree")).mock(
118+
return_value=httpx.Response(
119+
200,
120+
json={
121+
"truncated": False,
122+
"tree": [
123+
{"path": "src", "type": "tree", "sha": "t_src"},
124+
{"path": "docs", "type": "tree", "sha": "t_docs"},
125+
],
126+
},
127+
)
128+
)
129+
src_route = respx.get(_tree_url("t_src")).mock(
130+
return_value=httpx.Response(
131+
200,
132+
json={
133+
"truncated": False,
134+
"tree": [{"path": "app.py", "type": "blob", "sha": "app"}],
135+
},
136+
)
137+
)
138+
docs_route = respx.get(_tree_url("t_docs")).mock(
139+
return_value=httpx.Response(
140+
200,
141+
json={
142+
"truncated": False,
143+
"tree": [{"path": "guide.py", "type": "blob", "sha": "g"}],
144+
},
145+
)
146+
)
147+
148+
files = await list_github_files("tok", _REPO, _REF, "svc", exclude=[], root="src")
149+
150+
by_path = {f.rel_path: f.blob_sha for f in files}
151+
assert by_path == {"app.py": "app"}
152+
assert src_route.called
153+
assert not docs_route.called

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.